@kaifronsdal/transcript-viewer/src/lib/server/data-loading/bulk-loader.ts

A web-based viewer for AI conversation transcripts with rollback support

import type { TranscriptDisplay } from '$lib/shared/types';
import { scanDirectoryForTranscripts, checkPathAccess } from './file-scanner';
import { processTranscriptFiles } from './file-processor';
import type { LoadingResult, LoadingError, LoadingStats } from './types';

// Server-side check
if (typeof window !== 'undefined') {
  throw new Error('bulk-loader can only be used on the server side');
}

/**
 * Load transcripts with full error reporting and statistics
 */
export async function loadRealTranscriptsWithErrors(
  outputDir: string = './transcripts',
  includeErrors: boolean = true
): Promise<LoadingResult> {
  console.log(`📂 [DEBUG] Starting transcript loading from: ${outputDir}`);
  console.log(`📊 [DEBUG] Include errors: ${includeErrors}`);
  
  // Check if directory exists
  const directoryExists = await checkPathAccess(outputDir);
  if (!directoryExists) {
    const error: LoadingError = {
      type: 'file_not_found',
      message: `Directory not found: ${outputDir}`,
      file: outputDir
    };
    
    return {
      transcripts: [],
      errors: [error],
      directories: [],
      stats: {
        totalFiles: 0,
        successfulFiles: 0,
        failedFiles: 1,
        validationErrors: 0,
        parseErrors: 0,
        totalDirectories: 0,
        emptyDirectories: 0,
      }
    };
  }

  // Scan directory for transcript files
  console.log('🔍 [DEBUG] Scanning directory for transcript files...');
  const scanResult = await scanDirectoryForTranscripts(outputDir);
  console.log(`📁 [DEBUG] Found ${scanResult.files.length} files and ${scanResult.directories.length} directories`);
  
  // Process transcript files
  console.log('⚙️ [DEBUG] Processing transcript files...');
  const processResult = await processTranscriptFiles(scanResult.files, false, 10);
  console.log(`✅ [DEBUG] Successfully processed ${processResult.transcripts.length} transcripts`);
  console.log(`❌ [DEBUG] Failed to process ${processResult.errors.length} files`);
  
  // Combine all errors
  const allErrors = [...scanResult.errors, ...processResult.errors];
  
  // Calculate statistics
  const stats: LoadingStats = {
    totalFiles: scanResult.files.length,
    successfulFiles: processResult.transcripts.length,
    failedFiles: processResult.errors.length,
    validationErrors: processResult.errors.filter(e => e.type === 'validation_error').length,
    parseErrors: processResult.errors.filter(e => e.type === 'parse_error').length,
    totalDirectories: scanResult.directories.length,
    emptyDirectories: scanResult.directories.filter(d => d.isEmpty).length,
  };
  
  console.log('📈 [DEBUG] Final statistics:', stats);
  
  return {
    transcripts: processResult.transcripts,
    errors: includeErrors ? allErrors : [],
    directories: scanResult.directories,
    stats
  };
}

/**
 * Load transcript metadata only (faster, less memory)
 */
export async function loadRealTranscriptsMetadataOnly(
  outputDir: string = './transcripts',
  includeErrors: boolean = true
): Promise<LoadingResult> {
  console.log(`📂 [DEBUG] Starting metadata-only loading from: ${outputDir}`);
  console.log(`📊 [DEBUG] Include errors: ${includeErrors}`);
  
  // Check if directory exists
  const directoryExists = await checkPathAccess(outputDir);
  if (!directoryExists) {
    const error: LoadingError = {
      type: 'file_not_found',
      message: `Directory not found: ${outputDir}`,
      file: outputDir
    };
    
    return {
      transcripts: [],
      errors: [error],
      directories: [],
      stats: {
        totalFiles: 0,
        successfulFiles: 0,
        failedFiles: 1,
        validationErrors: 0,
        parseErrors: 0,
        totalDirectories: 0,
        emptyDirectories: 0,
      }
    };
  }

  // Scan directory for transcript files
  console.log('🔍 [DEBUG] Scanning directory for transcript files...');
  const scanResult = await scanDirectoryForTranscripts(outputDir);
  console.log(`📁 [DEBUG] Found ${scanResult.files.length} files and ${scanResult.directories.length} directories`);
  
  // Process transcript files (metadata only)
  console.log('⚙️ [DEBUG] Processing transcript files (metadata only)...');
  const processResult = await processTranscriptFiles(scanResult.files, true, 10);
  console.log(`✅ [DEBUG] Successfully processed ${processResult.transcripts.length} transcripts (metadata only)`);
  console.log(`❌ [DEBUG] Failed to process ${processResult.errors.length} files`);
  
  // Combine all errors
  const allErrors = [...scanResult.errors, ...processResult.errors];
  
  // Calculate statistics
  const stats: LoadingStats = {
    totalFiles: scanResult.files.length,
    successfulFiles: processResult.transcripts.length,
    failedFiles: processResult.errors.length,
    validationErrors: processResult.errors.filter(e => e.type === 'validation_error').length,
    parseErrors: processResult.errors.filter(e => e.type === 'parse_error').length,
    totalDirectories: scanResult.directories.length,
    emptyDirectories: scanResult.directories.filter(d => d.isEmpty).length,
  };
  
  console.log('📈 [DEBUG] Final statistics:', stats);
  
  return {
    transcripts: processResult.transcripts,
    errors: includeErrors ? allErrors : [],
    directories: scanResult.directories,
    stats
  };
}

/**
 * Backward compatibility wrapper - returns only transcripts (legacy API)
 */
export async function loadRealTranscripts(outputDir: string = './transcripts'): Promise<TranscriptDisplay[]> {
  const result = await loadRealTranscriptsWithErrors(outputDir);
  
  // Log errors but return only successful transcripts for backward compatibility
  if (result.errors.length > 0) {
    console.warn(`⚠️ [DEBUG] ${result.errors.length} errors occurred during loading (suppressed for backward compatibility)`);
  }
  
  return result.transcripts;
}