@kaifronsdal/transcript-viewer/src/lib/server/data-loading/file-processor.ts

A web-based viewer for AI conversation transcripts with rollback support

import { promises as fs } from 'fs';
import path from 'path';
import type { TranscriptDisplay } from '$lib/shared/types';
import { validateTranscriptData, formatValidationErrors } from '$lib/shared/schema-validator';
import { createTranscriptDisplay, extractTranscriptMetadata } from './transcript-transformer';
import type { LoadingError } from './types';

// Server-side check
if (typeof window !== 'undefined') {
  throw new Error('file-processor can only be used on the server side');
}

/**
 * Process a single transcript file
 */
export async function processTranscriptFile(
  filePath: string,
  metadataOnly: boolean = false
): Promise<{ transcript?: TranscriptDisplay; error?: LoadingError }> {
  try {
    // Read and parse the JSON file
    const fileContent = await fs.readFile(filePath, 'utf8');
    let jsonData;
    
    try {
      jsonData = JSON.parse(fileContent);
    } catch (parseError) {
      return {
        error: {
          type: 'parse_error',
          message: `Invalid JSON in file: ${filePath}`,
          file: filePath,
          details: parseError instanceof Error ? parseError.message : String(parseError)
        }
      };
    }

    // Validate the transcript
    const validationResult = validateTranscriptData(jsonData);
    
    if (!validationResult.isValid) {
      return {
        error: {
          type: 'validation_error',
          message: `Schema validation failed for file: ${filePath}`,
          file: filePath,
          validationErrors: formatValidationErrors(validationResult.errors || []),
          partialData: validationResult.partialData
        }
      };
    }

    // Create display object
    const transcript = metadataOnly 
      ? extractTranscriptMetadata(validationResult.transcript, filePath)
      : createTranscriptDisplay(validationResult.transcript, filePath);
    
    return { transcript };
    
  } catch (error) {
    const errorType = (error as any)?.code === 'ENOENT' ? 'file_not_found' : 'unknown_error';
    
    return {
      error: {
        type: errorType,
        message: `Failed to process file: ${filePath}`,
        file: filePath,
        details: error instanceof Error ? error.message : String(error)
      }
    };
  }
}

/**
 * Process multiple transcript files in parallel
 */
export async function processTranscriptFiles(
  filePaths: string[],
  metadataOnly: boolean = false,
  concurrency: number = 10
): Promise<{ transcripts: TranscriptDisplay[]; errors: LoadingError[] }> {
  const transcripts: TranscriptDisplay[] = [];
  const errors: LoadingError[] = [];
  
  // Process files in batches to avoid overwhelming the system
  for (let i = 0; i < filePaths.length; i += concurrency) {
    const batch = filePaths.slice(i, i + concurrency);
    
    const batchPromises = batch.map(filePath => 
      processTranscriptFile(filePath, metadataOnly)
    );
    
    const batchResults = await Promise.all(batchPromises);
    
    for (const result of batchResults) {
      if (result.transcript) {
        transcripts.push(result.transcript);
      }
      if (result.error) {
        errors.push(result.error);
      }
    }
  }
  
  return { transcripts, errors };
}