dadacat-lambda-pipeline/src/PipelineOrchestrator.js

JavaScript client for the dadacat 3-lambda image generation pipeline

/**
 * Main orchestrator for the dadacat lambda pipeline
 * @module PipelineOrchestrator
 */

import { DadacatClient } from './clients/DadacatClient.js';
import { ImageGenClient } from './clients/ImageGenClient.js';
import { B2UploadClient } from './clients/B2UploadClient.js';
import { ImageGenerationOptions } from './ImageGenerationOptions.js';

/**
 * Result of a pipeline execution
 * @typedef {Object} PipelineResult
 * @property {boolean} success - Whether the pipeline completed successfully
 * @property {string} testId - Unique identifier for this pipeline run
 * @property {string} timestamp - ISO timestamp of when the pipeline started
 * @property {Object} inputData - Original input data
 * @property {Array<Object>} steps - Array of step results
 * @property {Object} [finalResult] - Final result object
 * @property {string} [error] - Error message if pipeline failed
 * @property {number} [totalDuration] - Total duration in seconds
 */

/**
 * Configuration object for the pipeline
 * @typedef {Object} PipelineConfig
 * @property {string} dadacatUrl - URL for the dadacat-agent-x86 lambda
 * @property {string} imageGenUrl - URL for the ImageGenerationProcessor API
 * @property {string} b2UploadUrl - URL for the ImageB2Uploader lambda
 * @property {number} [maxRetries=3] - Maximum number of retries for failed requests
 * @property {number} [retryDelay=5] - Delay between retries in seconds
 * @property {number} [timeout=300] - Maximum timeout for operations in seconds
 * @property {number} [pollingInterval=5] - Polling interval for job status checks in seconds
 */

/**
 * Main orchestrator for lambda pipeline operations
 */
export class PipelineOrchestrator {
  /**
   * Initialize with configuration
   * @param {PipelineConfig} config - Pipeline configuration
   */
  constructor(config) {
    this.config = {
      maxRetries: 3,
      retryDelay: 5,
      timeout: 300,
      pollingInterval: 5,
      ...config
    };

    // Initialize lambda clients
    this.dadacatClient = new DadacatClient(this.config.dadacatUrl);
    this.imageGenClient = new ImageGenClient(this.config.imageGenUrl);
    this.b2UploadClient = new B2UploadClient(this.config.b2UploadUrl);
  }

  /**
   * Run basic sequential pipeline: prompt -> agent -> image -> B2
   * @param {string} prompt - Input prompt for the agent
   * @returns {Promise<PipelineResult>} Pipeline execution result
   */
  async runSequentialPipeline(prompt) {
    const testId = `sequential_${Date.now()}`;
    const startTime = Date.now();

    console.log(`[PipelineOrchestrator] Starting sequential pipeline ${testId}`);
    console.log(`[PipelineOrchestrator] Input prompt type: ${typeof prompt}`);
    console.log(`[PipelineOrchestrator] Input prompt value: "${prompt}"`);
    console.log(`[PipelineOrchestrator] Input prompt length: ${prompt ? prompt.length : 'null/undefined'}`);

    const steps = [];
    let error = null;

    try {
      // Step 1: dadacat-agent-x86
      const step1Start = Date.now();
      console.log('[PipelineOrchestrator] Step 1: Calling dadacat-agent-x86');
      console.log(`[PipelineOrchestrator] Passing prompt to dadacat: "${prompt}"`);

      const agentResponse = await this.dadacatClient.generateResponse(prompt);
      const step1Duration = (Date.now() - step1Start) / 1000;

      steps.push({
        step: 1,
        lambda: 'dadacat-agent-x86',
        input: { prompt },
        output: agentResponse,
        duration: step1Duration,
        success: agentResponse.status === 'success'
      });

      if (agentResponse.status !== 'success') {
        throw new Error(`Step 1 failed: ${JSON.stringify(agentResponse)}`);
      }

      const agentText = agentResponse.response || '';
      console.log(`Step 1 completed in ${step1Duration.toFixed(2)}s: ${agentText.substring(0, 100)}...`);

      // Step 2: ImageGenerationProcessor
      const step2Start = Date.now();
      console.log('Step 2: Calling ImageGenerationProcessor');

      const imageRequest = {
        prompt: agentText,
        options: {
          model: 'dall-e-3',
          size: '1024x1024',
          quality: 'standard'
        }
      };

      const imageQueueResponse = await this.imageGenClient.generateImage(imageRequest);
      
      if (!imageQueueResponse.success) {
        throw new Error(`Step 2 failed to queue: ${JSON.stringify(imageQueueResponse)}`);
      }

      const jobId = imageQueueResponse.data?.job_id;
      const runId = imageQueueResponse.data?.run_id;

      console.log(`Step 2 queued: job_id=${jobId}, run_id=${runId}`);
      console.log('Polling for image generation completion...');

      const imageResult = await this.imageGenClient.pollForCompletion(
        jobId,
        this.config.timeout,
        this.config.pollingInterval
      );

      const step2Duration = (Date.now() - step2Start) / 1000;

      steps.push({
        step: 2,
        lambda: 'ImageGenerationProcessor',
        input: imageRequest,
        output: imageResult,
        duration: step2Duration,
        success: imageResult.success && imageResult.data?.status === 'completed'
      });

      if (!imageResult.success || imageResult.data?.status !== 'completed') {
        throw new Error(`Step 2 failed: ${JSON.stringify(imageResult)}`);
      }

      const imageUrl = imageResult.data?.url;
      console.log(`Step 2 completed in ${step2Duration.toFixed(2)}s: ${imageUrl}`);

      // Step 3: ImageB2Uploader
      const step3Start = Date.now();
      console.log('Step 3: Calling ImageB2Uploader');

      const uploadRequest = {
        job_id: jobId,
        run_id: runId,
        image_url: imageUrl,
        metadata: {
          original_prompt: prompt,
          agent_response: agentText,
          generation_timestamp: new Date().toISOString()
        }
      };

      const uploadResult = await this.b2UploadClient.uploadImage(uploadRequest);
      const step3Duration = (Date.now() - step3Start) / 1000;

      steps.push({
        step: 3,
        lambda: 'ImageB2Uploader',
        input: uploadRequest,
        output: uploadResult,
        duration: step3Duration,
        success: uploadResult.success
      });

      if (!uploadResult.success) {
        throw new Error(`Step 3 failed: ${JSON.stringify(uploadResult)}`);
      }

      const b2Url = uploadResult.data?.b2_url;
      console.log(`Step 3 completed in ${step3Duration.toFixed(2)}s: ${b2Url}`);

      const totalDuration = (Date.now() - startTime) / 1000;

      return {
        success: true,
        testId,
        timestamp: new Date(startTime).toISOString(),
        inputData: { prompt },
        steps,
        finalResult: {
          agentResponse: agentText,
          imageUrl,
          b2Url,
          jobId,
          runId
        },
        totalDuration
      };

    } catch (err) {
      error = err.message;
      console.error(`Pipeline failed: ${error}`);

      const totalDuration = (Date.now() - startTime) / 1000;

      return {
        success: false,
        testId,
        timestamp: new Date(startTime).toISOString(),
        inputData: { prompt },
        steps,
        error,
        totalDuration
      };
    }
  }

  /**
   * Run metadata-aware pipeline with custom data
   * @param {Object} inputData - Input data with metadata
   * @param {string} inputData.original_prompt - The original prompt
   * @param {string} [inputData.batch_id] - Batch ID for tracking
   * @param {Object} [inputData.user_context] - User context information
   * @returns {Promise<PipelineResult>} Pipeline execution result
   */
  async runMetadataPipeline(inputData) {
    const testId = `metadata_${Date.now()}`;
    const startTime = Date.now();

    console.log(`Starting metadata pipeline ${testId}`);
    console.log('Input data:', JSON.stringify(inputData, null, 2));

    const steps = [];
    let error = null;

    try {
      const { original_prompt, batch_id, user_context } = inputData;

      // Step 1: dadacat-agent-x86 with metadata
      const step1Start = Date.now();
      console.log('Step 1: Calling dadacat-agent-x86 with metadata');

      const agentResponse = await this.dadacatClient.generateResponse(original_prompt);
      const step1Duration = (Date.now() - step1Start) / 1000;

      if (agentResponse.status !== 'success') {
        throw new Error(`Step 1 failed: ${JSON.stringify(agentResponse)}`);
      }

      const agentText = agentResponse.response || '';

      // Step 2: ImageGenerationProcessor with metadata
      const step2Start = Date.now();
      console.log('Step 2: Calling ImageGenerationProcessor with metadata');

      const imageRequest = {
        prompt: agentText,
        batch_id,
        options: {
          model: 'dall-e-3',
          size: '1024x1024',
          quality: 'standard'
        }
      };

      const imageQueueResponse = await this.imageGenClient.generateImage(imageRequest);
      
      if (!imageQueueResponse.success) {
        throw new Error(`Step 2 failed to queue: ${JSON.stringify(imageQueueResponse)}`);
      }

      const jobId = imageQueueResponse.data?.job_id;
      const runId = imageQueueResponse.data?.run_id;

      const imageResult = await this.imageGenClient.pollForCompletion(
        jobId,
        this.config.timeout,
        this.config.pollingInterval
      );

      const step2Duration = (Date.now() - step2Start) / 1000;

      if (!imageResult.success || imageResult.data?.status !== 'completed') {
        throw new Error(`Step 2 failed: ${JSON.stringify(imageResult)}`);
      }

      const imageUrl = imageResult.data?.url;

      // Step 3: ImageB2Uploader with metadata
      const step3Start = Date.now();
      console.log('Step 3: Calling ImageB2Uploader with metadata');

      const uploadRequest = {
        job_id: jobId,
        run_id: runId,
        image_url: imageUrl,
        metadata: {
          original_prompt,
          agent_response: agentText,
          batch_id,
          user_context,
          generation_timestamp: new Date().toISOString()
        }
      };

      const uploadResult = await this.b2UploadClient.uploadImage(uploadRequest);
      const step3Duration = (Date.now() - step3Start) / 1000;

      if (!uploadResult.success) {
        throw new Error(`Step 3 failed: ${JSON.stringify(uploadResult)}`);
      }

      const b2Url = uploadResult.data?.b2_url;
      const totalDuration = (Date.now() - startTime) / 1000;

      steps.push(
        {
          step: 1,
          lambda: 'dadacat-agent-x86',
          input: { prompt: original_prompt },
          output: agentResponse,
          duration: step1Duration,
          success: true
        },
        {
          step: 2,
          lambda: 'ImageGenerationProcessor',
          input: imageRequest,
          output: imageResult,
          duration: step2Duration,
          success: true
        },
        {
          step: 3,
          lambda: 'ImageB2Uploader',
          input: uploadRequest,
          output: uploadResult,
          duration: step3Duration,
          success: true
        }
      );

      return {
        success: true,
        testId,
        timestamp: new Date(startTime).toISOString(),
        inputData,
        steps,
        finalResult: {
          agentResponse: agentText,
          imageUrl,
          b2Url,
          jobId,
          runId,
          metadata: {
            batch_id,
            user_context,
            original_prompt
          }
        },
        totalDuration
      };

    } catch (err) {
      error = err.message;
      console.error(`Metadata pipeline failed: ${error}`);

      const totalDuration = (Date.now() - startTime) / 1000;

      return {
        success: false,
        testId,
        timestamp: new Date(startTime).toISOString(),
        inputData,
        steps,
        error,
        totalDuration
      };
    }
  }

  /**
   * Run configurable pipeline with comprehensive metadata flow
   * @param {Object} inputData - Comprehensive input data
   * @param {string} inputData.human_prompt - Original user prompt
   * @param {string} [inputData.batch_id] - Batch identifier for tracking
   * @param {string} [inputData.additional_prompt] - Additional context for image generation
   * @param {Object} [inputData.options] - Image generation options
   * @param {string} [inputData.bucket] - Override default B2 bucket
   * @param {string} [inputData.folder] - Override default B2 folder  
   * @param {string} [inputData.artproject] - Art project identifier
   * @param {string} [inputData.on_website] - Website display flag
   * @param {Object} [inputData.custom_metadata] - Additional custom metadata
   * @returns {Promise<PipelineResult>} Pipeline execution result
   */
  async runConfigurablePipeline(inputData) {
    const testId = `configurable_${Date.now()}`;
    const startTime = Date.now();

    console.log(`Starting configurable pipeline ${testId}`);
    console.log('Input data:', JSON.stringify(inputData, null, 2));

    const steps = [];
    let error = null;

    try {
      // Extract and validate input data
      const {
        human_prompt,
        batch_id,
        additional_prompt,
        options = {},
        bucket,
        folder,
        artproject,
        on_website = 'yes',
        custom_metadata = {}
      } = inputData;

      if (!human_prompt) {
        throw new Error('human_prompt is required');
      }

      // Validate and clean options based on model
      const model = options.model || 'dall-e-3';
      const optionValidation = ImageGenerationOptions.validateOptions(options, model);
      
      if (optionValidation.errors.length > 0) {
        throw new Error(`Invalid options: ${optionValidation.errors.join(', ')}`);
      }
      
      // Log warnings if any
      if (optionValidation.warnings.length > 0) {
        console.warn('Option warnings:', optionValidation.warnings.join(', '));
      }
      
      const validatedOptions = optionValidation.validatedOptions;

      // Step 1: dadacat-agent-x86 (only accepts prompt)
      const step1Start = Date.now();
      console.log('[PipelineOrchestrator] Step 1: Calling dadacat-agent-x86');
      console.log(`[PipelineOrchestrator] human_prompt type: ${typeof human_prompt}`);
      console.log(`[PipelineOrchestrator] human_prompt value: "${human_prompt}"`);
      console.log(`[PipelineOrchestrator] human_prompt length: ${human_prompt ? human_prompt.length : 'null/undefined'}`);

      const agentResponse = await this.dadacatClient.generateResponse(human_prompt);
      const step1Duration = (Date.now() - step1Start) / 1000;

      steps.push({
        step: 1,
        lambda: 'dadacat-agent-x86',
        input: { prompt: human_prompt },
        output: agentResponse,
        duration: step1Duration,
        success: agentResponse.status === 'success'
      });

      if (agentResponse.status !== 'success') {
        throw new Error(`Step 1 failed: ${JSON.stringify(agentResponse)}`);
      }

      const agentText = agentResponse.response || '';
      console.log(`Step 1 completed in ${step1Duration.toFixed(2)}s`);

      // Step 2: ImageGenerationProcessor with metadata
      const step2Start = Date.now();
      console.log('Step 2: Calling ImageGenerationProcessor');

      const imageRequest = {
        prompt: agentText,
        ...(additional_prompt && { additional_prompt }),
        ...(batch_id && { batch_id }),
        options: validatedOptions
      };

      const imageQueueResponse = await this.imageGenClient.generateImage(imageRequest);
      
      if (!imageQueueResponse.success) {
        throw new Error(`Step 2 failed to queue: ${JSON.stringify(imageQueueResponse)}`);
      }

      const jobId = imageQueueResponse.data?.job_id;
      const runId = imageQueueResponse.data?.run_id;
      const returnedBatchId = imageQueueResponse.data?.batch_id;

      console.log(`Step 2 queued: job_id=${jobId}, run_id=${runId}, batch_id=${returnedBatchId}`);
      console.log('Polling for image generation completion...');

      const imageResult = await this.imageGenClient.pollForCompletion(
        jobId,
        this.config.timeout,
        this.config.pollingInterval
      );

      const step2Duration = (Date.now() - step2Start) / 1000;

      steps.push({
        step: 2,
        lambda: 'ImageGenerationProcessor',
        input: imageRequest,
        output: imageResult,
        duration: step2Duration,
        success: imageResult.success && imageResult.data?.status === 'completed'
      });

      if (!imageResult.success || imageResult.data?.status !== 'completed') {
        throw new Error(`Step 2 failed: ${JSON.stringify(imageResult)}`);
      }

      const imageUrl = imageResult.data?.url;
      const timestamp = imageResult.data?.timestamp;
      console.log(`Step 2 completed in ${step2Duration.toFixed(2)}s: ${imageUrl}`);

      // Step 3: ImageB2Uploader with comprehensive metadata
      const step3Start = Date.now();
      console.log('Step 3: Calling ImageB2Uploader');

      // Build comprehensive B2Upload request following the final understanding
      const uploadRequest = {
        // Required fields
        job_id: jobId,
        run_id: runId,
        image_url: imageUrl,
        
        // Optional but important for index.json
        ...(returnedBatchId && { batch_id: returnedBatchId }),
        prompt: additional_prompt ? `${additional_prompt} ${agentText}` : agentText,
        ...(additional_prompt && { additional_prompt }),
        human_prompt,
        options: validatedOptions,
        ...(artproject && { artproject }),
        ...(timestamp && { timestamp }),
        ...(bucket && { bucket }),
        ...(folder && { folder }),
        ...(on_website && { on_website })
      };

      // Add any custom metadata fields
      Object.keys(custom_metadata).forEach(key => {
        if (!uploadRequest.hasOwnProperty(key)) {
          uploadRequest[key] = custom_metadata[key];
        }
      });

      const uploadResult = await this.b2UploadClient.uploadImage(uploadRequest);
      const step3Duration = (Date.now() - step3Start) / 1000;

      steps.push({
        step: 3,
        lambda: 'ImageB2Uploader',
        input: uploadRequest,
        output: uploadResult,
        duration: step3Duration,
        success: uploadResult.success
      });

      if (!uploadResult.success) {
        throw new Error(`Step 3 failed: ${JSON.stringify(uploadResult)}`);
      }

      const b2Url = uploadResult.data?.b2_url;
      console.log(`Step 3 completed in ${step3Duration.toFixed(2)}s: ${b2Url}`);

      const totalDuration = (Date.now() - startTime) / 1000;

      return {
        success: true,
        testId,
        timestamp: new Date(startTime).toISOString(),
        inputData,
        steps,
        finalResult: {
          agentResponse: agentText,
          imageUrl,
          b2Url,
          jobId,
          runId,
          metadataFlow: {
            human_prompt,
            dadacat_response: agentText,
            combined_prompt: uploadRequest.prompt,
            batch_id: returnedBatchId,
            additional_prompt,
            options: validatedOptions,
            artproject,
            custom_metadata
          }
        },
        totalDuration
      };

    } catch (err) {
      error = err.message;
      console.error(`Configurable pipeline failed: ${error}`);

      const totalDuration = (Date.now() - startTime) / 1000;

      return {
        success: false,
        testId,
        timestamp: new Date(startTime).toISOString(),
        inputData,
        steps,
        error,
        totalDuration
      };
    }
  }

  /**
   * Test connection to all lambda functions
   * @returns {Promise<Object>} Connection test results
   */
  async testConnections() {
    console.log('Testing connections to all lambda functions...');

    const results = {};

    try {
      results.dadacat = await this.dadacatClient.testConnection();
      console.log(`dadacat-agent-x86: ${results.dadacat ? '✅' : '❌'}`);
    } catch (error) {
      results.dadacat = false;
      console.error(`dadacat-agent-x86 test failed: ${error.message}`);
    }

    try {
      results.imageGen = await this.imageGenClient.testConnection();
      console.log(`ImageGenerationProcessor: ${results.imageGen ? '✅' : '❌'}`);
    } catch (error) {
      results.imageGen = false;
      console.error(`ImageGenerationProcessor test failed: ${error.message}`);
    }

    try {
      results.b2Upload = await this.b2UploadClient.testConnection();
      console.log(`ImageB2Uploader: ${results.b2Upload ? '✅' : '❌'}`);
    } catch (error) {
      results.b2Upload = false;
      console.error(`ImageB2Uploader test failed: ${error.message}`);
    }

    const allPassing = Object.values(results).every(Boolean);
    
    return {
      success: allPassing,
      results,
      summary: `${Object.values(results).filter(Boolean).length}/3 connections successful`
    };
  }
}