UNPKG

contaigents

Version:

Modular AI Content Ecosystem with Audio Generation

786 lines (761 loc) • 35.5 kB
import { BaseTool } from './BaseTool.js'; import { FileService } from '../fileService.js'; import { ImageGenerationTool } from './ImageGenerationTool.js'; import { StateManager } from '../stateManager.js'; import fs from 'fs/promises'; import path from 'path'; /** * CinematicWorkflowTool - Handles cinematic script workflows with automatic image generation * * Features: * - Version control for scene files (moves old versions to old_versions folder) * - Automatic image generation for scene visual sketches * - Rollback capability when generation fails * - Scene file parsing and visual description extraction * - State-aware workflow management (respects story project state machine) * - User preference awareness (auto-sketch generation, style preferences) */ export class CinematicWorkflowTool extends BaseTool { constructor(baseDir) { super(); this.baseDir = baseDir || process.cwd(); this.fileService = new FileService(this.baseDir); this.imageGenerationTool = new ImageGenerationTool(this.baseDir); this.stateManager = new StateManager(this.baseDir); } getName() { return 'cinematic_workflow'; } getDescription() { return 'Manages cinematic script workflows with enhanced automatic storybook-style sketch generation, version control, and rollback capabilities for scene files. Automatically creates detailed pencil sketch prompts that help directors/producers understand the story with comprehensive scene descriptions, choreography, focus, and character details.'; } getParameters() { return [ { name: 'operation', type: 'string', description: 'Operation to perform: create_scene, update_scene, generate_scene_images, rollback_scene, confirm_outline, start_screenplay, enter_iteration', required: true }, { name: 'scene_file_path', type: 'string', description: 'Path to the scene file (e.g., "short_film_script.md")', required: false }, { name: 'content', type: 'string', description: 'Scene file content (required for create_scene and update_scene operations)', required: false }, { name: 'auto_generate_images', type: 'boolean', description: 'Whether to automatically generate images for visual sketches (default: respects user preference)', required: false }, { name: 'image_style', type: 'string', description: 'Style for generated images (storyboard, cinematic, concept_art, sketch, etc.) - default: respects user preference', required: false }, { name: 'rollback_version', type: 'string', description: 'Version timestamp to rollback to (for rollback_scene operation)', required: false } ]; } async execute(parameters) { const validation = this.validateParameters(parameters); if (validation) return validation; const { operation, scene_file_path, content, auto_generate_images, image_style, rollback_version } = parameters; try { // Check if we're in a story project const hasStoryProject = await this.stateManager.hasStoryProject(); // Get user preferences if in story project let effectiveAutoGenerate = auto_generate_images; let effectiveStyle = image_style; if (hasStoryProject) { const autoSketchPref = await this.stateManager.isAutoSketchEnabled(); const stylePref = await this.stateManager.getSketchStyle(); // Use user preferences as defaults if not explicitly provided if (effectiveAutoGenerate === undefined) { effectiveAutoGenerate = autoSketchPref; } if (!effectiveStyle) { effectiveStyle = stylePref || 'storyboard'; } } else { // Default values when not in story project if (effectiveAutoGenerate === undefined) { effectiveAutoGenerate = true; } if (!effectiveStyle) { effectiveStyle = 'storyboard'; } } // Handle state transition operations if (operation === 'confirm_outline') { return await this.confirmOutline(); } if (operation === 'start_screenplay') { return await this.startScreenplay(); } if (operation === 'enter_iteration') { return await this.enterIteration(); } // Check state constraints for content operations if (hasStoryProject) { const stateCheck = await this.checkStateConstraints(operation); if (!stateCheck.allowed) { return { success: false, error: stateCheck.error || 'Operation not allowed in current workflow state' }; } } switch (operation) { case 'create_scene': return await this.createScene(scene_file_path, content, effectiveAutoGenerate, effectiveStyle); case 'update_scene': return await this.updateScene(scene_file_path, content, effectiveAutoGenerate, effectiveStyle); case 'generate_scene_images': return await this.generateSceneImages(scene_file_path, effectiveStyle); case 'rollback_scene': return await this.rollbackScene(scene_file_path, rollback_version); default: return { success: false, error: `Unknown operation: ${operation}. Supported operations: create_scene, update_scene, generate_scene_images, rollback_scene, confirm_outline, start_screenplay, enter_iteration` }; } } catch (error) { return { success: false, error: `Cinematic workflow operation failed: ${error.message}` }; } } /** * Check if operation is allowed in current workflow state */ async checkStateConstraints(operation) { const currentState = await this.stateManager.getCurrentState(); if (!currentState) { return { allowed: true }; // No state, allow all operations } const constraints = this.stateManager.getStateConstraints(currentState); switch (operation) { case 'create_scene': case 'update_scene': // Screenplay operations if (!constraints.canGenerateScreenplay) { return { allowed: false, error: `Cannot create/update screenplay in ${currentState} state. Current state only allows: ${this.getAllowedOperations(constraints)}` }; } break; case 'generate_scene_images': // Sketch generation if (!constraints.canGenerateSketches) { return { allowed: false, error: `Cannot generate sketches in ${currentState} state. Sketches can only be generated in ITERATION state.` }; } break; case 'rollback_scene': // Rollback is always allowed return { allowed: true }; default: return { allowed: true }; } return { allowed: true }; } /** * Get human-readable list of allowed operations */ getAllowedOperations(constraints) { const allowed = []; if (constraints.canEditOutline) allowed.push('edit outline'); if (constraints.canGenerateScreenplay) allowed.push('create/update screenplay'); if (constraints.canGenerateSketches) allowed.push('generate sketches'); if (constraints.canIterate) allowed.push('iterate on content'); return allowed.join(', ') || 'none'; } /** * State Transition: Confirm outline and move to OUTLINE_CONFIRMED */ async confirmOutline() { try { const hasProject = await this.stateManager.hasStoryProject(); if (!hasProject) { return { success: false, error: 'No story project found. Create a story_outline.md file first.' }; } const currentState = await this.stateManager.getCurrentState(); if (currentState !== 'OUTLINE_DRAFT') { return { success: false, error: `Cannot confirm outline from ${currentState} state. Outline can only be confirmed from OUTLINE_DRAFT state.` }; } await this.stateManager.updateWorkflowState('OUTLINE_CONFIRMED'); return { success: true, message: 'āœ… Outline confirmed! You can now proceed to screenplay writing.', data: { previous_state: 'OUTLINE_DRAFT', new_state: 'OUTLINE_CONFIRMED', next_steps: 'Use start_screenplay operation to begin writing the screenplay.' } }; } catch (error) { return { success: false, error: `Failed to confirm outline: ${error.message}` }; } } /** * State Transition: Start screenplay writing and move to SCREENPLAY_WRITING */ async startScreenplay() { try { const hasProject = await this.stateManager.hasStoryProject(); if (!hasProject) { return { success: false, error: 'No story project found. Create a story_outline.md file first.' }; } const currentState = await this.stateManager.getCurrentState(); if (currentState !== 'OUTLINE_CONFIRMED') { return { success: false, error: `Cannot start screenplay from ${currentState} state. You must confirm the outline first.` }; } await this.stateManager.updateWorkflowState('SCREENPLAY_WRITING'); return { success: true, message: 'āœ… Screenplay writing started! You can now create and update scene files.', data: { previous_state: 'OUTLINE_CONFIRMED', new_state: 'SCREENPLAY_WRITING', next_steps: 'Create scene files using create_scene operation. When complete, use enter_iteration to enable sketch generation.' } }; } catch (error) { return { success: false, error: `Failed to start screenplay: ${error.message}` }; } } /** * State Transition: Enter iteration mode and move to ITERATION */ async enterIteration() { try { const hasProject = await this.stateManager.hasStoryProject(); if (!hasProject) { return { success: false, error: 'No story project found. Create a story_outline.md file first.' }; } const currentState = await this.stateManager.getCurrentState(); if (currentState !== 'SCREENPLAY_WRITING') { return { success: false, error: `Cannot enter iteration from ${currentState} state. Complete screenplay writing first.` }; } await this.stateManager.updateWorkflowState('ITERATION'); const autoSketch = await this.stateManager.isAutoSketchEnabled(); const sketchStyle = await this.stateManager.getSketchStyle(); return { success: true, message: 'āœ… Entered iteration mode! You can now iterate on screenplay and generate sketches.', data: { previous_state: 'SCREENPLAY_WRITING', new_state: 'ITERATION', auto_generate_sketches: autoSketch, sketch_style: sketchStyle, next_steps: 'You can now update scenes and generate sketches. Sketches will be auto-generated based on your preference.' } }; } catch (error) { return { success: false, error: `Failed to enter iteration mode: ${error.message}` }; } } async createScene(filePath, content, autoGenerateImages, imageStyle) { try { // Check if file already exists const fullPath = path.join(this.baseDir, filePath); const fileExists = await this.fileExists(fullPath); if (fileExists) { return { success: false, error: `Scene file already exists: ${filePath}. Use 'update_scene' operation to modify existing files.` }; } // Create the scene file await this.fileService.saveFile({ path: filePath, content }); const result = { operation: 'create_scene', file_path: filePath, content_length: content.length, lines_written: content.split('\n').length, images_generated: [] }; let message = `Successfully created scene file: ${filePath} (${content.split('\n').length} lines, ${content.length} characters)`; // Auto-generate images if requested if (autoGenerateImages) { const imageResult = await this.generateSceneImages(filePath, imageStyle); if (imageResult.success && imageResult.data) { result.images_generated = imageResult.data.images_generated || []; message += `\nšŸŽØ Generated ${result.images_generated.length} scene images automatically`; } else { message += `\nāš ļø Scene file created but image generation failed: ${imageResult.error}`; } } return { success: true, data: result, message }; } catch (error) { return { success: false, error: `Failed to create scene: ${error.message}` }; } } async updateScene(filePath, content, autoGenerateImages, imageStyle) { try { const fullPath = path.join(this.baseDir, filePath); const fileExists = await this.fileExists(fullPath); if (!fileExists) { return { success: false, error: `Scene file does not exist: ${filePath}. Use 'create_scene' operation to create new files.` }; } // Create version backup before updating const backupResult = await this.createVersionBackup(filePath); try { // Update the scene file await this.fileService.saveFile({ path: filePath, content }); const result = { operation: 'update_scene', file_path: filePath, content_length: content.length, lines_written: content.split('\n').length, backup_created: backupResult.backup_path, images_generated: [] }; let message = `Successfully updated scene file: ${filePath} (${content.split('\n').length} lines, ${content.length} characters)`; message += `\nšŸ“¦ Backup created: ${backupResult.backup_path}`; // Auto-generate images if requested if (autoGenerateImages) { const imageResult = await this.generateSceneImages(filePath, imageStyle); if (imageResult.success && imageResult.data) { result.images_generated = imageResult.data.images_generated || []; message += `\nšŸŽØ Generated ${result.images_generated.length} scene images automatically`; } else { // If image generation fails, rollback the file message += `\nāŒ Image generation failed, rolling back changes: ${imageResult.error}`; await this.restoreFromBackup(filePath, backupResult.backup_path); message += `\nšŸ”„ File restored from backup`; return { success: false, error: `Scene update rolled back due to image generation failure: ${imageResult.error}`, data: { rollback_performed: true, backup_restored: backupResult.backup_path } }; } } return { success: true, data: result, message }; } catch (updateError) { // Restore from backup if update fails await this.restoreFromBackup(filePath, backupResult.backup_path); throw new Error(`Update failed and was rolled back: ${updateError.message}`); } } catch (error) { return { success: false, error: `Failed to update scene: ${error.message}` }; } } async generateSceneImages(filePath, imageStyle) { try { // Read the scene file const content = await this.fileService.readFile(filePath); // Parse visual descriptions from the scene file const visualDescriptions = this.extractVisualDescriptions(content); if (visualDescriptions.length === 0) { return { success: false, error: 'No visual descriptions found in scene file. Add "## Visual Sketch Description" or "### Visual Sketch Description" sections with bullet points to generate images.' }; } const generatedImages = []; const imageDir = path.join(path.dirname(filePath), 'images'); // Ensure images directory exists await this.ensureDirectoryExists(imageDir); // Generate images for each visual description for (let i = 0; i < visualDescriptions.length; i++) { const description = visualDescriptions[i]; const imageName = `scene_${i + 1}_${Date.now()}.png`; const imagePath = path.join(imageDir, imageName); // Create enhanced prompt for cinematic image generation const enhancedPrompt = this.createCinematicPrompt(description, imageStyle); try { const imageResult = await this.imageGenerationTool.execute({ prompt: enhancedPrompt, output_path: imagePath, style: imageStyle, width: 1920, height: 1080 }); if (imageResult.success) { generatedImages.push({ description: description.substring(0, 100) + '...', image_path: imagePath, prompt: enhancedPrompt }); } } catch (imageError) { console.warn(`Failed to generate image ${i + 1}: ${imageError.message}`); } } return { success: true, data: { images_generated: generatedImages, total_descriptions: visualDescriptions.length, successful_generations: generatedImages.length }, message: `Generated ${generatedImages.length} out of ${visualDescriptions.length} scene images` }; } catch (error) { return { success: false, error: `Failed to generate scene images: ${error.message}` }; } } async rollbackScene(filePath, versionTimestamp) { try { const oldVersionsDir = path.join(path.dirname(filePath), 'old_versions'); const fileName = path.basename(filePath); if (versionTimestamp) { // Rollback to specific version const backupPath = path.join(oldVersionsDir, `${fileName}.${versionTimestamp}`); await this.restoreFromBackup(filePath, backupPath); return { success: true, message: `Successfully rolled back ${filePath} to version ${versionTimestamp}`, data: { restored_from: backupPath } }; } else { // Rollback to most recent version const backupFiles = await this.getBackupFiles(filePath); if (backupFiles.length === 0) { return { success: false, error: `No backup versions found for ${filePath}` }; } const mostRecentBackup = backupFiles[0]; // Already sorted by timestamp await this.restoreFromBackup(filePath, mostRecentBackup); return { success: true, message: `Successfully rolled back ${filePath} to most recent version`, data: { restored_from: mostRecentBackup } }; } } catch (error) { return { success: false, error: `Failed to rollback scene: ${error.message}` }; } } // Helper methods async fileExists(filePath) { try { await fs.access(filePath); return true; } catch { return false; } } async createVersionBackup(filePath) { const content = await this.fileService.readFile(filePath); const timestamp = new Date().toISOString().replace(/[:.]/g, '-'); const fileName = path.basename(filePath); const oldVersionsDir = path.join(path.dirname(filePath), 'old_versions'); await this.ensureDirectoryExists(oldVersionsDir); const backupPath = path.join(oldVersionsDir, `${fileName}.${timestamp}`); await fs.writeFile(path.join(this.baseDir, backupPath), content, 'utf-8'); return { backup_path: backupPath }; } async restoreFromBackup(filePath, backupPath) { const backupContent = await fs.readFile(path.join(this.baseDir, backupPath), 'utf-8'); await this.fileService.saveFile({ path: filePath, content: backupContent }); } async getBackupFiles(filePath) { const oldVersionsDir = path.join(this.baseDir, path.dirname(filePath), 'old_versions'); const fileName = path.basename(filePath); try { const files = await fs.readdir(oldVersionsDir); return files .filter(file => file.startsWith(fileName + '.')) .map(file => path.join(path.dirname(filePath), 'old_versions', file)) .sort((a, b) => b.localeCompare(a)); // Sort by timestamp descending } catch { return []; } } async ensureDirectoryExists(dirPath) { const fullPath = path.join(this.baseDir, dirPath); try { await fs.mkdir(fullPath, { recursive: true }); } catch (error) { // Directory might already exist, ignore error } } extractVisualDescriptions(content) { const descriptions = []; const lines = content.split('\n'); let inVisualSection = false; for (const line of lines) { // Support both ## and ### for Visual Sketch Description if (line.trim().match(/^#{2,3}\s+Visual Sketch Description/i)) { inVisualSection = true; continue; } if (inVisualSection) { // End section when we hit another heading at the same or higher level if (line.trim().match(/^#{1,3}\s+/) && !line.includes('Visual Sketch Description')) { // End of visual section inVisualSection = false; } else if (line.trim().match(/^-\s+\*\*/)) { // Each bullet point is a separate visual description // Handles both "- **" and "- **" formats descriptions.push(line.trim()); } } } return descriptions; } createCinematicPrompt(description, style) { // Remove markdown formatting and create cinematic prompt const cleanDescription = description .replace(/\*\*(.*?)\*\*/g, '$1') // Remove bold formatting .replace(/- \*\*(.*?)\*\*:/g, '$1:') // Clean bullet points .replace(/\s+/g, ' ') // Normalize whitespace .trim(); // Enhanced style modifiers with detailed storybook sketch guardrails const styleModifiers = { 'cinematic': this.createStoryboardSketchPrompt(cleanDescription), 'concept_art': 'concept art, digital painting, detailed illustration, artistic rendering', 'storyboard': this.createStoryboardSketchPrompt(cleanDescription), 'photographic': 'photorealistic, high quality photography, professional lighting', 'sketch': this.createStoryboardSketchPrompt(cleanDescription) }; const modifier = styleModifiers[style] || this.createStoryboardSketchPrompt(cleanDescription); // For storyboard/sketch styles, return the enhanced prompt directly if (style === 'storyboard' || style === 'sketch' || style === 'cinematic') { return modifier; } return `${cleanDescription}, ${modifier}, high quality, detailed, atmospheric`; } /** * Creates detailed storybook-style pencil sketch prompts with enhanced guardrails * Automatically includes the detailed instructions that help directors/producers understand the story */ createStoryboardSketchPrompt(sceneDescription) { // Extract key elements from the scene description for enhanced prompting const hasCharacters = /character|person|man|woman|detective|protagonist|antagonist/i.test(sceneDescription); const hasLocation = /room|house|street|lab|office|exterior|interior/i.test(sceneDescription); const hasAction = /walking|running|sitting|standing|examining|looking|moving/i.test(sceneDescription); const hasLighting = /light|shadow|glow|bright|dark|illuminat/i.test(sceneDescription); const hasCameraWork = /wide shot|close-up|medium shot|dolly|pan|tilt/i.test(sceneDescription); // Build comprehensive storybook sketch prompt with automatic guardrails let enhancedPrompt = `Create a single pencil sketch image like in real storybooks that helps directors and producers understand the story. `; // Core scene description enhancedPrompt += `${sceneDescription}. `; // Automatic style and technique specifications enhancedPrompt += `Professional storyboard pencil sketch, detailed line art, black and white drawing, `; enhancedPrompt += `hand-drawn illustration style, film production art, cinematic composition. `; // Enhanced descriptive elements based on scene content if (hasCharacters) { enhancedPrompt += `Clear character expressions and body language, detailed facial features and gestures that convey emotion and intent. `; } if (hasLocation) { enhancedPrompt += `Detailed environmental context with architectural elements, props, and spatial relationships clearly defined. `; } if (hasAction) { enhancedPrompt += `Dynamic choreography and movement captured with clear action lines and directional flow. `; } if (hasLighting) { enhancedPrompt += `Dramatic lighting and shadow work using cross-hatching and shading techniques to create depth and mood. `; } if (hasCameraWork) { enhancedPrompt += `Precise camera framing and perspective that matches the described shot composition. `; } // Professional storyboard specifications enhancedPrompt += `Focus on storytelling clarity, visual narrative flow, and production-ready detail. `; enhancedPrompt += `Include background elements, foreground details, and mid-ground composition for depth. `; enhancedPrompt += `Professional film storyboard quality with clean, readable linework and clear visual hierarchy. `; enhancedPrompt += `Capture the essence of the scene's emotional tone and narrative purpose. `; // Technical drawing specifications enhancedPrompt += `High contrast pencil drawing, detailed cross-hatching for shadows, clean line art, `; enhancedPrompt += `traditional animation and film production sketch style, masterful draftsmanship`; return enhancedPrompt; } getAgentGuidance() { return ` CINEMATIC WORKFLOW TOOL GUIDANCE: This tool is STATE-AWARE and automatically handles version control, workflow state management, and ENHANCED STORYBOOK SKETCH GENERATION for cinematic scripts. ## šŸŽÆ STATE-AWARE WORKFLOW: **Story Project State Machine**: 1. **OUTLINE_DRAFT** → User creates/edits story outline - āŒ Cannot create screenplay - āŒ Cannot generate sketches - āœ… Can edit outline - **Action**: Use 'confirm_outline' operation when outline is ready 2. **OUTLINE_CONFIRMED** → Outline is confirmed, ready for screenplay - āœ… Can create screenplay - āŒ Cannot generate sketches yet - āŒ Cannot edit outline - **Action**: Use 'start_screenplay' operation to begin 3. **SCREENPLAY_WRITING** → Writing screenplay scenes - āœ… Can create/update scenes - āŒ Cannot generate sketches yet - āŒ Cannot edit outline - **Action**: Use 'enter_iteration' when screenplay is complete 4. **ITERATION** → Full iteration mode - āœ… Can create/update scenes - āœ… Can generate sketches (respects auto-generation preference) - āœ… Can iterate freely - **Action**: Continue iterating or restart with new outline **State Transition Operations**: - \`confirm_outline\` - OUTLINE_DRAFT → OUTLINE_CONFIRMED - \`start_screenplay\` - OUTLINE_CONFIRMED → SCREENPLAY_WRITING - \`enter_iteration\` - SCREENPLAY_WRITING → ITERATION **IMPORTANT**: The tool will automatically check state constraints and block operations that aren't allowed in the current state. ## šŸŽØ ENHANCED SKETCH GENERATION FEATURES: - **User Preference Aware**: Respects auto_generate_sketches and sketch_style_preference from story_outline.md - **Automatic Storybook Style**: Generates detailed pencil sketch prompts like real storybooks - **Director/Producer Ready**: Creates sketches that help visualize story for production teams - **Comprehensive Scene Capture**: Automatically includes choreography, focus, characters, lighting, and camera work - **Professional Storyboard Quality**: Uses film production art standards with detailed line work - **Intelligent Content Analysis**: Analyzes scene descriptions to enhance prompts with relevant details ## šŸ”§ OPERATIONS: **State Transitions**: - \`confirm_outline\` - Confirm outline and enable screenplay writing - \`start_screenplay\` - Start screenplay writing phase - \`enter_iteration\` - Enter iteration mode with sketch generation **Content Operations** (state-aware): - \`create_scene\` - Create new scene file (requires SCREENPLAY_WRITING or ITERATION state) - \`update_scene\` - Update existing scene (requires SCREENPLAY_WRITING or ITERATION state) - \`generate_scene_images\` - Generate sketches (requires ITERATION state) - \`rollback_scene\` - Rollback to previous version (always allowed) **Version Control**: Old versions are saved to 'old_versions' folder with timestamps **Auto-Generation Behavior**: - If in story project: Uses user's auto_generate_sketches preference - If explicit parameter provided: Overrides preference - If not in story project: Defaults to true ## šŸ“ AUTOMATIC PROMPT ENHANCEMENT: The tool automatically enhances sketch prompts with: - Storybook-style pencil sketch specifications - Character expression and body language details - Environmental context and spatial relationships - Dynamic choreography and movement capture - Dramatic lighting and shadow work - Professional camera framing and perspective - Production-ready detail and visual hierarchy ## šŸ’” USAGE EXAMPLES: **Starting a Story Project**: 1. User creates story_outline.md with outline 2. Agent: "Your outline looks complete. Shall I confirm it?" 3. User: "Yes" 4. Agent: Use \`confirm_outline\` operation 5. Agent: "Outline confirmed! Ready to start screenplay?" 6. User: "Yes" 7. Agent: Use \`start_screenplay\` operation 8. Agent: Now create scenes with \`create_scene\` **Creating Scenes**: \`\`\` operation: create_scene scene_file_path: "scenes/scene_01.md" content: [scene content with Visual Sketch Description sections] \`\`\` **Entering Iteration**: When screenplay is complete: \`\`\` operation: enter_iteration \`\`\` **Scene Structure**: \`\`\` ## SCENE 1: TITLE ### Scene Description [narrative description] ### Action/Dialogue [character dialogue and actions] ### Visual Sketch Description - **WIDE SHOT**: Description of wide shot with characters and environment - **CLOSE-UP**: Description of close-up with character emotions - **LIGHTING**: Lighting description with mood and atmosphere - **MOVEMENT**: Character choreography and camera movement \`\`\` **IMPORTANT**: The "Visual Sketch Description" section is REQUIRED for sketch generation. - Use either "## Visual Sketch Description" or "### Visual Sketch Description" - Include bullet points with shot descriptions (e.g., "- **WIDE SHOT**: ...") - Each bullet point will be converted into a separate sketch image The tool will automatically transform these descriptions into detailed storybook sketch prompts! `; } }