@mixio-pro/kalaasetu-mcp/src/tools/image-to-video.sdk-backup.ts

A powerful Model Context Protocol server providing AI tools for content generation and analysis

import { z } from "zod";
import * as fs from "fs";
import * as path from "path";
import { GoogleGenAI } from "@google/genai";

async function wait(ms: number): Promise<void> {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function fileToBase64(filePath: string): { data: string; mimeType: string } {
  if (!fs.existsSync(filePath)) {
    throw new Error(`File not found: ${filePath}`);
  }
  const buf = fs.readFileSync(filePath);
  const data = Buffer.from(buf).toString("base64");
  // Detect mime type from extension
  const ext = path.extname(filePath).toLowerCase();
  const mimeType = ext === '.jpg' || ext === '.jpeg' ? 'image/jpeg' : 
                   ext === '.png' ? 'image/png' :
                   ext === '.webp' ? 'image/webp' : 'image/png';
  return { data, mimeType };
}

export const imageToVideo = {
  name: "imageToVideo",
  description: "Generate videos from an image as starting first frame using Vertex Veo models (predictLongRunning + fetchPredictOperation).",
  parameters: z.object({
    prompt: z.string().describe("Text description for the video"),
    image_path: z.string().optional().describe("Path to source image for image-to-video generation"),
    aspect_ratio: z.string().optional().describe("Video aspect ratio: '16:9' or '9:16' (default: '9:16')"),
    duration_seconds: z.number().optional().describe("Video duration in seconds: 4, 6, or 8 (default: 6)"),
    resolution: z.string().optional().describe("Video resolution: '720p' or '1080p' (default: '720p')"),
    negative_prompt: z.string().optional().describe("Text describing what not to include in the video"),
    person_generation: z.string().optional().describe("Controls generation of people: 'allow_adult' (default for image-to-video) or 'allow_all'"),
    reference_images: z.array(z.string()).optional().describe("Additional image paths for reference (max 3)"),
    output_path: z.string().optional().describe("Output MP4 file path (if multiple predictions, index suffix is added)"),
    gemini_api_key: z.string().optional().describe("Gemini API key (uses GEMINI_API_KEY env var if not provided)"),
    model_id: z.string().optional().describe("Model ID (default: veo-2.0-generate-001)"),
  }),
  execute: async (args: {
    prompt: string;
    image_path?: string;
    aspect_ratio?: string;
    duration_seconds?: number;
    resolution?: string;
    negative_prompt?: string;
    person_generation?: string;
    reference_images?: string[];
    output_path?: string;
    gemini_api_key?: string;
    model_id?: string;
  }) => {
    const apiKey = args.gemini_api_key || process.env.GEMINI_API_KEY;
    if (!apiKey) {
      throw new Error("Gemini API key is required. Set GEMINI_API_KEY environment variable or pass gemini_api_key parameter. Get one at https://aistudio.google.com/app/apikey");
    }

    const model = args.model_id || "veo-2.0-generate-001";
    
    // Initialize Google GenAI client
    const genai = new GoogleGenAI({ apiKey });

    // Build config for video generation
    const config: any = {};
    
    if (args.duration_seconds !== undefined) {
      config.duration_seconds = args.duration_seconds;
    } else {
      config.duration_seconds = 6; // default
    }
    
    if (args.aspect_ratio) {
      config.aspect_ratio = args.aspect_ratio;
    }

    try {
      // Start video generation operation
      console.log(`Starting video generation with model: ${model}`);
      let operation = await genai.models.generateVideos({
        model,
        prompt: args.prompt,
        config,
      });

      console.log("Operation started, waiting for completion...");
      
      // Poll until operation is complete (max 10 minutes)
      let tries = 0;
      const maxTries = 60; // 10 minutes with 10s intervals
      
      while (!operation.done && tries < maxTries) {
        await wait(10000); // Wait 10 seconds
        tries++;
        console.log(`Polling attempt ${tries}/${maxTries}...`);
        
        operation = await genai.operations.getVideosOperation({
          operation: operation,
        });
      }

      if (!operation.done) {
        throw new Error("Video generation timed out after 10 minutes");
      }

      console.log("Operation completed!");
      console.log("Full Response:", JSON.stringify(operation.response, null, 2));

      // Extract generated videos from response
      const generatedVideos = operation.response?.generatedVideos || [];
      
      if (!generatedVideos || generatedVideos.length === 0) {
        const respStr = JSON.stringify(operation.response, null, 2);
        return `Video generation completed but no videos found in response.\n\nFull Response:\n${respStr.slice(0, 2000)}${respStr.length > 2000 ? '\n...(truncated)' : ''}`;
      }

      // Download and save videos
      const outputs: string[] = [];
      
      for (let i = 0; i < generatedVideos.length; i++) {
        const generatedVideo = generatedVideos[i];
        const videoUri = generatedVideo?.video?.uri;
        
        if (!videoUri) {
          console.warn(`Video ${i} has no URI`);
          continue;
        }

        console.log(`Downloading video ${i + 1}/${generatedVideos.length}...`);
        
        // Download video from URI
        const videoUrl = `${videoUri}&key=${apiKey}`;
        const response = await fetch(videoUrl);
        
        if (!response.ok) {
          throw new Error(`Failed to download video: ${response.status} ${response.statusText}`);
        }
        
        const buffer = await response.arrayBuffer();
        
        // Save video to file
        const filePath = args.output_path
          ? (i === 0 ? args.output_path : args.output_path.replace(/\.mp4$/i, `_${i}.mp4`))
          : `video_output_${Date.now()}${i === 0 ? '' : '_' + i}.mp4`;
        const absPath = path.resolve(filePath);
        
        fs.writeFileSync(absPath, Buffer.from(buffer));
        outputs.push(absPath);
        console.log(`Saved video to: ${absPath}`);
      }

      if (outputs.length > 0) {
        return `Video(s) saved successfully:\n${outputs.map((p, i) => `${i + 1}. ${p}`).join('\n')}`;
      }

      return "Video generation completed but no videos were saved.";
      
    } catch (error: any) {
      throw new Error(`Video generation failed: ${error.message || JSON.stringify(error)}`);
    }
  },
};