@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
168 lines (167 loc) • 6.83 kB
JavaScript
/**
* Replicate Video Handler
*
* Routes video generation through the universal Replicate prediction
* lifecycle. Default model is Wan-Alpha (RGBA video); callers can specify
* any image-to-video model on Replicate via `options.model`.
*
* @module adapters/video/replicateVideoHandler
* @see https://replicate.com/atonamy/wan-alpha
*/
import { ErrorCategory, ErrorSeverity } from "../../constants/enums.js";
import { VIDEO_ERROR_CODES } from "../../constants/videoErrors.js";
import { logger } from "../../utils/logger.js";
import { VideoError } from "../../utils/videoProcessor.js";
import { getReplicateAuth } from "../replicate/auth.js";
import { downloadPredictionOutput, predict, } from "../replicate/predictionLifecycle.js";
const DEFAULT_MODEL = "atonamy/wan-alpha";
/**
* Replicate Video Handler.
*
* Capabilities depend on the specific Replicate model — this handler
* advertises conservative bounds (any provider-supported aspect ratio /
* resolution; up to 10s typical for Wan-Alpha).
*/
export class ReplicateVideoHandler {
maxDurationSeconds = 10;
supportedAspectRatios = ["9:16", "16:9", "1:1"];
supportedResolutions = [
"720p",
"1080p",
];
instanceCredentials;
constructor(credentials) {
this.instanceCredentials = credentials;
}
isConfigured() {
return getReplicateAuth(this.instanceCredentials) !== null;
}
async generate(image, prompt, options) {
const perCallCreds = options.credentials?.replicate;
const auth = getReplicateAuth(perCallCreds ?? this.instanceCredentials);
if (!auth) {
throw new VideoError({
code: VIDEO_ERROR_CODES.PROVIDER_NOT_CONFIGURED,
message: "REPLICATE_API_TOKEN not configured",
category: ErrorCategory.CONFIGURATION,
severity: ErrorSeverity.HIGH,
retriable: false,
});
}
const startTime = Date.now();
const model = options.model ?? DEFAULT_MODEL;
const dataUri = `data:image/${this.detectImageType(image)};base64,${image.toString("base64")}`;
// Replicate image-to-video models do not share an input schema. The
// legacy shape below (`image` + `num_frames`/`fps`/`aspect_ratio`) fits
// Wan-Alpha-era models; current models (minimax/hailuo-2.3-*,
// wan-video/wan-2.7-i2v) key the image as `first_frame_image` /
// `first_frame` and take `duration` instead of frame counts — a missing
// required image key fails the prediction on submit. `imageInputKey`
// selects the image key AND the modern `duration`/`resolution` shape;
// omitting it preserves the legacy payload exactly.
//
// `calculateDimensions` still populates the metadata `dimensions` field
// so downstream consumers always receive correct width/height regardless
// of whether the model honoured the resolution hint.
const inputPayload = options.imageInputKey !== undefined
? {
[options.imageInputKey]: dataUri,
prompt,
duration: options.length ?? 4,
...(options.resolution !== undefined
? { resolution: options.resolution }
: {}),
}
: {
image: dataUri,
prompt,
num_frames: (options.length ?? 4) * 24, // Assume 24 fps
fps: 24,
aspect_ratio: options.aspectRatio,
...(options.resolution !== undefined
? { resolution: options.resolution }
: {}),
};
let prediction;
try {
prediction = await predict(auth, { model, input: inputPayload }, { abortSignal: options.abortSignal });
}
catch (err) {
throw new VideoError({
code: VIDEO_ERROR_CODES.GENERATION_FAILED,
message: `Replicate video generation failed: ${err instanceof Error ? err.message : String(err)}`,
category: ErrorCategory.EXECUTION,
severity: ErrorSeverity.HIGH,
retriable: true,
context: { model, options },
originalError: err instanceof Error ? err : undefined,
});
}
let videoBuffer;
try {
videoBuffer = await downloadPredictionOutput(prediction);
}
catch (err) {
throw new VideoError({
code: VIDEO_ERROR_CODES.GENERATION_FAILED,
message: `Replicate video download failed: ${err instanceof Error ? err.message : String(err)}`,
category: ErrorCategory.NETWORK,
severity: ErrorSeverity.MEDIUM,
retriable: true,
context: { predictionId: prediction.id },
originalError: err instanceof Error ? err : undefined,
});
}
const processingTime = Date.now() - startTime;
logger.info(`[ReplicateVideoHandler] Generated ${videoBuffer.length} bytes in ${processingTime}ms — model ${model}`);
return {
data: videoBuffer,
mediaType: "video/mp4",
metadata: {
duration: options.length ?? 4,
dimensions: this.calculateDimensions(options),
model,
provider: "replicate",
aspectRatio: options.aspectRatio ?? "16:9",
audioEnabled: false, // Most Replicate video models are silent
processingTime,
},
};
}
detectImageType(buffer) {
if (buffer.length < 4) {
return "jpeg";
}
if (buffer[0] === 0x89 && buffer[1] === 0x50) {
return "png";
}
if (buffer[0] === 0xff && buffer[1] === 0xd8) {
return "jpeg";
}
if (buffer[0] === 0x52 &&
buffer[1] === 0x49 &&
buffer[2] === 0x46 &&
buffer[3] === 0x46) {
return "webp";
}
return "jpeg";
}
calculateDimensions(options) {
const aspectRatio = options.aspectRatio ?? "16:9";
const resolution = options.resolution ?? "720p";
if (resolution === "1080p") {
if (aspectRatio === "1:1") {
return { width: 1080, height: 1080 };
}
return aspectRatio === "9:16"
? { width: 1080, height: 1920 }
: { width: 1920, height: 1080 };
}
if (aspectRatio === "1:1") {
return { width: 720, height: 720 };
}
return aspectRatio === "9:16"
? { width: 720, height: 1280 }
: { width: 1280, height: 720 };
}
}