@juspay/neurolink
Version:
The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002
355 lines (354 loc) • 18.1 kB
JavaScript
import { tracers, ATTR, withSpan } from "../../telemetry/index.js";
import { logger } from "../../utils/logger.js";
import { buildMessagesArray, buildMultimodalMessagesArray, } from "../../utils/messageBuilder.js";
/**
* Compute total content length across all messages for span attributes.
*/
function computeTotalContentLength(messages) {
let total = 0;
for (const msg of messages) {
if (typeof msg.content === "string") {
total += msg.content.length;
}
else if (Array.isArray(msg.content)) {
for (const part of msg.content) {
if ("text" in part &&
typeof part.text === "string") {
total += part.text.length;
}
}
}
}
return total;
}
/**
* Check whether input contains multimodal content (images, files, PDFs, CSVs,
* audio, video). #284: audioFiles/videoFiles were previously ignored here, so a
* request carrying only audio or video was misrouted to the text-only path.
*
* Intentional (round-2 review): routing audio/video through `isMultimodal`
* does NOT mean vision is required. `buildMultimodalMessagesArray` folds
* `audioFiles`/`videoFiles` into `inp.files` for auto-detection, and
* `appendDetectedFileResult` injects their content as plain text markers
* (`## Audio File:` / `## Video File:`), not image/vision blocks — see
* `test/continuous-test-suite-bugfixes.ts` ("MessageBuilder #284: ...").
* "Multimodal" here means "needs the multimodal builder so the payload isn't
* dropped", not "needs a vision model".
*/
function detectMultimodal(opts) {
const input = opts.input;
const hasImages = !!input?.images?.length;
const hasContent = !!input?.content?.length;
const hasCSVFiles = !!input?.csvFiles?.length;
const hasPdfFiles = !!input?.pdfFiles?.length;
const hasFiles = !!input?.files?.length;
const hasAudioFiles = !!input?.audioFiles?.length;
const hasVideoFiles = !!input?.videoFiles?.length;
return {
isMultimodal: hasImages ||
hasContent ||
hasCSVFiles ||
hasPdfFiles ||
hasFiles ||
hasAudioFiles ||
hasVideoFiles,
hasImages,
hasFiles: hasCSVFiles || hasPdfFiles || hasFiles || hasAudioFiles || hasVideoFiles,
};
}
/**
* MessageBuilder class - Handles message construction for AI providers
*/
export class MessageBuilder {
providerName;
modelName;
constructor(providerName, modelName) {
this.providerName = providerName;
this.modelName = modelName;
}
/**
* Build messages array for generation
* Detects multimodal input and routes to appropriate message builder
*/
async buildMessages(options) {
return withSpan({
name: "neurolink.message.build",
tracer: tracers.sdk,
attributes: {
[ATTR.NL_PROVIDER]: this.providerName,
[ATTR.NL_MODEL]: this.modelName,
},
}, async (span) => {
const { isMultimodal, hasImages, hasFiles } = detectMultimodal(options);
span.setAttribute(ATTR.MSG_IS_MULTIMODAL, isMultimodal);
let messages;
if (isMultimodal) {
if (process.env.NEUROLINK_DEBUG === "true") {
logger.debug("Detected multimodal input, using multimodal message builder");
}
const input = options.input;
const multimodalOptions = {
input: {
text: options.prompt || options.input?.text || "",
images: input?.images,
content: input?.content,
csvFiles: input?.csvFiles,
pdfFiles: input?.pdfFiles,
// #284: audioFiles/videoFiles must be forwarded too, or the
// multimodal builder receives an empty payload despite
// detectMultimodal() correctly routing audio/video-only
// requests here.
audioFiles: input?.audioFiles,
videoFiles: input?.videoFiles,
files: input?.files,
},
csvOptions: options.csvOptions,
pdfOptions: options.pdfOptions,
imageOptions: options.imageOptions,
// The audio option bag is carried the same way: omitted here, a
// caller's transcription backend, model and language never reach
// AudioProcessor and it silently falls back to auto-selection.
audioOptions: options.audioOptions,
// Same reason as csvOptions: the keyframe budget and the
// transcription flag are read downstream by the detector, and
// a reconstruction that omits them silently restores defaults.
videoOptions: options.videoOptions,
// This object is rebuilt field by field, so anything not named
// here never reaches the multimodal builder — the same way
// audioFiles/videoFiles went missing in #284. Office options
// steer XLSX sheet selection and rendering.
officeOptions: options.officeOptions,
provider: options.provider,
model: options.model,
temperature: options.temperature,
maxTokens: options.maxTokens,
systemPrompt: options.systemPrompt,
enableAnalytics: options.enableAnalytics,
enableEvaluation: options.enableEvaluation,
context: options.context,
conversationHistory: options.conversationMessages,
schema: options.schema,
output: options.output,
fileRegistry: options.fileRegistry,
};
messages = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
// Propagate any systemPrompt augmentation (e.g. inline-file
// handling guidance from processUnifiedFilesArray) back to the
// caller's options. Providers like GoogleVertex's native
// @google/genai stream path read `options.systemPrompt` directly
// — without this propagation the augmentation lives only on the
// local `multimodalOptions` clone and never reaches the model.
if (multimodalOptions.systemPrompt &&
multimodalOptions.systemPrompt !== options.systemPrompt) {
options.systemPrompt = multimodalOptions.systemPrompt;
}
}
else {
if (process.env.NEUROLINK_DEBUG === "true") {
logger.debug("No multimodal input detected, using standard message builder");
}
messages = await buildMessagesArray(options);
}
// Convert messages to Vercel AI SDK format
// Preserve providerOptions (e.g. Anthropic cache_control) through conversion
const coreMessages = messages.map((msg) => {
const providerOptions = msg
.providerOptions;
if (typeof msg.content === "string") {
return {
role: msg.role,
content: msg.content,
...(providerOptions && { providerOptions }),
};
}
else {
return {
role: msg.role,
content: msg.content.map((item) => {
const itemProviderOptions = item.providerOptions;
if (item.type === "text") {
return {
type: "text",
text: item.text || "",
...(itemProviderOptions && {
providerOptions: itemProviderOptions,
}),
};
}
else if (item.type === "image") {
// Upstream image builders label the resolved format as
// either `mediaType` (canonical ImagePart field) or
// `mimeType` (messageBuilder's convertSimpleImagesToProviderFormat).
// Dropping it here silently reset every transcoded image
// back to the provider's hardcoded PNG default even
// though the transcoded bytes themselves were correct.
const mediaType = (item.mediaType ?? item.mimeType);
return {
type: "image",
image: item.image || "",
...(mediaType && { mediaType }),
...(itemProviderOptions && {
providerOptions: itemProviderOptions,
}),
};
}
return item;
}),
...(providerOptions && { providerOptions }),
};
}
});
span.setAttribute(ATTR.MSG_COUNT, coreMessages.length);
span.setAttribute(ATTR.MSG_HAS_IMAGES, hasImages);
span.setAttribute(ATTR.MSG_HAS_FILES, hasFiles);
span.setAttribute(ATTR.MSG_HAS_SYSTEM_PROMPT, !!options.systemPrompt);
span.setAttribute(ATTR.MSG_TOTAL_CONTENT_LENGTH, computeTotalContentLength(coreMessages));
return coreMessages;
});
}
/**
* Build messages array for streaming operations
* This is a protected helper method that providers can use to build messages
* with automatic multimodal detection, eliminating code duplication
*
* @param options - Stream options or text generation options
* @returns Promise resolving to ModelMessage array ready for AI SDK
*/
async buildMessagesForStream(options) {
return withSpan({
name: "neurolink.message.build_for_stream",
tracer: tracers.sdk,
attributes: {
[ATTR.NL_PROVIDER]: this.providerName,
[ATTR.NL_MODEL]: this.modelName,
},
}, async (span) => {
const { isMultimodal, hasImages, hasFiles } = detectMultimodal(options);
span.setAttribute(ATTR.MSG_IS_MULTIMODAL, isMultimodal);
let messages;
if (isMultimodal) {
if (process.env.NEUROLINK_DEBUG === "true") {
logger.debug(`${this.providerName}: Detected multimodal input, using multimodal message builder`);
}
const input = options.input;
const multimodalOptions = {
input: {
text: options.prompt ||
options.input?.text ||
"",
images: input?.images,
content: input?.content,
csvFiles: input?.csvFiles,
pdfFiles: input?.pdfFiles,
// #284: forward audioFiles/videoFiles here too — see the
// matching comment in buildMessages() above.
audioFiles: input?.audioFiles,
videoFiles: input?.videoFiles,
files: input?.files,
},
csvOptions: options.csvOptions,
pdfOptions: options.pdfOptions,
imageOptions: options.imageOptions,
// The audio option bag is carried the same way: omitted here, a
// caller's transcription backend, model and language never reach
// AudioProcessor and it silently falls back to auto-selection.
audioOptions: options.audioOptions,
// Same reason as csvOptions: the keyframe budget and the
// transcription flag are read downstream by the detector, and
// a reconstruction that omits them silently restores defaults.
videoOptions: options.videoOptions,
// This object is rebuilt field by field, so anything not named
// here never reaches the multimodal builder — the same way
// audioFiles/videoFiles went missing in #284. Office options
// steer XLSX sheet selection and rendering.
officeOptions: options.officeOptions,
provider: options.provider,
model: options.model,
temperature: options.temperature,
maxTokens: options.maxTokens,
systemPrompt: options.systemPrompt,
enableAnalytics: options.enableAnalytics,
enableEvaluation: options.enableEvaluation,
context: options.context,
conversationHistory: options
.conversationMessages,
schema: options.schema,
output: options.output,
fileRegistry: options.fileRegistry,
};
messages = await buildMultimodalMessagesArray(multimodalOptions, this.providerName, this.modelName);
// Propagate any systemPrompt augmentation (e.g. inline-file
// handling guidance from processUnifiedFilesArray) back to the
// caller's options. Providers like GoogleVertex's native
// @google/genai stream path read `options.systemPrompt` directly
// — without this propagation the augmentation lives only on the
// local `multimodalOptions` clone and never reaches the model.
if (multimodalOptions.systemPrompt &&
multimodalOptions.systemPrompt !== options.systemPrompt) {
options.systemPrompt = multimodalOptions.systemPrompt;
}
}
else {
if (process.env.NEUROLINK_DEBUG === "true") {
logger.debug(`${this.providerName}: No multimodal input detected, using standard message builder`);
}
messages = await buildMessagesArray(options);
}
// Convert messages to Vercel AI SDK format
// Preserve providerOptions (e.g. Anthropic cache_control) through conversion
const coreMessages = messages.map((msg) => {
const providerOptions = msg
.providerOptions;
if (typeof msg.content === "string") {
return {
role: msg.role,
content: msg.content,
...(providerOptions && { providerOptions }),
};
}
else {
return {
role: msg.role,
content: msg.content.map((item) => {
const itemProviderOptions = item.providerOptions;
if (item.type === "text") {
return {
type: "text",
text: item.text || "",
...(itemProviderOptions && {
providerOptions: itemProviderOptions,
}),
};
}
else if (item.type === "image") {
// Upstream image builders label the resolved format as
// either `mediaType` (canonical ImagePart field) or
// `mimeType` (messageBuilder's convertSimpleImagesToProviderFormat).
// Dropping it here silently reset every transcoded image
// back to the provider's hardcoded PNG default even
// though the transcoded bytes themselves were correct.
const mediaType = (item.mediaType ?? item.mimeType);
return {
type: "image",
image: item.image || "",
...(mediaType && { mediaType }),
...(itemProviderOptions && {
providerOptions: itemProviderOptions,
}),
};
}
return item;
}),
...(providerOptions && { providerOptions }),
};
}
});
span.setAttribute(ATTR.MSG_COUNT, coreMessages.length);
span.setAttribute(ATTR.MSG_HAS_IMAGES, hasImages);
span.setAttribute(ATTR.MSG_HAS_FILES, hasFiles);
span.setAttribute(ATTR.MSG_HAS_SYSTEM_PROMPT, !!options.systemPrompt);
span.setAttribute(ATTR.MSG_TOTAL_CONTENT_LENGTH, computeTotalContentLength(coreMessages));
return coreMessages;
});
}
}