assistant-stream/src/core/converters/toGenericMessages.ts

Streaming utilities for AI assistants

/**
 * Generic message types for framework-agnostic LLM message interchange.
 * These types represent a common format that can be converted to/from
 * various LLM provider formats (AI SDK, LangChain, etc.).
 */

import { NO_RESULT } from "../tool/ToolResponse";
import type { ToolModelContentPart } from "../tool/tool-types";

export type GenericTextPart = {
  type: "text";
  text: string;
};

export type GenericFilePart = {
  type: "file";
  data: string | URL;
  mediaType: string;
  filename?: string;
};

export type GenericToolCallPart = {
  type: "tool-call";
  toolCallId: string;
  toolName: string;
  args: Record<string, unknown>;
};

export type GenericToolResultPart = {
  type: "tool-result";
  toolCallId: string;
  toolName: string;
  result: unknown;
  /** Explicit model-visible content for a settled tool result. */
  modelContent?: readonly ToolModelContentPart[];
  isError?: boolean;
};

export type GenericSystemMessage = {
  role: "system";
  content: string;
};

export type GenericUserMessage = {
  role: "user";
  content: (GenericTextPart | GenericFilePart)[];
};

export type GenericAssistantMessage = {
  role: "assistant";
  content: (GenericTextPart | GenericFilePart | GenericToolCallPart)[];
};

export type GenericToolMessage = {
  role: "tool";
  content: GenericToolResultPart[];
};

export type GenericMessage =
  | GenericSystemMessage
  | GenericUserMessage
  | GenericAssistantMessage
  | GenericToolMessage;

type MessagePartLike = {
  type: string;
  text?: string;
  image?: string;
  audio?: {
    data?: string;
    format?: "mp3" | "wav";
  };
  data?: string;
  mimeType?: string;
  filename?: string;
  toolCallId?: string;
  toolName?: string;
  args?: Record<string, unknown>;
  state?: string;
  result?: unknown;
  modelContent?: readonly ToolModelContentPart[];
  isError?: boolean;
  isPreliminary?: boolean;
  approval?: {
    approved?: boolean;
    resolution?: string;
    [key: string]: unknown;
  };
  interrupt?: unknown;
};

type AttachmentLike = {
  contentType?: string;
  content: readonly MessagePartLike[];
};

type ThreadMessageLike = {
  role: "system" | "user" | "assistant";
  content: readonly MessagePartLike[];
  attachments?: readonly AttachmentLike[];
  status?: { type: string };
};

const IMAGE_MEDIA_TYPES: Record<string, string> = {
  jpg: "image/jpeg",
  jpeg: "image/jpeg",
  png: "image/png",
  gif: "image/gif",
  webp: "image/webp",
  svg: "image/svg+xml",
  avif: "image/avif",
  bmp: "image/bmp",
  ico: "image/x-icon",
  tiff: "image/tiff",
  tif: "image/tiff",
  heic: "image/heic",
  heif: "image/heif",
};

function getDataUrlMediaType(value: string): string | undefined {
  return value.match(/^data:([^;,]+)(?:[;,])/i)?.[1]?.toLowerCase();
}

function inferImageMediaType(url: string, contentType?: string): string {
  // Providers reject image/* as a URL media type.
  const declared = contentType?.toLowerCase();
  if (declared?.startsWith("image/") && !declared.includes("*")) {
    return declared;
  }

  // Handle data URLs: data:[<mediatype>][;base64],<data>
  if (/^data:/i.test(url)) {
    const match = url.match(/^data:([^;,]+)/i);
    if (match?.[1]) return match[1].toLowerCase();
  }

  // Extract extension from URL path, ignoring query string and hash
  const [pathWithoutParams = ""] = url.split(/[?#]/);
  const ext = pathWithoutParams.split(".").pop()?.toLowerCase() ?? "";
  return IMAGE_MEDIA_TYPES[ext] ?? "image/png";
}

function toUrlOrString(value: string): string | URL {
  try {
    return new URL(value);
  } catch {
    return value;
  }
}

function toGenericFilePart(
  part: MessagePartLike,
  contentType?: string,
): GenericFilePart | undefined {
  if (
    part.type === "audio" &&
    typeof part.audio?.data === "string" &&
    part.audio.format !== undefined
  ) {
    const mediaType = `audio/${part.audio.format}`;
    const base64DataUrl = part.audio.data.match(
      /^data:[^;,]*(?:;[^;,]+)*;base64,(.*)$/i,
    );
    return {
      type: "file",
      data: toUrlOrString(
        base64DataUrl
          ? `data:${mediaType};base64,${base64DataUrl[1]}`
          : part.audio.data,
      ),
      mediaType,
    };
  }
  if (part.type === "image" && part.image) {
    return {
      type: "file",
      data: toUrlOrString(part.image),
      mediaType: inferImageMediaType(part.image, contentType),
      ...(part.filename && { filename: part.filename }),
    };
  }
  if (part.type === "file" && typeof part.data === "string") {
    return {
      type: "file",
      data: toUrlOrString(part.data),
      mediaType:
        (typeof part.mimeType === "string" && part.mimeType) ||
        getDataUrlMediaType(part.data) ||
        "application/octet-stream",
      ...(part.filename && { filename: part.filename }),
    };
  }
  return undefined;
}

type ToolCallAccumulator = {
  textParts: (GenericTextPart | GenericFilePart | GenericToolCallPart)[];
  toolResults: GenericToolResultPart[];
};

function isAwaitingHost(part: MessagePartLike): boolean {
  if (part.interrupt != null) return true;
  if (part.approval == null) return false;
  return (
    part.approval.approved !== false && part.approval.resolution === undefined
  );
}

function processToolCall(
  part: MessagePartLike,
  accumulator: ToolCallAccumulator,
  inFlight: boolean,
): boolean {
  if (!part.toolCallId || !part.toolName) return false;

  accumulator.textParts.push({
    type: "tool-call",
    toolCallId: part.toolCallId,
    toolName: part.toolName,
    args: part.args ?? {},
  });

  const settled =
    !part.isPreliminary &&
    (part.state === "result" || part.result !== undefined);
  // The in-flight message is the last one and is converted on every roundtrip, so a call in it still awaiting a decision or an execution is live rather than failed; a call in an earlier or settled message will never be answered.
  if (!settled && inFlight && isAwaitingHost(part)) return false;

  // Providers reject an assistant tool call that no tool result answers.
  const toolResult: GenericToolResultPart = {
    type: "tool-result",
    toolCallId: part.toolCallId,
    toolName: part.toolName,
    result: !settled
      ? { error: "Tool call was not completed" }
      : part.result === undefined
        ? NO_RESULT
        : part.result,
    ...(settled && part.modelContent !== undefined
      ? { modelContent: part.modelContent }
      : {}),
  };
  if (!settled || part.isError) {
    toolResult.isError = true;
  }
  accumulator.toolResults.push(toolResult);
  return true;
}

function flushAccumulator(
  accumulator: ToolCallAccumulator,
  result: GenericMessage[],
): void {
  if (accumulator.textParts.length > 0) {
    result.push({ role: "assistant", content: accumulator.textParts });
    accumulator.textParts = [];
  }
  if (accumulator.toolResults.length > 0) {
    result.push({ role: "tool", content: accumulator.toolResults });
    accumulator.toolResults = [];
  }
}

function convertSystemMessage(
  message: ThreadMessageLike,
  result: GenericMessage[],
): void {
  const textPart = message.content?.find((p) => p?.type === "text");
  if (textPart?.text) {
    result.push({ role: "system", content: textPart.text });
  }
}

function convertUserMessage(
  message: ThreadMessageLike,
  result: GenericMessage[],
): void {
  const attachments = message.attachments ?? [];
  const allContent = [
    ...(message.content ?? []).map((part) => ({
      part,
      contentType: undefined,
    })),
    ...attachments.flatMap((attachment) =>
      (attachment?.content ?? []).map((part) => ({
        part,
        contentType: attachment.contentType,
      })),
    ),
  ];

  const content: (GenericTextPart | GenericFilePart)[] = [];

  for (const { part, contentType } of allContent) {
    if (!part) continue;
    if (part.type === "text" && part.text) {
      content.push({ type: "text", text: part.text });
    } else {
      const filePart = toGenericFilePart(part, contentType);
      if (filePart) content.push(filePart);
    }
  }

  if (content.length > 0) {
    result.push({ role: "user", content });
  }
}

function convertAssistantMessage(
  message: ThreadMessageLike,
  result: GenericMessage[],
  isLast: boolean,
): void {
  const accumulator: ToolCallAccumulator = {
    textParts: [],
    toolResults: [],
  };
  let hasPendingToolResults = false;
  const inFlight =
    isLast &&
    message.status?.type !== "complete" &&
    message.status?.type !== "incomplete";

  for (const part of message.content ?? []) {
    if (!part) continue;
    if (part.type === "text" && part.text) {
      // Flush pending tool results before adding more text
      if (hasPendingToolResults) {
        flushAccumulator(accumulator, result);
        hasPendingToolResults = false;
      }
      accumulator.textParts.push({ type: "text", text: part.text });
    } else if (part.type === "tool-call") {
      if (processToolCall(part, accumulator, inFlight)) {
        hasPendingToolResults = true;
      }
    } else {
      const filePart = toGenericFilePart(part);
      if (filePart) {
        if (hasPendingToolResults) {
          flushAccumulator(accumulator, result);
          hasPendingToolResults = false;
        }
        accumulator.textParts.push(filePart);
      }
    }
  }

  flushAccumulator(accumulator, result);
}

/**
 * Converts thread messages to generic LLM messages.
 * This format can then be easily converted to provider-specific formats.
 */
export function toGenericMessages(
  messages: readonly ThreadMessageLike[],
): GenericMessage[] {
  const result: GenericMessage[] = [];
  const present = messages.filter(Boolean);

  for (const [index, message] of present.entries()) {
    switch (message.role) {
      case "system":
        convertSystemMessage(message, result);
        break;
      case "user":
        convertUserMessage(message, result);
        break;
      case "assistant":
        convertAssistantMessage(message, result, index === present.length - 1);
        break;
    }
  }

  return result;
}