@push.rocks/smartai/ts/abstract.classes.multimodal.ts

SmartAi is a versatile TypeScript library designed to facilitate integration and interaction with various AI models, offering functionalities for chat, audio generation, document processing, and vision tasks.

import * as plugins from './plugins.js';

/**
 * Message format for chat interactions
 */
export interface ChatMessage {
  role: 'assistant' | 'user' | 'system';
  content: string;
}

/**
 * Options for chat interactions
 */
export interface ChatOptions {
  systemMessage: string;
  userMessage: string;
  messageHistory: ChatMessage[];
}

/**
 * Response format for chat interactions
 */
export interface ChatResponse {
  role: 'assistant';
  message: string;
}

/**
 * Abstract base class for multi-modal AI models.
 * Provides a common interface for different AI providers (OpenAI, Anthropic, Perplexity, Ollama)
 */
export abstract class MultiModalModel {
  /**
   * SmartPdf instance for document processing
   * Shared across all methods that need PDF functionality
   */
  protected smartpdfInstance: plugins.smartpdf.SmartPdf;

  /**
   * Initializes the model and any necessary resources
   * Should be called before using any other methods
   */
  public async start(): Promise<void> {
    this.smartpdfInstance = new plugins.smartpdf.SmartPdf();
    await this.smartpdfInstance.start();
  }
  
  /**
   * Cleans up any resources used by the model
   * Should be called when the model is no longer needed
   */
  public async stop(): Promise<void> {
    if (this.smartpdfInstance) {
      await this.smartpdfInstance.stop();
    }
  }

  /**
   * Synchronous chat interaction with the model
   * @param optionsArg Options containing system message, user message, and message history
   * @returns Promise resolving to the assistant's response
   */
  public abstract chat(optionsArg: ChatOptions): Promise<ChatResponse>;
  
  /**
   * Streaming interface for chat interactions
   * Allows for real-time responses from the model
   * @param input Stream of user messages
   * @returns Stream of model responses
   */
  public abstract chatStream(input: ReadableStream<Uint8Array>): Promise<ReadableStream<string>>;

  /**
   * Text-to-speech conversion
   * @param optionsArg Options containing the message to convert to speech
   * @returns Promise resolving to a readable stream of audio data
   * @throws Error if the provider doesn't support audio generation
   */
  public abstract audio(optionsArg: { message: string }): Promise<NodeJS.ReadableStream>;

  /**
   * Vision-language processing
   * @param optionsArg Options containing the image and prompt for analysis
   * @returns Promise resolving to the model's description or analysis of the image
   * @throws Error if the provider doesn't support vision tasks
   */
  public abstract vision(optionsArg: { image: Buffer; prompt: string }): Promise<string>;

  /**
   * Document analysis and processing
   * @param optionsArg Options containing system message, user message, PDF documents, and message history
   * @returns Promise resolving to the model's analysis of the documents
   * @throws Error if the provider doesn't support document processing
   */
  public abstract document(optionsArg: {
    systemMessage: string;
    userMessage: string;
    pdfDocuments: Uint8Array[];
    messageHistory: ChatMessage[];
  }): Promise<{ message: any }>;
}