@smythos/sdk/src/LLM/LLMInstance.class.ts

SRE SDK

import {
    ILLMConnectorRequest,
    LLMConnector,
    ConnectorService,
    AccessCandidate,
    TCustomLLMModel,
    TLLMConnectorParams,
    TLLMModel,
    TLLMProvider,
    DEFAULT_TEAM_ID,
    BinaryInput,
} from '@smythos/sre';
import { EventEmitter } from 'events';
import { Chat } from './Chat.class';
import { SDKObject } from '../Core/SDKObject.class';
import { adaptModelParams } from './utils';
import { isFile, uid } from '../utils/general.utils';
import { ChatOptions } from '../types/SDKTypes';
import * as fs from 'fs';

class LLMCommand {
    constructor(private _llm: LLMInstance, private _params: any, private _options?: any) {}

    /**
     * Run the command and return the result as a promise.
     * @param resolve - The function to call when the command is resolved
     * @param reject - The function to call when the command is rejected
     * @returns a promise that resolves to the result of the command
     */
    then(resolve: (value: string) => void, reject?: (reason: any) => void) {
        return this.run().then(resolve, reject);
    }

    private async getFiles() {
        let files = [];
        if (this._options?.files) {
            files = await Promise.all(
                this._options.files.map(async (file: string) => {
                    if (isFile(file)) {
                        // Read local file using Node.js fs API with explicit null encoding to get raw binary data
                        const buffer = await fs.promises.readFile(file, null);
                        const binaryInput = BinaryInput.from(buffer);
                        await binaryInput.ready();
                        return binaryInput;
                    } else {
                        return BinaryInput.from(file);
                    }
                })
            );
        }
        return files.length > 0 ? files : undefined;
    }

    async run(): Promise<string> {
        await this._llm.ready;

        let files = await this.getFiles();
        const params = { ...this._params, ...this._llm.modelSettings, files }; // update model settings

        const result = await this._llm.requester.request(params as TLLMConnectorParams);

        if (result.finishReason !== 'stop' && result.finishReason !== 'end_turn') {
            this._llm.emit(
                'error',
                new Error('The model stopped before completing the response, this is usually due to output token limit reached.')
            );
        }
        return result?.content ?? '';
    }

    /**
     * Stream the response from the model as an EventEmitter.
     *
     * **Available Events: are declared in LLMEvent type**
     *
     * - LLMEvent.Content ('content') - Text chunk received from the model
     * - LLMEvent.End ('end') - The model has finished sending data
     * - LLMEvent.Error ('error') - The model encountered an error
     * - ... (see LLMEvent type for more events)
     *
     *
     * @example
     * ```typescript
     * const stream = await llmCommand.stream();
     * stream.on(LLMEvent.Content, (chunk) => console.log(chunk));
     * stream.on(LLMEvent.End, () => console.log('Stream ended'));
     * stream.on(LLMEvent.Error, (err) => console.error(err));
     * ```
     */
    async stream(): Promise<EventEmitter> {
        await this._llm.ready;
        const files = await this.getFiles();
        const params = { ...this._params, ...this._llm.modelSettings, files }; // update model settings

        return await this._llm.requester.streamRequest(params as TLLMConnectorParams);
    }

    // Future extensibility:
    // async batch(): Promise<string[]>
    // temperature(temp: number): PromptBuilder : override the modelParams
    // maxTokens(maxTokens: number): PromptBuilder : override the modelParams
    // ...
    // params(...): PromptBuilder : override the modelParams

    // temperature(temp: number): LLMCommand {
    //     this.llm.modelParams.temperature = temp;
    //     return this;
    // }
}

export type TLLMInstanceParams = {
    model?: string | TLLMModel | TCustomLLMModel;
    apiKey?: string;
    provider?: TLLMProvider;

    /** The maximum number of tokens to generate */
    maxTokens?: number;
    /** The maximum number of tokens to think */
    maxThinkingTokens?: number;
    /** The temperature of the model */
    temperature?: number;
    /** The stop sequences of the model */
    stopSequences?: string[];
    /** The top P of the model */
    topP?: number;
    /** The top K of the model */
    topK?: number;
    /** The frequency penalty of the model */
    frequencyPenalty?: number;
    /** The presence penalty of the model */
    presencePenalty?: number;

    /** The dimensions parameter for text embeddings models */
    dimensions?: number;

    /** the maximum input tokens that the model should accept (Context window size) */
    inputTokens?: number;

    /** the maximum output tokens that the model should generate */
    outputTokens?: number;

    [key: string]: any;
};

/**
 * Represents a LLM instance. These instances are created by the LLM Factory ({@link LLM}).
 *
 *
 * @example Usage example
 * ```typescript
 * const llm = LLM.OpenAI({ model: 'gpt-4o' });
 * //the above is equivalent to:
 * const llm = new LLMInstance(TLLMProvider.OpenAI, { model: 'gpt-4o' });
 *
 * //then you can prompt the LLM to get the response in one shot
 * const response = await llm.prompt('Hello, world!');
 *
 * //or as a stream
 * const stream = await llm.prompt('Hello, world!').stream();
 * stream.on('data', (chunk) => console.log(chunk));
 * stream.on('end', () => console.log('Stream ended'));
 * stream.on('error', (err) => console.error(err));
 *
 * //or as a chat (@see )
 * const chat = llm.chat();
 * chat.prompt('Hello, world!');
 *
 * ```
 * More examples are available in the **{@link LLM}** namespace.
 */
export class LLMInstance extends SDKObject {
    private _llmRequester: ILLMConnectorRequest;
    public get modelSettings() {
        return this._modelSettings;
    }

    public get requester() {
        return this._llmRequester;
    }

    constructor(private _providerId: TLLMProvider, private _modelSettings: TLLMInstanceParams, private _candidate?: AccessCandidate) {
        super();
    }

    protected async init() {
        await super.init();
        const llmConnector = ConnectorService.getLLMConnector(this._providerId);
        this._candidate = this._candidate || AccessCandidate.team(DEFAULT_TEAM_ID);
        this._llmRequester = llmConnector.user(this._candidate);
        this._modelSettings = adaptModelParams(this._modelSettings, this._providerId);
    }

    /**
     * Query the LLM with a prompt.
     *
     * The returned command can be executed in two ways:
     * - **Promise mode**: returns the final result as a string
     * - **Streaming mode**: returns a stream event emitter
     *
     *
     * @example
     * ```typescript
     * // Promise mode : returns the final result as a string
     * const response = await llm.prompt("Hello, world!");
     *
     * // Streaming mode : returns an EventEmitter
     * const stream = await llm.prompt("Tell me a story").stream();
     * stream.on('data', chunk => process.stdout.write(chunk));
     * ```
     */
    public prompt(prompt: string, options?: any): LLMCommand {
        return new LLMCommand(this, { ...this._modelSettings, messages: [{ role: 'user', content: prompt }] }, options);
    }

    public chat(options?: ChatOptions | string) {
        const model = this._modelSettings.model;

        if (typeof options === 'string') {
            options = { id: options, persist: true };
        }

        const chatOptions = {
            ...options,
            candidate: this._candidate,
        };

        return new Chat(chatOptions, model);
    }
}