inference-server/dist/experiments/VoiceFunctionCall.js

Libraries and server to build AI applications. Adapters to various native bindings allowing local inference. Integrate it with your application, or use as a microservice.

import { CustomEngine } from '../engines/index.js';
// an experimental engine that forwards a transcription to a (function calling) chat model
export class VoiceFunctionCallEngine extends CustomEngine {
    speechToTextModel;
    chatModel;
    tools;
    constructor({ speechToTextModel, chatModel, tools }) {
        super();
        this.speechToTextModel = speechToTextModel;
        this.chatModel = chatModel;
        this.tools = tools;
    }
    async createTranscription(task) {
        const speechToTextModel = await this.pool.requestInstance({
            model: this.speechToTextModel,
        });
        const transcriptionTask = speechToTextModel.instance.processSpeechToTextTask({
            ...task,
            model: this.speechToTextModel,
        });
        const transcription = await transcriptionTask.result;
        speechToTextModel.release();
        return transcription.text;
    }
    async processSpeechToTextTask(task) {
        const [transcription, chatModel] = await Promise.all([
            this.createTranscription(task),
            this.pool.requestInstance({
                model: this.chatModel,
            }),
        ]);
        const chatTask = chatModel.instance.processChatCompletionTask({
            onChunk: task.onChunk,
            model: this.chatModel,
            tools: this.tools ? { definitions: this.tools } : undefined,
            messages: [
                {
                    role: 'user',
                    content: transcription,
                },
            ],
        });
        const chatResponse = await chatTask.result;
        chatModel.release();
        return {
            text: chatResponse.message.content,
        };
    }
}
//# sourceMappingURL=VoiceFunctionCall.js.map