@genkit-ai/compat-oai/lib/audio.d.ts

Genkit AI framework plugin for OpenAI APIs.

import { GenerateRequest, z, Genkit, ModelReference } from 'genkit';
import { ModelInfo, ModelAction } from 'genkit/model';
import OpenAI from 'openai';
import { SpeechCreateParams, TranscriptionCreateParams } from 'openai/resources/audio/index.mjs';

/**
 * Copyright 2024 The Fire Company
 * Copyright 2024 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

type SpeechRequestBuilder = (req: GenerateRequest, params: SpeechCreateParams) => void;
type TranscriptionRequestBuilder = (req: GenerateRequest, params: TranscriptionCreateParams) => void;
declare const TRANSCRIPTION_MODEL_INFO: {
    supports: {
        media: boolean;
        output: string[];
        multiturn: boolean;
        systemRole: boolean;
        tools: boolean;
    };
};
declare const SPEECH_MODEL_INFO: ModelInfo;
declare const TranscriptionConfigSchema: z.ZodObject<Pick<{
    version: z.ZodOptional<z.ZodString>;
    temperature: z.ZodOptional<z.ZodNumber>;
    maxOutputTokens: z.ZodOptional<z.ZodNumber>;
    topK: z.ZodOptional<z.ZodNumber>;
    topP: z.ZodOptional<z.ZodNumber>;
    stopSequences: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
}, "temperature"> & {
    chunking_strategy: z.ZodOptional<z.ZodUnion<[z.ZodLiteral<"auto">, z.ZodObject<{
        type: z.ZodString;
        prefix_padding_ms: z.ZodOptional<z.ZodNumber>;
        silence_duration_ms: z.ZodOptional<z.ZodNumber>;
        threshold: z.ZodOptional<z.ZodNumber>;
    }, "strip", z.ZodTypeAny, {
        type: string;
        prefix_padding_ms?: number | undefined;
        silence_duration_ms?: number | undefined;
        threshold?: number | undefined;
    }, {
        type: string;
        prefix_padding_ms?: number | undefined;
        silence_duration_ms?: number | undefined;
        threshold?: number | undefined;
    }>]>>;
    include: z.ZodOptional<z.ZodArray<z.ZodAny, "many">>;
    language: z.ZodOptional<z.ZodString>;
    timestamp_granularities: z.ZodOptional<z.ZodArray<z.ZodEnum<["word", "segment"]>, "many">>;
    response_format: z.ZodOptional<z.ZodEnum<["json", "text", "srt", "verbose_json", "vtt"]>>;
}, "passthrough", z.ZodTypeAny, z.objectOutputType<Pick<{
    version: z.ZodOptional<z.ZodString>;
    temperature: z.ZodOptional<z.ZodNumber>;
    maxOutputTokens: z.ZodOptional<z.ZodNumber>;
    topK: z.ZodOptional<z.ZodNumber>;
    topP: z.ZodOptional<z.ZodNumber>;
    stopSequences: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
}, "temperature"> & {
    chunking_strategy: z.ZodOptional<z.ZodUnion<[z.ZodLiteral<"auto">, z.ZodObject<{
        type: z.ZodString;
        prefix_padding_ms: z.ZodOptional<z.ZodNumber>;
        silence_duration_ms: z.ZodOptional<z.ZodNumber>;
        threshold: z.ZodOptional<z.ZodNumber>;
    }, "strip", z.ZodTypeAny, {
        type: string;
        prefix_padding_ms?: number | undefined;
        silence_duration_ms?: number | undefined;
        threshold?: number | undefined;
    }, {
        type: string;
        prefix_padding_ms?: number | undefined;
        silence_duration_ms?: number | undefined;
        threshold?: number | undefined;
    }>]>>;
    include: z.ZodOptional<z.ZodArray<z.ZodAny, "many">>;
    language: z.ZodOptional<z.ZodString>;
    timestamp_granularities: z.ZodOptional<z.ZodArray<z.ZodEnum<["word", "segment"]>, "many">>;
    response_format: z.ZodOptional<z.ZodEnum<["json", "text", "srt", "verbose_json", "vtt"]>>;
}, z.ZodTypeAny, "passthrough">, z.objectInputType<Pick<{
    version: z.ZodOptional<z.ZodString>;
    temperature: z.ZodOptional<z.ZodNumber>;
    maxOutputTokens: z.ZodOptional<z.ZodNumber>;
    topK: z.ZodOptional<z.ZodNumber>;
    topP: z.ZodOptional<z.ZodNumber>;
    stopSequences: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
}, "temperature"> & {
    chunking_strategy: z.ZodOptional<z.ZodUnion<[z.ZodLiteral<"auto">, z.ZodObject<{
        type: z.ZodString;
        prefix_padding_ms: z.ZodOptional<z.ZodNumber>;
        silence_duration_ms: z.ZodOptional<z.ZodNumber>;
        threshold: z.ZodOptional<z.ZodNumber>;
    }, "strip", z.ZodTypeAny, {
        type: string;
        prefix_padding_ms?: number | undefined;
        silence_duration_ms?: number | undefined;
        threshold?: number | undefined;
    }, {
        type: string;
        prefix_padding_ms?: number | undefined;
        silence_duration_ms?: number | undefined;
        threshold?: number | undefined;
    }>]>>;
    include: z.ZodOptional<z.ZodArray<z.ZodAny, "many">>;
    language: z.ZodOptional<z.ZodString>;
    timestamp_granularities: z.ZodOptional<z.ZodArray<z.ZodEnum<["word", "segment"]>, "many">>;
    response_format: z.ZodOptional<z.ZodEnum<["json", "text", "srt", "verbose_json", "vtt"]>>;
}, z.ZodTypeAny, "passthrough">>;
declare const SpeechConfigSchema: z.ZodObject<{
    voice: z.ZodDefault<z.ZodEnum<["alloy", "echo", "fable", "onyx", "nova", "shimmer"]>>;
    speed: z.ZodOptional<z.ZodNumber>;
    response_format: z.ZodOptional<z.ZodEnum<["mp3", "opus", "aac", "flac", "wav", "pcm"]>>;
}, "strip", z.ZodTypeAny, {
    voice: "alloy" | "echo" | "fable" | "onyx" | "nova" | "shimmer";
    response_format?: "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm" | undefined;
    speed?: number | undefined;
}, {
    response_format?: "mp3" | "opus" | "aac" | "flac" | "wav" | "pcm" | undefined;
    voice?: "alloy" | "echo" | "fable" | "onyx" | "nova" | "shimmer" | undefined;
    speed?: number | undefined;
}>;
/**
 * Supported media formats for Audio generation
 */
declare const RESPONSE_FORMAT_MEDIA_TYPES: {
    mp3: string;
    opus: string;
    aac: string;
    flac: string;
    wav: string;
    pcm: string;
};
/**
 * Method to define a new Genkit Model that is compatible with the Open AI Audio
 * API.
 *
 * These models are to be used to create audio speech from a given request.
 * @param params An object containing parameters for defining the OpenAI speech
 * model.
 * @param params.ai The Genkit AI instance.
 * @param params.name The name of the model.
 * @param params.client The OpenAI client instance.
 * @param params.modelRef Optional reference to the model's configuration and
 * custom options.

 * @returns the created {@link ModelAction}
 */
declare function defineCompatOpenAISpeechModel<CustomOptions extends z.ZodTypeAny = z.ZodTypeAny>(params: {
    ai: Genkit;
    name: string;
    client: OpenAI;
    modelRef?: ModelReference<CustomOptions>;
    requestBuilder?: SpeechRequestBuilder;
}): ModelAction;
/** Speech generation ModelRef helper, with reasonable defaults for
 * OpenAI-compatible providers */
declare function compatOaiSpeechModelRef<CustomOptions extends z.ZodTypeAny = z.ZodTypeAny>(params: {
    name: string;
    info?: ModelInfo;
    configSchema?: CustomOptions;
    config?: any;
}): ModelReference<any>;
/**
 * Method to define a new Genkit Model that is compatible with Open AI
 * Transcriptions API.
 *
 * These models are to be used to transcribe audio to text.
 *
 * @param params An object containing parameters for defining the OpenAI
 * transcription model.
 * @param params.ai The Genkit AI instance.
 * @param params.name The name of the model.
 * @param params.client The OpenAI client instance.
 * @param params.modelRef Optional reference to the model's configuration and
 * custom options.

 * @returns the created {@link ModelAction}
 */
declare function defineCompatOpenAITranscriptionModel<CustomOptions extends z.ZodTypeAny = z.ZodTypeAny>(params: {
    ai: Genkit;
    name: string;
    client: OpenAI;
    modelRef?: ModelReference<CustomOptions>;
    requestBuilder?: TranscriptionRequestBuilder;
}): ModelAction;
/** Transcription ModelRef helper, with reasonable defaults for
 * OpenAI-compatible providers */
declare function compatOaiTranscriptionModelRef<CustomOptions extends z.ZodTypeAny = z.ZodTypeAny>(params: {
    name: string;
    info?: ModelInfo;
    configSchema?: CustomOptions;
    config?: any;
}): ModelReference<any>;

export { RESPONSE_FORMAT_MEDIA_TYPES, SPEECH_MODEL_INFO, SpeechConfigSchema, type SpeechRequestBuilder, TRANSCRIPTION_MODEL_INFO, TranscriptionConfigSchema, type TranscriptionRequestBuilder, compatOaiSpeechModelRef, compatOaiTranscriptionModelRef, defineCompatOpenAISpeechModel, defineCompatOpenAITranscriptionModel };