@axflow/models/dist/togetherai/inference.d.ts

Zero-dependency, modular SDK for building robust natural language applications

declare namespace TogetherAIInferenceTypes {
    type RequestOptions = {
        apiKey?: string;
        apiUrl?: string;
        fetch?: typeof fetch;
        headers?: Record<string, string>;
        signal?: AbortSignal;
    };
    type Request = {
        model: string;
        prompt: string;
        max_tokens: number;
        stop?: string[];
        temperature?: number;
        top_p?: number;
        top_k?: number;
        repetition_penalty?: number;
        logprobs?: number;
        safety_model?: string;
    };
    type Response = {
        id: string;
        status: string;
        prompt: string[];
        model: string;
        model_owner: string;
        tags?: Record<string, any>;
        num_returns: number;
        args: {
            model: string;
            prompt: string;
            max_tokens: number;
            stream: boolean;
            temperature?: number;
            top_p?: number;
            top_k?: number;
        };
        subjobs: any[];
        output: {
            usage?: {
                prompt_tokens: number;
                completion_tokens: number;
                total_tokens: number;
            };
            result_type: string;
            raw_compute_time?: number;
            choices: Array<{
                text: string;
                index?: number;
                finish_reason?: string;
            }>;
        };
    };
    type Chunk = {
        id: string;
        choices: Array<{
            text: string;
        }>;
        generated_text?: string;
        token: {
            id: number;
            text: string;
            logprob: number;
            special: boolean;
        };
        stats: null | {
            total_time: {
                secs: number;
                nanos: number;
            };
            validation_time: {
                secs: number;
                nanos: number;
            };
            queue_time: {
                secs: number;
                nanos: number;
            };
            inference_time: {
                secs: number;
                nanos: number;
            };
            time_per_token: {
                secs: number;
                nanos: number;
            };
        };
        usage: null | {
            prompt_tokens: number;
            completion_tokens: number;
            total_tokens: number;
        };
    };
}
/**
 * Run a prediction request against TogetherAI's inference API.
 *
 * @see https://docs.together.ai/reference/inference
 *
 * @param request The request body sent to TogetherAI. See https://docs.together.ai/reference/inference.
 * @param options
 * @param options.apiKey TogetherAI API key.
 * @param options.apiUrl The url of the TogetherAI (or compatible) API. Defaults to https://api.together.xyz/inference.
 * @param options.fetch A custom implementation of fetch. Defaults to globalThis.fetch.
 * @param options.headers Optionally add additional HTTP headers to the request.
 * @param options.signal An AbortSignal that can be used to abort the fetch request.
 * @returns TogetherAI inference response. See https://docs.together.ai/reference/inference.
 */
declare function run(request: TogetherAIInferenceTypes.Request, options: TogetherAIInferenceTypes.RequestOptions): Promise<TogetherAIInferenceTypes.Response>;
/**
 * Run a streaming prediction request against TogetherAI's inference API. The resulting stream is the raw unmodified bytes from the API.
 *
 * @see https://docs.together.ai/reference/inference
 *
 * @param request The request body sent to TogetherAI. See https://docs.together.ai/reference/inference.
 * @param options
 * @param options.apiKey TogetherAI API key.
 * @param options.apiUrl The url of the TogetherAI (or compatible) API. Defaults to https://api.together.xyz/inference.
 * @param options.fetch A custom implementation of fetch. Defaults to globalThis.fetch.
 * @param options.headers Optionally add additional HTTP headers to the request.
 * @param options.signal An AbortSignal that can be used to abort the fetch request.
 * @returns A stream of bytes directly from the API.
 */
declare function streamBytes(request: TogetherAIInferenceTypes.Request, options: TogetherAIInferenceTypes.RequestOptions): Promise<ReadableStream<Uint8Array>>;
/**
 * Run a streaming prediction request against TogetherAI's inference API. The resulting stream is the parsed stream data as JavaScript objects.
 *
 * @see https://docs.together.ai/reference/inference
 *
 * @param request The request body sent to TogetherAI. See https://docs.together.ai/reference/inference.
 * @param options
 * @param options.apiKey TogetherAI API key.
 * @param options.apiUrl The url of the TogetherAI (or compatible) API. Defaults to https://api.together.xyz/inference.
 * @param options.fetch A custom implementation of fetch. Defaults to globalThis.fetch.
 * @param options.headers Optionally add additional HTTP headers to the request.
 * @param options.signal An AbortSignal that can be used to abort the fetch request.
 * @returns A stream of objects representing each chunk from the API.
 */
declare function stream(request: TogetherAIInferenceTypes.Request, options: TogetherAIInferenceTypes.RequestOptions): Promise<ReadableStream<TogetherAIInferenceTypes.Chunk>>;
/**
 * Run a streaming prediction request against TogetherAI's inference API. The resulting stream emits only the string tokens.
 *
 * @see https://docs.together.ai/reference/inference
 *
 * @param request The request body sent to TogetherAI. See https://docs.together.ai/reference/inference.
 * @param options
 * @param options.apiKey TogetherAI API key.
 * @param options.apiUrl The url of the TogetherAI (or compatible) API. Defaults to https://api.together.xyz/inference.
 * @param options.fetch A custom implementation of fetch. Defaults to globalThis.fetch.
 * @param options.headers Optionally add additional HTTP headers to the request.
 * @param options.signal An AbortSignal that can be used to abort the fetch request.
 * @returns A stream of tokens from the API.
 */
declare function streamTokens(request: TogetherAIInferenceTypes.Request, options: TogetherAIInferenceTypes.RequestOptions): Promise<ReadableStream<string>>;
/**
 * An object that encapsulates methods for calling the TogetherAI inference API.
 */
declare class TogetherAIInference {
    static run: typeof run;
    static stream: typeof stream;
    static streamBytes: typeof streamBytes;
    static streamTokens: typeof streamTokens;
}

export { TogetherAIInference, TogetherAIInferenceTypes };