ai-utils.js/model-provider/cohere/CohereTokenizer.d.ts

Build AI applications, chatbots, and agents with JavaScript and TypeScript.

import z from "zod";
import { FullTokenizer } from "../../model-function/tokenize-text/Tokenizer.js";
import { Run } from "../../run/Run.js";
import { RetryFunction } from "../../util/api/RetryFunction.js";
import { ThrottleFunction } from "../../util/api/ThrottleFunction.js";
import { CohereTextGenerationModelType } from "./CohereTextGenerationModel.js";
import { CohereTextEmbeddingModelType } from "./index.js";
export type CohereTokenizerModelType = CohereTextGenerationModelType | CohereTextEmbeddingModelType;
export interface CohereTokenizerSettings {
    model: CohereTokenizerModelType;
    baseUrl?: string;
    apiKey?: string;
    retry?: RetryFunction;
    throttle?: ThrottleFunction;
}
/**
 * Tokenizer for the Cohere models. It uses the Co.Tokenize and Co.Detokenize APIs.
 *
 * @see https://docs.cohere.com/reference/tokenize
 * @see https://docs.cohere.com/reference/detokenize-1
 *
 * @example
 * const tokenizer = new CohereTokenizer({ model: "command-nightly" });
 *
 * const text = "At first, Nox didn't know what to do with the pup.";
 *
 * const tokenCount = await countTokens(tokenizer, text);
 * const tokens = await tokenizer.tokenize(text);
 * const tokensAndTokenTexts = await tokenizer.tokenizeWithTexts(text);
 * const reconstructedText = await tokenizer.detokenize(tokens);
 */
export declare class CohereTokenizer implements FullTokenizer {
    readonly settings: CohereTokenizerSettings;
    constructor(settings: CohereTokenizerSettings);
    private get apiKey();
    callTokenizeAPI(text: string, context?: Run): Promise<CohereTokenizationResponse>;
    callDeTokenizeAPI(tokens: number[], context?: Run): Promise<CohereDetokenizationResponse>;
    tokenize(text: string): Promise<number[]>;
    tokenizeWithTexts(text: string): Promise<{
        tokens: number[];
        tokenTexts: string[];
    }>;
    detokenize(tokens: number[]): Promise<string>;
}
declare const cohereDetokenizationResponseSchema: z.ZodObject<{
    text: z.ZodString;
    meta: z.ZodObject<{
        api_version: z.ZodObject<{
            version: z.ZodString;
        }, "strip", z.ZodTypeAny, {
            version: string;
        }, {
            version: string;
        }>;
    }, "strip", z.ZodTypeAny, {
        api_version: {
            version: string;
        };
    }, {
        api_version: {
            version: string;
        };
    }>;
}, "strip", z.ZodTypeAny, {
    text: string;
    meta: {
        api_version: {
            version: string;
        };
    };
}, {
    text: string;
    meta: {
        api_version: {
            version: string;
        };
    };
}>;
export type CohereDetokenizationResponse = z.infer<typeof cohereDetokenizationResponseSchema>;
declare const cohereTokenizationResponseSchema: z.ZodObject<{
    tokens: z.ZodArray<z.ZodNumber, "many">;
    token_strings: z.ZodArray<z.ZodString, "many">;
    meta: z.ZodObject<{
        api_version: z.ZodObject<{
            version: z.ZodString;
        }, "strip", z.ZodTypeAny, {
            version: string;
        }, {
            version: string;
        }>;
    }, "strip", z.ZodTypeAny, {
        api_version: {
            version: string;
        };
    }, {
        api_version: {
            version: string;
        };
    }>;
}, "strip", z.ZodTypeAny, {
    meta: {
        api_version: {
            version: string;
        };
    };
    tokens: number[];
    token_strings: string[];
}, {
    meta: {
        api_version: {
            version: string;
        };
    };
    tokens: number[];
    token_strings: string[];
}>;
export type CohereTokenizationResponse = z.infer<typeof cohereTokenizationResponseSchema>;
export {};