inference-server
Version:
Libraries and server to build AI applications. Adapters to various native bindings allowing local inference. Integrate it with your application, or use as a microservice.
38 lines (37 loc) • 2.26 kB
TypeScript
import { InferenceModel, EmbeddingModel } from 'gpt4all';
import { ChatCompletionTaskResult, TextCompletionTaskResult, EngineContext, EmbeddingTaskResult, FileDownloadProgress, ModelConfig, ChatMessage, TextCompletionTaskArgs, EngineTextCompletionTaskContext, TextCompletionParamsBase, ChatCompletionTaskArgs, EmbeddingTaskArgs, EngineTaskContext } from '../../types/index.js';
export type GPT4AllInstance = InferenceModel | EmbeddingModel;
export interface GPT4AllModelMeta {
url: string;
md5sum: string;
filename: string;
promptTemplate: string;
systemPrompt: string;
filesize: number;
ramrequired: number;
}
export interface GPT4AllModelConfig extends ModelConfig {
location: string;
md5?: string;
url?: string;
contextSize?: number;
batchSize?: number;
task: 'text-completion' | 'embedding';
initialMessages?: ChatMessage[];
completionDefaults?: TextCompletionParamsBase;
device?: {
gpu?: boolean | 'auto' | (string & {});
gpuLayers?: number;
cpuThreads?: number;
};
}
export declare const autoGpu = true;
export declare function prepareModel({ config, log }: EngineContext<GPT4AllModelConfig>, onProgress?: (progress: FileDownloadProgress) => void, signal?: AbortSignal): Promise<{
gguf?: any;
gpt4allMeta: GPT4AllModelMeta | undefined;
} | undefined>;
export declare function createInstance({ config, log }: EngineContext<GPT4AllModelConfig>, signal?: AbortSignal): Promise<InferenceModel | EmbeddingModel>;
export declare function disposeInstance(instance: GPT4AllInstance): Promise<void>;
export declare function processTextCompletionTask(task: TextCompletionTaskArgs, ctx: EngineTextCompletionTaskContext<GPT4AllInstance, GPT4AllModelConfig, GPT4AllModelMeta>, signal?: AbortSignal): Promise<TextCompletionTaskResult>;
export declare function processChatCompletionTask(task: ChatCompletionTaskArgs, ctx: EngineTextCompletionTaskContext<GPT4AllInstance, GPT4AllModelConfig, GPT4AllModelMeta>, signal?: AbortSignal): Promise<ChatCompletionTaskResult>;
export declare function processEmbeddingTask(task: EmbeddingTaskArgs, ctx: EngineTaskContext<GPT4AllInstance, GPT4AllModelConfig, GPT4AllModelMeta>, signal?: AbortSignal): Promise<EmbeddingTaskResult>;