node-llama-cpp
Version:
Run AI models locally on your machine with node.js bindings for llama.cpp. Enforce a JSON schema on the model output on the generation level
17 lines (16 loc) • 787 B
TypeScript
import { LlamaModelOptions } from "../../../evaluator/LlamaModel/LlamaModel.js";
import { BuildGpu } from "../../../bindings/types.js";
import type { GgufInsights } from "../GgufInsights.js";
export declare function resolveModelGpuLayersOption(gpuLayers: LlamaModelOptions["gpuLayers"], { ggufInsights, ignoreMemorySafetyChecks, getVramState, llamaVramPaddingSize, llamaGpu, llamaSupportsGpuOffloading, defaultContextFlashAttention, useMmap }: {
ggufInsights: GgufInsights;
ignoreMemorySafetyChecks?: boolean;
getVramState(): Promise<{
total: number;
free: number;
}>;
llamaVramPaddingSize: number;
llamaGpu: BuildGpu;
llamaSupportsGpuOffloading: boolean;
defaultContextFlashAttention: boolean;
useMmap?: boolean;
}): Promise<number>;