@juspay/neurolink/dist/providers/llamaCpp.js

The pipe layer of an AI nervous system: one interface connecting provider neurons to your application, across three inference types — generate, stream and decide. `decide` returns typed, calibrated judgements from a non-generative model (~400ms, ~$0.00002

import { ProviderError } from "../types/index.js";
import { classifyProviderError, DEFAULT_ERROR_RULES, } from "../utils/errorClassifier.js";
import { logger } from "../utils/logger.js";
import { redactUrlCredentials } from "../utils/logSanitize.js";
import { OpenAIChatCompletionsProvider } from "./openaiChatCompletionsBase.js";
import { buildLocalUnreachableErrorRule } from "./localRuntimeOpenAICompat.js";
const LLAMACPP_DEFAULT_BASE_URL = "http://localhost:8080/v1";
const LLAMACPP_PLACEHOLDER_KEY = "llamacpp";
const getLlamaCppBaseURL = () => {
    return process.env.LLAMACPP_BASE_URL || LLAMACPP_DEFAULT_BASE_URL;
};
/**
 * llama.cpp Provider — direct HTTP, no AI SDK.
 *
 * Wraps a llama-server process (https://github.com/ggerganov/llama.cpp) that
 * exposes an OpenAI-compatible API at http://localhost:8080/v1 by default.
 * llama-server hosts ONE model loaded at startup; /v1/models returns just that.
 * All request/stream/tool-loop orchestration lives in
 * `OpenAIChatCompletionsProvider`; this class only declares configuration
 * and provider-specific error mapping.
 *
 * @see https://github.com/ggerganov/llama.cpp
 */
export class LlamaCppProvider extends OpenAIChatCompletionsProvider {
    constructor(modelName, sdk, _region, credentials) {
        const baseURL = credentials?.baseURL?.trim() || getLlamaCppBaseURL();
        // llama-server doesn't authenticate, but the base class requires an
        // apiKey. Allow override via credentials/env for users who run
        // llama-server behind an auth-proxying reverse-proxy.
        const apiKey = credentials?.apiKey?.trim() ||
            process.env.LLAMACPP_API_KEY ||
            LLAMACPP_PLACEHOLDER_KEY;
        super("llamacpp", modelName, sdk, { baseURL, apiKey });
        logger.debug("llama.cpp Provider initialized", {
            modelName: this.modelName,
            providerName: this.providerName,
            baseURL: redactUrlCredentials(this.config.baseURL),
        });
    }
    async validateConfiguration() {
        return this.probeModelsEndpoint(this.getAuthHeaders());
    }
    getProviderName() {
        return "llamacpp";
    }
    getDefaultModel() {
        return process.env.LLAMACPP_MODEL || "";
    }
    getFallbackModelName() {
        return "loaded-model";
    }
    getFallbackModels() {
        return ["loaded-model"];
    }
    formatProviderError(error) {
        const rules = [
            buildLocalUnreachableErrorRule(error, () => `llama.cpp server not reachable at ${redactUrlCredentials(this.config.baseURL)}. ` +
                "Start it with: ./llama-server -m model.gguf --port 8080"),
            {
                match: (ctx) => ctx.statusCode === 400 ||
                    (ctx.statusCode === undefined && /\b400\b/.test(ctx.message)),
                errorClass: ProviderError,
                message: "llama.cpp rejected the request. Common cause: model doesn't support tools (start llama-server with --jinja for tool support).",
            },
            ...DEFAULT_ERROR_RULES,
        ];
        return classifyProviderError(error, rules, this.providerName, this.modelName);
    }
}