@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 6.93 kB
Source Map (JSON)
{"version":3,"file":"llama_cpp.cjs","names":["LLM","createLlamaModel","createLlamaContext","createLlamaJsonSchemaGrammar","createCustomGrammar","createLlamaSession","GenerationChunk"],"sources":["../../src/llms/llama_cpp.ts"],"sourcesContent":["import {\n LlamaModel,\n LlamaContext,\n LlamaChatSession,\n LlamaJsonSchemaGrammar,\n LlamaGrammar,\n getLlama,\n GbnfJsonSchema,\n} from \"node-llama-cpp\";\nimport {\n LLM,\n type BaseLLMCallOptions,\n type BaseLLMParams,\n} from \"@langchain/core/language_models/llms\";\nimport { CallbackManagerForLLMRun } from \"@langchain/core/callbacks/manager\";\nimport { GenerationChunk } from \"@langchain/core/outputs\";\n\nimport {\n LlamaBaseCppInputs,\n createLlamaModel,\n createLlamaContext,\n createLlamaSession,\n createLlamaJsonSchemaGrammar,\n createCustomGrammar,\n} from \"../utils/llama_cpp.js\";\n\n/**\n * Note that the modelPath is the only required parameter. For testing you\n * can set this in the environment variable `LLAMA_PATH`.\n */\nexport interface LlamaCppInputs extends LlamaBaseCppInputs, BaseLLMParams {}\n\nexport interface LlamaCppCallOptions extends BaseLLMCallOptions {\n /** The maximum number of tokens the response should contain. */\n maxTokens?: number;\n /** A function called when matching the provided token array */\n onToken?: (tokens: number[]) => void;\n}\n\n/**\n * To use this model you need to have the `node-llama-cpp` module installed.\n * This can be installed using `npm install -S node-llama-cpp` and the minimum\n * version supported in version 2.0.0.\n * This also requires that have a locally built version of Llama3 installed.\n */\nexport class LlamaCpp extends LLM<LlamaCppCallOptions> {\n lc_serializable = true;\n\n static inputs: LlamaCppInputs;\n\n maxTokens?: number;\n\n temperature?: number;\n\n topK?: number;\n\n topP?: number;\n\n trimWhitespaceSuffix?: boolean;\n\n _model: LlamaModel;\n\n _context: LlamaContext;\n\n _session: LlamaChatSession;\n\n _jsonSchema: LlamaJsonSchemaGrammar<GbnfJsonSchema> | undefined;\n\n _gbnf: LlamaGrammar | undefined;\n\n static lc_name() {\n return \"LlamaCpp\";\n }\n\n public constructor(inputs: LlamaCppInputs) {\n super(inputs);\n this.maxTokens = inputs?.maxTokens;\n this.temperature = inputs?.temperature;\n this.topK = inputs?.topK;\n this.topP = inputs?.topP;\n this.trimWhitespaceSuffix = inputs?.trimWhitespaceSuffix;\n }\n\n /**\n * Initializes the llama_cpp model for usage.\n * @param inputs - the inputs passed onto the model.\n * @returns A Promise that resolves to the LlamaCpp type class.\n */\n public static async initialize(inputs: LlamaCppInputs): Promise<LlamaCpp> {\n const instance = new LlamaCpp(inputs);\n const llama = await getLlama();\n\n instance._model = await createLlamaModel(inputs, llama);\n instance._context = await createLlamaContext(instance._model, inputs);\n instance._jsonSchema = await createLlamaJsonSchemaGrammar(\n inputs?.jsonSchema,\n llama\n );\n instance._gbnf = await createCustomGrammar(inputs?.gbnf, llama);\n instance._session = createLlamaSession(instance._context);\n\n return instance;\n }\n\n _llmType() {\n return \"llama_cpp\";\n }\n\n /** @ignore */\n async _call(\n prompt: string,\n options?: this[\"ParsedCallOptions\"]\n ): Promise<string> {\n try {\n let promptGrammer;\n\n if (this._jsonSchema !== undefined) {\n promptGrammer = this._jsonSchema;\n } else if (this._gbnf !== undefined) {\n promptGrammer = this._gbnf;\n } else {\n promptGrammer = undefined;\n }\n const promptOptions = {\n grammar: promptGrammer,\n onToken: options?.onToken,\n maxTokens: this?.maxTokens,\n temperature: this?.temperature,\n topK: this?.topK,\n topP: this?.topP,\n trimWhitespaceSuffix: this?.trimWhitespaceSuffix,\n };\n\n const completion = await this._session.prompt(prompt, promptOptions);\n\n if (this._jsonSchema !== undefined && completion !== undefined) {\n return this._jsonSchema.parse(completion) as unknown as string;\n }\n\n return completion;\n } catch {\n throw new Error(\"Error getting prompt completion.\");\n }\n }\n\n async *_streamResponseChunks(\n prompt: string,\n _options: this[\"ParsedCallOptions\"],\n runManager?: CallbackManagerForLLMRun\n ): AsyncGenerator<GenerationChunk> {\n const promptOptions = {\n temperature: this?.temperature,\n maxTokens: this?.maxTokens,\n topK: this?.topK,\n topP: this?.topP,\n };\n\n if (this._context.sequencesLeft === 0) {\n this._context = await createLlamaContext(this._model, LlamaCpp.inputs);\n }\n const sequence = this._context.getSequence();\n const tokens = this._model.tokenize(prompt);\n\n const stream = await this.caller.call(async () =>\n sequence.evaluate(tokens, promptOptions)\n );\n\n for await (const chunk of stream) {\n yield new GenerationChunk({\n text: this._model.detokenize([chunk]),\n generationInfo: {},\n });\n await runManager?.handleLLMNewToken(\n this._model.detokenize([chunk]) ?? \"\"\n );\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;AA6CA,IAAa,WAAb,MAAa,iBAAiBA,qCAAAA,IAAyB;CACrD,kBAAkB;CAElB,OAAO;CAEP;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,OAAO,UAAU;AACf,SAAO;;CAGT,YAAmB,QAAwB;AACzC,QAAM,OAAO;AACb,OAAK,YAAY,QAAQ;AACzB,OAAK,cAAc,QAAQ;AAC3B,OAAK,OAAO,QAAQ;AACpB,OAAK,OAAO,QAAQ;AACpB,OAAK,uBAAuB,QAAQ;;;;;;;CAQtC,aAAoB,WAAW,QAA2C;EACxE,MAAM,WAAW,IAAI,SAAS,OAAO;EACrC,MAAM,QAAQ,OAAA,GAAA,eAAA,WAAgB;AAE9B,WAAS,SAAS,MAAMC,kBAAAA,iBAAiB,QAAQ,MAAM;AACvD,WAAS,WAAW,MAAMC,kBAAAA,mBAAmB,SAAS,QAAQ,OAAO;AACrE,WAAS,cAAc,MAAMC,kBAAAA,6BAC3B,QAAQ,YACR,MACD;AACD,WAAS,QAAQ,MAAMC,kBAAAA,oBAAoB,QAAQ,MAAM,MAAM;AAC/D,WAAS,WAAWC,kBAAAA,mBAAmB,SAAS,SAAS;AAEzD,SAAO;;CAGT,WAAW;AACT,SAAO;;;CAIT,MAAM,MACJ,QACA,SACiB;AACjB,MAAI;GACF,IAAI;AAEJ,OAAI,KAAK,gBAAgB,KAAA,EACvB,iBAAgB,KAAK;YACZ,KAAK,UAAU,KAAA,EACxB,iBAAgB,KAAK;OAErB,iBAAgB,KAAA;GAElB,MAAM,gBAAgB;IACpB,SAAS;IACT,SAAS,SAAS;IAClB,WAAW,MAAM;IACjB,aAAa,MAAM;IACnB,MAAM,MAAM;IACZ,MAAM,MAAM;IACZ,sBAAsB,MAAM;IAC7B;GAED,MAAM,aAAa,MAAM,KAAK,SAAS,OAAO,QAAQ,cAAc;AAEpE,OAAI,KAAK,gBAAgB,KAAA,KAAa,eAAe,KAAA,EACnD,QAAO,KAAK,YAAY,MAAM,WAAW;AAG3C,UAAO;UACD;AACN,SAAM,IAAI,MAAM,mCAAmC;;;CAIvD,OAAO,sBACL,QACA,UACA,YACiC;EACjC,MAAM,gBAAgB;GACpB,aAAa,MAAM;GACnB,WAAW,MAAM;GACjB,MAAM,MAAM;GACZ,MAAM,MAAM;GACb;AAED,MAAI,KAAK,SAAS,kBAAkB,EAClC,MAAK,WAAW,MAAMH,kBAAAA,mBAAmB,KAAK,QAAQ,SAAS,OAAO;EAExE,MAAM,WAAW,KAAK,SAAS,aAAa;EAC5C,MAAM,SAAS,KAAK,OAAO,SAAS,OAAO;EAE3C,MAAM,SAAS,MAAM,KAAK,OAAO,KAAK,YACpC,SAAS,SAAS,QAAQ,cAAc,CACzC;AAED,aAAW,MAAM,SAAS,QAAQ;AAChC,SAAM,IAAII,wBAAAA,gBAAgB;IACxB,MAAM,KAAK,OAAO,WAAW,CAAC,MAAM,CAAC;IACrC,gBAAgB,EAAE;IACnB,CAAC;AACF,SAAM,YAAY,kBAChB,KAAK,OAAO,WAAW,CAAC,MAAM,CAAC,IAAI,GACpC"}