@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 7.04 kB
Source Map (JSON)
{"version":3,"file":"llama_cpp.cjs","names":["LLM","inputs: LlamaCppInputs","createLlamaModel","createLlamaContext","createLlamaJsonSchemaGrammar","createCustomGrammar","createLlamaSession","prompt: string","options?: this[\"ParsedCallOptions\"]","_options: this[\"ParsedCallOptions\"]","runManager?: CallbackManagerForLLMRun","GenerationChunk"],"sources":["../../src/llms/llama_cpp.ts"],"sourcesContent":["import {\n LlamaModel,\n LlamaContext,\n LlamaChatSession,\n LlamaJsonSchemaGrammar,\n LlamaGrammar,\n getLlama,\n GbnfJsonSchema,\n} from \"node-llama-cpp\";\nimport {\n LLM,\n type BaseLLMCallOptions,\n type BaseLLMParams,\n} from \"@langchain/core/language_models/llms\";\nimport { CallbackManagerForLLMRun } from \"@langchain/core/callbacks/manager\";\nimport { GenerationChunk } from \"@langchain/core/outputs\";\n\nimport {\n LlamaBaseCppInputs,\n createLlamaModel,\n createLlamaContext,\n createLlamaSession,\n createLlamaJsonSchemaGrammar,\n createCustomGrammar,\n} from \"../utils/llama_cpp.js\";\n\n/**\n * Note that the modelPath is the only required parameter. For testing you\n * can set this in the environment variable `LLAMA_PATH`.\n */\nexport interface LlamaCppInputs extends LlamaBaseCppInputs, BaseLLMParams {}\n\nexport interface LlamaCppCallOptions extends BaseLLMCallOptions {\n /** The maximum number of tokens the response should contain. */\n maxTokens?: number;\n /** A function called when matching the provided token array */\n onToken?: (tokens: number[]) => void;\n}\n\n/**\n * To use this model you need to have the `node-llama-cpp` module installed.\n * This can be installed using `npm install -S node-llama-cpp` and the minimum\n * version supported in version 2.0.0.\n * This also requires that have a locally built version of Llama3 installed.\n */\nexport class LlamaCpp extends LLM<LlamaCppCallOptions> {\n lc_serializable = true;\n\n static inputs: LlamaCppInputs;\n\n maxTokens?: number;\n\n temperature?: number;\n\n topK?: number;\n\n topP?: number;\n\n trimWhitespaceSuffix?: boolean;\n\n _model: LlamaModel;\n\n _context: LlamaContext;\n\n _session: LlamaChatSession;\n\n _jsonSchema: LlamaJsonSchemaGrammar<GbnfJsonSchema> | undefined;\n\n _gbnf: LlamaGrammar | undefined;\n\n static lc_name() {\n return \"LlamaCpp\";\n }\n\n public constructor(inputs: LlamaCppInputs) {\n super(inputs);\n this.maxTokens = inputs?.maxTokens;\n this.temperature = inputs?.temperature;\n this.topK = inputs?.topK;\n this.topP = inputs?.topP;\n this.trimWhitespaceSuffix = inputs?.trimWhitespaceSuffix;\n }\n\n /**\n * Initializes the llama_cpp model for usage.\n * @param inputs - the inputs passed onto the model.\n * @returns A Promise that resolves to the LlamaCpp type class.\n */\n public static async initialize(inputs: LlamaCppInputs): Promise<LlamaCpp> {\n const instance = new LlamaCpp(inputs);\n const llama = await getLlama();\n\n instance._model = await createLlamaModel(inputs, llama);\n instance._context = await createLlamaContext(instance._model, inputs);\n instance._jsonSchema = await createLlamaJsonSchemaGrammar(\n inputs?.jsonSchema,\n llama\n );\n instance._gbnf = await createCustomGrammar(inputs?.gbnf, llama);\n instance._session = createLlamaSession(instance._context);\n\n return instance;\n }\n\n _llmType() {\n return \"llama_cpp\";\n }\n\n /** @ignore */\n async _call(\n prompt: string,\n options?: this[\"ParsedCallOptions\"]\n ): Promise<string> {\n try {\n let promptGrammer;\n\n if (this._jsonSchema !== undefined) {\n promptGrammer = this._jsonSchema;\n } else if (this._gbnf !== undefined) {\n promptGrammer = this._gbnf;\n } else {\n promptGrammer = undefined;\n }\n const promptOptions = {\n grammar: promptGrammer,\n onToken: options?.onToken,\n maxTokens: this?.maxTokens,\n temperature: this?.temperature,\n topK: this?.topK,\n topP: this?.topP,\n trimWhitespaceSuffix: this?.trimWhitespaceSuffix,\n };\n\n const completion = await this._session.prompt(prompt, promptOptions);\n\n if (this._jsonSchema !== undefined && completion !== undefined) {\n return this._jsonSchema.parse(completion) as unknown as string;\n }\n\n return completion;\n } catch {\n throw new Error(\"Error getting prompt completion.\");\n }\n }\n\n async *_streamResponseChunks(\n prompt: string,\n _options: this[\"ParsedCallOptions\"],\n runManager?: CallbackManagerForLLMRun\n ): AsyncGenerator<GenerationChunk> {\n const promptOptions = {\n temperature: this?.temperature,\n maxTokens: this?.maxTokens,\n topK: this?.topK,\n topP: this?.topP,\n };\n\n if (this._context.sequencesLeft === 0) {\n this._context = await createLlamaContext(this._model, LlamaCpp.inputs);\n }\n const sequence = this._context.getSequence();\n const tokens = this._model.tokenize(prompt);\n\n const stream = await this.caller.call(async () =>\n sequence.evaluate(tokens, promptOptions)\n );\n\n for await (const chunk of stream) {\n yield new GenerationChunk({\n text: this._model.detokenize([chunk]),\n generationInfo: {},\n });\n await runManager?.handleLLMNewToken(\n this._model.detokenize([chunk]) ?? \"\"\n );\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;AA6CA,IAAa,WAAb,MAAa,iBAAiBA,0CAAyB;CACrD,kBAAkB;CAElB,OAAO;CAEP;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,OAAO,UAAU;AACf,SAAO;CACR;CAED,AAAO,YAAYC,QAAwB;EACzC,MAAM,OAAO;EACb,KAAK,YAAY,QAAQ;EACzB,KAAK,cAAc,QAAQ;EAC3B,KAAK,OAAO,QAAQ;EACpB,KAAK,OAAO,QAAQ;EACpB,KAAK,uBAAuB,QAAQ;CACrC;;;;;;CAOD,aAAoB,WAAWA,QAA2C;EACxE,MAAM,WAAW,IAAI,SAAS;EAC9B,MAAM,QAAQ,oCAAgB;EAE9B,SAAS,SAAS,MAAMC,mCAAiB,QAAQ,MAAM;EACvD,SAAS,WAAW,MAAMC,qCAAmB,SAAS,QAAQ,OAAO;EACrE,SAAS,cAAc,MAAMC,+CAC3B,QAAQ,YACR,MACD;EACD,SAAS,QAAQ,MAAMC,sCAAoB,QAAQ,MAAM,MAAM;EAC/D,SAAS,WAAWC,qCAAmB,SAAS,SAAS;AAEzD,SAAO;CACR;CAED,WAAW;AACT,SAAO;CACR;;CAGD,MAAM,MACJC,QACAC,SACiB;AACjB,MAAI;GACF,IAAI;AAEJ,OAAI,KAAK,gBAAgB,QACvB,gBAAgB,KAAK;YACZ,KAAK,UAAU,QACxB,gBAAgB,KAAK;QAErB,gBAAgB;GAElB,MAAM,gBAAgB;IACpB,SAAS;IACT,SAAS,SAAS;IAClB,WAAW,MAAM;IACjB,aAAa,MAAM;IACnB,MAAM,MAAM;IACZ,MAAM,MAAM;IACZ,sBAAsB,MAAM;GAC7B;GAED,MAAM,aAAa,MAAM,KAAK,SAAS,OAAO,QAAQ,cAAc;AAEpE,OAAI,KAAK,gBAAgB,UAAa,eAAe,OACnD,QAAO,KAAK,YAAY,MAAM,WAAW;AAG3C,UAAO;EACR,QAAO;AACN,SAAM,IAAI,MAAM;EACjB;CACF;CAED,OAAO,sBACLD,QACAE,UACAC,YACiC;EACjC,MAAM,gBAAgB;GACpB,aAAa,MAAM;GACnB,WAAW,MAAM;GACjB,MAAM,MAAM;GACZ,MAAM,MAAM;EACb;AAED,MAAI,KAAK,SAAS,kBAAkB,GAClC,KAAK,WAAW,MAAMP,qCAAmB,KAAK,QAAQ,SAAS,OAAO;EAExE,MAAM,WAAW,KAAK,SAAS,aAAa;EAC5C,MAAM,SAAS,KAAK,OAAO,SAAS,OAAO;EAE3C,MAAM,SAAS,MAAM,KAAK,OAAO,KAAK,YACpC,SAAS,SAAS,QAAQ,cAAc,CACzC;AAED,aAAW,MAAM,SAAS,QAAQ;GAChC,MAAM,IAAIQ,yCAAgB;IACxB,MAAM,KAAK,OAAO,WAAW,CAAC,KAAM,EAAC;IACrC,gBAAgB,CAAE;GACnB;GACD,MAAM,YAAY,kBAChB,KAAK,OAAO,WAAW,CAAC,KAAM,EAAC,IAAI,GACpC;EACF;CACF;AACF"}