@langchain/community/dist/chat_models/llama_cpp.cjs.map

Third-party integrations for LangChain.js

{"version":3,"file":"llama_cpp.cjs","names":["SimpleChatModel","inputs: LlamaCppInputs","inputs: LlamaBaseCppInputs","createLlamaModel","createLlamaContext","messages: BaseMessage[]","options: this[\"ParsedCallOptions\"]","runManager?: CallbackManagerForLLMRun","tokens: number[]","input: BaseMessage[]","_options: this[\"ParsedCallOptions\"]","ChatGenerationChunk","AIMessageChunk","noSystemMessages: BaseMessage[]","interactions: ChatHistoryItem[]","LlamaChatSession","result: ChatHistoryItem[]","llamaPrompt: ChatUserMessage","llamaResponse: ChatModelResponse","ChatMessage"],"sources":["../../src/chat_models/llama_cpp.ts"],"sourcesContent":["import {\n  LlamaModel,\n  LlamaContext,\n  LlamaChatSession,\n  type Token,\n  ChatUserMessage,\n  ChatModelResponse,\n  ChatHistoryItem,\n  getLlama,\n} from \"node-llama-cpp\";\n\nimport {\n  SimpleChatModel,\n  type BaseChatModelParams,\n} from \"@langchain/core/language_models/chat_models\";\nimport type { BaseLanguageModelCallOptions } from \"@langchain/core/language_models/base\";\nimport { CallbackManagerForLLMRun } from \"@langchain/core/callbacks/manager\";\nimport {\n  BaseMessage,\n  AIMessageChunk,\n  ChatMessage,\n} from \"@langchain/core/messages\";\nimport { ChatGenerationChunk } from \"@langchain/core/outputs\";\nimport {\n  LlamaBaseCppInputs,\n  createLlamaModel,\n  createLlamaContext,\n} from \"../utils/llama_cpp.js\";\n\n/**\n * Note that the modelPath is the only required parameter. For testing you\n * can set this in the environment variable `LLAMA_PATH`.\n */\nexport interface LlamaCppInputs\n  extends LlamaBaseCppInputs,\n    BaseChatModelParams {}\n\nexport interface LlamaCppCallOptions extends BaseLanguageModelCallOptions {\n  /** The maximum number of tokens the response should contain. */\n  maxTokens?: number;\n  /** A function called when matching the provided token array */\n  onToken?: (tokens: number[]) => void;\n}\n\n/**\n *  To use this model you need to have the `node-llama-cpp` module installed.\n *  This can be installed using `npm install -S node-llama-cpp` and the minimum\n *  version supported in version 2.0.0.\n *  This also requires that have a locally built version of Llama3 installed.\n * @example\n * ```typescript\n * // Initialize the ChatLlamaCpp model with the path to the model binary file.\n * const model = await ChatLlamaCpp.initialize({\n *   modelPath: \"/Replace/with/path/to/your/model/gguf-llama3-Q4_0.bin\",\n *   temperature: 0.5,\n * });\n *\n * // Call the model with a message and await the response.\n * const response = await model.invoke([\n *   new HumanMessage({ content: \"My name is John.\" }),\n * ]);\n *\n * // Log the response to the console.\n * console.log({ response });\n *\n * ```\n */\nexport class ChatLlamaCpp extends SimpleChatModel<LlamaCppCallOptions> {\n  static inputs: LlamaCppInputs;\n\n  maxTokens?: number;\n\n  temperature?: number;\n\n  topK?: number;\n\n  topP?: number;\n\n  trimWhitespaceSuffix?: boolean;\n\n  _model: LlamaModel;\n\n  _context: LlamaContext;\n\n  _session: LlamaChatSession | null;\n\n  lc_serializable = true;\n\n  static lc_name() {\n    return \"ChatLlamaCpp\";\n  }\n\n  public constructor(inputs: LlamaCppInputs) {\n    super(inputs);\n    this.maxTokens = inputs?.maxTokens;\n    this.temperature = inputs?.temperature;\n    this.topK = inputs?.topK;\n    this.topP = inputs?.topP;\n    this.trimWhitespaceSuffix = inputs?.trimWhitespaceSuffix;\n    this._session = null;\n  }\n\n  /**\n   * Initializes the llama_cpp model for usage in the chat models wrapper.\n   * @param inputs - the inputs passed onto the model.\n   * @returns A Promise that resolves to the ChatLlamaCpp type class.\n   */\n  public static async initialize(\n    inputs: LlamaBaseCppInputs\n  ): Promise<ChatLlamaCpp> {\n    const instance = new ChatLlamaCpp(inputs);\n    const llama = await getLlama();\n\n    instance._model = await createLlamaModel(inputs, llama);\n    instance._context = await createLlamaContext(instance._model, inputs);\n\n    return instance;\n  }\n\n  _llmType() {\n    return \"llama_cpp\";\n  }\n\n  /** @ignore */\n  _combineLLMOutput() {\n    return {};\n  }\n\n  invocationParams() {\n    return {\n      maxTokens: this.maxTokens,\n      temperature: this.temperature,\n      topK: this.topK,\n      topP: this.topP,\n      trimWhitespaceSuffix: this.trimWhitespaceSuffix,\n    };\n  }\n\n  /** @ignore */\n  async _call(\n    messages: BaseMessage[],\n    options: this[\"ParsedCallOptions\"],\n    runManager?: CallbackManagerForLLMRun\n  ): Promise<string> {\n    let prompt = \"\";\n\n    if (messages.length > 1) {\n      // We need to build a new _session\n      prompt = this._buildSession(messages);\n    } else if (!this._session) {\n      prompt = this._buildSession(messages);\n    } else {\n      if (typeof messages[0].content !== \"string\") {\n        throw new Error(\n          \"ChatLlamaCpp does not support non-string message content in sessions.\"\n        );\n      }\n      // If we already have a session then we should just have a single prompt\n      prompt = messages[0].content;\n    }\n\n    try {\n      const promptOptions = {\n        signal: options.signal,\n        onToken: async (tokens: number[]) => {\n          options.onToken?.(tokens);\n          await runManager?.handleLLMNewToken(\n            this._model.detokenize(tokens.map((num) => num as Token))\n          );\n        },\n        maxTokens: this?.maxTokens,\n        temperature: this?.temperature,\n        topK: this?.topK,\n        topP: this?.topP,\n        trimWhitespaceSuffix: this?.trimWhitespaceSuffix,\n      };\n      // @ts-expect-error - TS2531: Object is possibly 'null'.\n      const completion = await this._session.prompt(prompt, promptOptions);\n      return completion;\n    } catch (e) {\n      if (typeof e === \"object\") {\n        const error = e as Error;\n        if (error.message === \"AbortError\") {\n          throw error;\n        }\n      }\n      throw new Error(\"Error getting prompt completion.\");\n    }\n  }\n\n  async *_streamResponseChunks(\n    input: BaseMessage[],\n    _options: this[\"ParsedCallOptions\"],\n    runManager?: CallbackManagerForLLMRun\n  ): AsyncGenerator<ChatGenerationChunk> {\n    const promptOptions = {\n      temperature: this?.temperature,\n      topK: this?.topK,\n      topP: this?.topP,\n    };\n\n    const prompt = this._buildPrompt(input);\n    const sequence = this._context.getSequence();\n\n    const stream = await this.caller.call(async () =>\n      sequence.evaluate(this._model.tokenize(prompt), promptOptions)\n    );\n\n    for await (const chunk of stream) {\n      yield new ChatGenerationChunk({\n        text: this._model.detokenize([chunk]),\n        message: new AIMessageChunk({\n          content: this._model.detokenize([chunk]),\n        }),\n        generationInfo: {},\n      });\n      await runManager?.handleLLMNewToken(\n        this._model.detokenize([chunk]) ?? \"\"\n      );\n    }\n  }\n\n  // This constructs a new session if we need to adding in any sys messages or previous chats\n  protected _buildSession(messages: BaseMessage[]): string {\n    let prompt = \"\";\n    let sysMessage = \"\";\n    let noSystemMessages: BaseMessage[] = [];\n    let interactions: ChatHistoryItem[] = [];\n\n    // Let's see if we have a system message\n    if (messages.findIndex((msg) => msg.getType() === \"system\") !== -1) {\n      const sysMessages = messages.filter(\n        (message) => message.getType() === \"system\"\n      );\n\n      const systemMessageContent = sysMessages[sysMessages.length - 1].content;\n\n      if (typeof systemMessageContent !== \"string\") {\n        throw new Error(\n          \"ChatLlamaCpp does not support non-string message content in sessions.\"\n        );\n      }\n      // Only use the last provided system message\n      sysMessage = systemMessageContent;\n\n      // Now filter out the system messages\n      noSystemMessages = messages.filter(\n        (message) => message.getType() !== \"system\"\n      );\n    } else {\n      noSystemMessages = messages;\n    }\n\n    // Lets see if we just have a prompt left or are their previous interactions?\n    if (noSystemMessages.length > 1) {\n      // Is the last message a prompt?\n      if (noSystemMessages[noSystemMessages.length - 1].getType() === \"human\") {\n        const finalMessageContent =\n          noSystemMessages[noSystemMessages.length - 1].content;\n        if (typeof finalMessageContent !== \"string\") {\n          throw new Error(\n            \"ChatLlamaCpp does not support non-string message content in sessions.\"\n          );\n        }\n        prompt = finalMessageContent;\n        interactions = this._convertMessagesToInteractions(\n          noSystemMessages.slice(0, noSystemMessages.length - 1)\n        );\n      } else {\n        interactions = this._convertMessagesToInteractions(noSystemMessages);\n      }\n    } else {\n      if (typeof noSystemMessages[0].content !== \"string\") {\n        throw new Error(\n          \"ChatLlamaCpp does not support non-string message content in sessions.\"\n        );\n      }\n      // If there was only a single message we assume it's a prompt\n      prompt = noSystemMessages[0].content;\n    }\n\n    // Now lets construct a session according to what we got\n    if (sysMessage !== \"\" && interactions.length > 0) {\n      this._session = new LlamaChatSession({\n        contextSequence: this._context.getSequence(),\n        systemPrompt: sysMessage,\n      });\n      this._session.setChatHistory(interactions);\n    } else if (sysMessage !== \"\" && interactions.length === 0) {\n      this._session = new LlamaChatSession({\n        contextSequence: this._context.getSequence(),\n        systemPrompt: sysMessage,\n      });\n    } else if (sysMessage === \"\" && interactions.length > 0) {\n      this._session = new LlamaChatSession({\n        contextSequence: this._context.getSequence(),\n      });\n      this._session.setChatHistory(interactions);\n    } else {\n      this._session = new LlamaChatSession({\n        contextSequence: this._context.getSequence(),\n      });\n    }\n\n    return prompt;\n  }\n\n  // This builds a an array of interactions\n  protected _convertMessagesToInteractions(\n    messages: BaseMessage[]\n  ): ChatHistoryItem[] {\n    const result: ChatHistoryItem[] = [];\n\n    for (let i = 0; i < messages.length; i += 2) {\n      if (i + 1 < messages.length) {\n        const prompt = messages[i].content;\n        const response = messages[i + 1].content;\n        if (typeof prompt !== \"string\" || typeof response !== \"string\") {\n          throw new Error(\n            \"ChatLlamaCpp does not support non-string message content.\"\n          );\n        }\n        const llamaPrompt: ChatUserMessage = { type: \"user\", text: prompt };\n        const llamaResponse: ChatModelResponse = {\n          type: \"model\",\n          response: [response],\n        };\n        result.push(llamaPrompt);\n        result.push(llamaResponse);\n      }\n    }\n\n    return result;\n  }\n\n  protected _buildPrompt(input: BaseMessage[]): string {\n    const prompt = input\n      .map((message) => {\n        let messageText;\n        if (message.getType() === \"human\") {\n          messageText = `[INST] ${message.content} [/INST]`;\n        } else if (message.getType() === \"ai\") {\n          messageText = message.content;\n        } else if (message.getType() === \"system\") {\n          messageText = `<<SYS>> ${message.content} <</SYS>>`;\n        } else if (ChatMessage.isInstance(message)) {\n          messageText = `\\n\\n${message.role[0].toUpperCase()}${message.role.slice(\n            1\n          )}: ${message.content}`;\n        } else {\n          console.warn(\n            `Unsupported message type passed to llama_cpp: \"${message.getType()}\"`\n          );\n          messageText = \"\";\n        }\n        return messageText;\n      })\n      .join(\"\\n\");\n\n    return prompt;\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAmEA,IAAa,eAAb,MAAa,qBAAqBA,6DAAqC;CACrE,OAAO;CAEP;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA;CAEA,kBAAkB;CAElB,OAAO,UAAU;AACf,SAAO;CACR;CAED,AAAO,YAAYC,QAAwB;EACzC,MAAM,OAAO;EACb,KAAK,YAAY,QAAQ;EACzB,KAAK,cAAc,QAAQ;EAC3B,KAAK,OAAO,QAAQ;EACpB,KAAK,OAAO,QAAQ;EACpB,KAAK,uBAAuB,QAAQ;EACpC,KAAK,WAAW;CACjB;;;;;;CAOD,aAAoB,WAClBC,QACuB;EACvB,MAAM,WAAW,IAAI,aAAa;EAClC,MAAM,QAAQ,oCAAgB;EAE9B,SAAS,SAAS,MAAMC,mCAAiB,QAAQ,MAAM;EACvD,SAAS,WAAW,MAAMC,qCAAmB,SAAS,QAAQ,OAAO;AAErE,SAAO;CACR;CAED,WAAW;AACT,SAAO;CACR;;CAGD,oBAAoB;AAClB,SAAO,CAAE;CACV;CAED,mBAAmB;AACjB,SAAO;GACL,WAAW,KAAK;GAChB,aAAa,KAAK;GAClB,MAAM,KAAK;GACX,MAAM,KAAK;GACX,sBAAsB,KAAK;EAC5B;CACF;;CAGD,MAAM,MACJC,UACAC,SACAC,YACiB;EACjB,IAAI,SAAS;AAEb,MAAI,SAAS,SAAS,GAEpB,SAAS,KAAK,cAAc,SAAS;WAC5B,CAAC,KAAK,UACf,SAAS,KAAK,cAAc,SAAS;OAChC;AACL,OAAI,OAAO,SAAS,GAAG,YAAY,SACjC,OAAM,IAAI,MACR;GAIJ,SAAS,SAAS,GAAG;EACtB;AAED,MAAI;GACF,MAAM,gBAAgB;IACpB,QAAQ,QAAQ;IAChB,SAAS,OAAOC,WAAqB;KACnC,QAAQ,UAAU,OAAO;KACzB,MAAM,YAAY,kBAChB,KAAK,OAAO,WAAW,OAAO,IAAI,CAAC,QAAQ,IAAa,CAAC,CAC1D;IACF;IACD,WAAW,MAAM;IACjB,aAAa,MAAM;IACnB,MAAM,MAAM;IACZ,MAAM,MAAM;IACZ,sBAAsB,MAAM;GAC7B;GAED,MAAM,aAAa,MAAM,KAAK,SAAS,OAAO,QAAQ,cAAc;AACpE,UAAO;EACR,SAAQ,GAAG;AACV,OAAI,OAAO,MAAM,UAAU;IACzB,MAAM,QAAQ;AACd,QAAI,MAAM,YAAY,aACpB,OAAM;GAET;AACD,SAAM,IAAI,MAAM;EACjB;CACF;CAED,OAAO,sBACLC,OACAC,UACAH,YACqC;EACrC,MAAM,gBAAgB;GACpB,aAAa,MAAM;GACnB,MAAM,MAAM;GACZ,MAAM,MAAM;EACb;EAED,MAAM,SAAS,KAAK,aAAa,MAAM;EACvC,MAAM,WAAW,KAAK,SAAS,aAAa;EAE5C,MAAM,SAAS,MAAM,KAAK,OAAO,KAAK,YACpC,SAAS,SAAS,KAAK,OAAO,SAAS,OAAO,EAAE,cAAc,CAC/D;AAED,aAAW,MAAM,SAAS,QAAQ;GAChC,MAAM,IAAII,6CAAoB;IAC5B,MAAM,KAAK,OAAO,WAAW,CAAC,KAAM,EAAC;IACrC,SAAS,IAAIC,yCAAe,EAC1B,SAAS,KAAK,OAAO,WAAW,CAAC,KAAM,EAAC,CACzC;IACD,gBAAgB,CAAE;GACnB;GACD,MAAM,YAAY,kBAChB,KAAK,OAAO,WAAW,CAAC,KAAM,EAAC,IAAI,GACpC;EACF;CACF;CAGD,AAAU,cAAcP,UAAiC;EACvD,IAAI,SAAS;EACb,IAAI,aAAa;EACjB,IAAIQ,mBAAkC,CAAE;EACxC,IAAIC,eAAkC,CAAE;AAGxC,MAAI,SAAS,UAAU,CAAC,QAAQ,IAAI,SAAS,KAAK,SAAS,KAAK,IAAI;GAClE,MAAM,cAAc,SAAS,OAC3B,CAAC,YAAY,QAAQ,SAAS,KAAK,SACpC;GAED,MAAM,uBAAuB,YAAY,YAAY,SAAS,GAAG;AAEjE,OAAI,OAAO,yBAAyB,SAClC,OAAM,IAAI,MACR;GAIJ,aAAa;GAGb,mBAAmB,SAAS,OAC1B,CAAC,YAAY,QAAQ,SAAS,KAAK,SACpC;EACF,OACC,mBAAmB;AAIrB,MAAI,iBAAiB,SAAS,EAE5B,KAAI,iBAAiB,iBAAiB,SAAS,GAAG,SAAS,KAAK,SAAS;GACvE,MAAM,sBACJ,iBAAiB,iBAAiB,SAAS,GAAG;AAChD,OAAI,OAAO,wBAAwB,SACjC,OAAM,IAAI,MACR;GAGJ,SAAS;GACT,eAAe,KAAK,+BAClB,iBAAiB,MAAM,GAAG,iBAAiB,SAAS,EAAE,CACvD;EACF,OACC,eAAe,KAAK,+BAA+B,iBAAiB;OAEjE;AACL,OAAI,OAAO,iBAAiB,GAAG,YAAY,SACzC,OAAM,IAAI,MACR;GAIJ,SAAS,iBAAiB,GAAG;EAC9B;AAGD,MAAI,eAAe,MAAM,aAAa,SAAS,GAAG;GAChD,KAAK,WAAW,IAAIC,gCAAiB;IACnC,iBAAiB,KAAK,SAAS,aAAa;IAC5C,cAAc;GACf;GACD,KAAK,SAAS,eAAe,aAAa;EAC3C,WAAU,eAAe,MAAM,aAAa,WAAW,GACtD,KAAK,WAAW,IAAIA,gCAAiB;GACnC,iBAAiB,KAAK,SAAS,aAAa;GAC5C,cAAc;EACf;WACQ,eAAe,MAAM,aAAa,SAAS,GAAG;GACvD,KAAK,WAAW,IAAIA,gCAAiB,EACnC,iBAAiB,KAAK,SAAS,aAAa,CAC7C;GACD,KAAK,SAAS,eAAe,aAAa;EAC3C,OACC,KAAK,WAAW,IAAIA,gCAAiB,EACnC,iBAAiB,KAAK,SAAS,aAAa,CAC7C;AAGH,SAAO;CACR;CAGD,AAAU,+BACRV,UACmB;EACnB,MAAMW,SAA4B,CAAE;AAEpC,OAAK,IAAI,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK,EACxC,KAAI,IAAI,IAAI,SAAS,QAAQ;GAC3B,MAAM,SAAS,SAAS,GAAG;GAC3B,MAAM,WAAW,SAAS,IAAI,GAAG;AACjC,OAAI,OAAO,WAAW,YAAY,OAAO,aAAa,SACpD,OAAM,IAAI,MACR;GAGJ,MAAMC,cAA+B;IAAE,MAAM;IAAQ,MAAM;GAAQ;GACnE,MAAMC,gBAAmC;IACvC,MAAM;IACN,UAAU,CAAC,QAAS;GACrB;GACD,OAAO,KAAK,YAAY;GACxB,OAAO,KAAK,cAAc;EAC3B;AAGH,SAAO;CACR;CAED,AAAU,aAAaT,OAA8B;EACnD,MAAM,SAAS,MACZ,IAAI,CAAC,YAAY;GAChB,IAAI;AACJ,OAAI,QAAQ,SAAS,KAAK,SACxB,cAAc,CAAC,OAAO,EAAE,QAAQ,QAAQ,QAAQ,CAAC;YACxC,QAAQ,SAAS,KAAK,MAC/B,cAAc,QAAQ;YACb,QAAQ,SAAS,KAAK,UAC/B,cAAc,CAAC,QAAQ,EAAE,QAAQ,QAAQ,SAAS,CAAC;YAC1CU,sCAAY,WAAW,QAAQ,EACxC,cAAc,CAAC,IAAI,EAAE,QAAQ,KAAK,GAAG,aAAa,GAAG,QAAQ,KAAK,MAChE,EACD,CAAC,EAAE,EAAE,QAAQ,SAAS;QAClB;IACL,QAAQ,KACN,CAAC,+CAA+C,EAAE,QAAQ,SAAS,CAAC,CAAC,CAAC,CACvE;IACD,cAAc;GACf;AACD,UAAO;EACR,EAAC,CACD,KAAK,KAAK;AAEb,SAAO;CACR;AACF"}