@langchain/community
Version:
Third-party integrations for LangChain.js
1 lines • 5.13 kB
Source Map (JSON)
{"version":3,"file":"llama_cpp.cjs","names":["inputs: LlamaBaseCppInputs","llama: Llama","options: LlamaModelOptions","model: LlamaModel","options: LlamaContextOptions","options: LlamaEmbeddingContextOptions","context: LlamaContext","LlamaChatSession","schemaString: object | undefined","filePath: string | undefined"],"sources":["../../src/utils/llama_cpp.ts"],"sourcesContent":["import {\n LlamaModel,\n LlamaContext,\n LlamaEmbeddingContext,\n LlamaChatSession,\n LlamaJsonSchemaGrammar,\n LlamaGrammar,\n type LlamaModelOptions,\n LlamaContextOptions,\n LlamaEmbeddingContextOptions,\n GbnfJsonSchema,\n Llama,\n} from \"node-llama-cpp\";\n\n/**\n * Note that the modelPath is the only required parameter. For testing you\n * can set this in the environment variable `LLAMA_PATH`.\n */\nexport interface LlamaBaseCppInputs {\n /** Prompt processing batch size. */\n batchSize?: number;\n /** Text context size. */\n contextSize?: number;\n /** Embedding mode only. */\n embedding?: boolean;\n /** Use fp16 for KV cache. */\n f16Kv?: boolean;\n /** Number of layers to store in VRAM. */\n gpuLayers?: number;\n /** The llama_eval() call computes all logits, not just the last one. */\n logitsAll?: boolean;\n /** */\n maxTokens?: number;\n /** Path to the model on the filesystem. */\n modelPath: string;\n /** Add the begining of sentence token. */\n prependBos?: boolean;\n /** If null, a random seed will be used. */\n seed?: null | number;\n /** The randomness of the responses, e.g. 0.1 deterministic, 1.5 creative, 0.8 balanced, 0 disables. */\n temperature?: number;\n /** Number of threads to use to evaluate tokens. */\n threads?: number;\n /** Trim whitespace from the end of the generated text Disabled by default. */\n trimWhitespaceSuffix?: boolean;\n /** Consider the n most likely tokens, where n is 1 to vocabulary size, 0 disables (uses full vocabulary). Note: only applies when `temperature` > 0. */\n topK?: number;\n /** Selects the smallest token set whose probability exceeds P, where P is between 0 - 1, 1 disables. Note: only applies when `temperature` > 0. */\n topP?: number;\n /** Force system to keep model in RAM. */\n useMlock?: boolean;\n /** Use mmap if possible. */\n useMmap?: boolean;\n /** Only load the vocabulary, no weights. */\n vocabOnly?: boolean;\n /** JSON schema to be used to format output. Also known as `grammar`. */\n jsonSchema?: object;\n /** GBNF string to be used to format output. Also known as `grammar`. */\n gbnf?: string;\n}\n\nexport async function createLlamaModel(\n inputs: LlamaBaseCppInputs,\n llama: Llama\n): Promise<LlamaModel> {\n const options: LlamaModelOptions = {\n gpuLayers: inputs?.gpuLayers,\n modelPath: inputs.modelPath,\n useMlock: inputs?.useMlock,\n useMmap: inputs?.useMmap,\n vocabOnly: inputs?.vocabOnly,\n };\n\n return llama.loadModel(options);\n}\n\nexport async function createLlamaContext(\n model: LlamaModel,\n inputs: LlamaBaseCppInputs\n): Promise<LlamaContext> {\n const options: LlamaContextOptions = {\n batchSize: inputs?.batchSize,\n contextSize: inputs?.contextSize,\n threads: inputs?.threads,\n };\n\n return model.createContext(options);\n}\n\nexport async function createLlamaEmbeddingContext(\n model: LlamaModel,\n inputs: LlamaBaseCppInputs\n): Promise<LlamaEmbeddingContext> {\n const options: LlamaEmbeddingContextOptions = {\n batchSize: inputs?.batchSize,\n contextSize: inputs?.contextSize,\n threads: inputs?.threads,\n };\n\n return model.createEmbeddingContext(options);\n}\n\nexport function createLlamaSession(context: LlamaContext): LlamaChatSession {\n return new LlamaChatSession({ contextSequence: context.getSequence() });\n}\n\nexport async function createLlamaJsonSchemaGrammar(\n schemaString: object | undefined,\n llama: Llama\n): Promise<LlamaJsonSchemaGrammar<GbnfJsonSchema> | undefined> {\n if (schemaString === undefined) {\n return undefined;\n }\n\n const schemaJSON = schemaString as GbnfJsonSchema;\n return await llama.createGrammarForJsonSchema(schemaJSON);\n}\n\nexport async function createCustomGrammar(\n filePath: string | undefined,\n llama: Llama\n): Promise<LlamaGrammar | undefined> {\n if (filePath === undefined) {\n return undefined;\n }\n\n return llama.createGrammar({\n grammar: filePath,\n });\n}\n"],"mappings":";;;;AA6DA,eAAsB,iBACpBA,QACAC,OACqB;CACrB,MAAMC,UAA6B;EACjC,WAAW,QAAQ;EACnB,WAAW,OAAO;EAClB,UAAU,QAAQ;EAClB,SAAS,QAAQ;EACjB,WAAW,QAAQ;CACpB;AAED,QAAO,MAAM,UAAU,QAAQ;AAChC;AAED,eAAsB,mBACpBC,OACAH,QACuB;CACvB,MAAMI,UAA+B;EACnC,WAAW,QAAQ;EACnB,aAAa,QAAQ;EACrB,SAAS,QAAQ;CAClB;AAED,QAAO,MAAM,cAAc,QAAQ;AACpC;AAED,eAAsB,4BACpBD,OACAH,QACgC;CAChC,MAAMK,UAAwC;EAC5C,WAAW,QAAQ;EACnB,aAAa,QAAQ;EACrB,SAAS,QAAQ;CAClB;AAED,QAAO,MAAM,uBAAuB,QAAQ;AAC7C;AAED,SAAgB,mBAAmBC,SAAyC;AAC1E,QAAO,IAAIC,gCAAiB,EAAE,iBAAiB,QAAQ,aAAa,CAAE;AACvE;AAED,eAAsB,6BACpBC,cACAP,OAC6D;AAC7D,KAAI,iBAAiB,OACnB,QAAO;CAGT,MAAM,aAAa;AACnB,QAAO,MAAM,MAAM,2BAA2B,WAAW;AAC1D;AAED,eAAsB,oBACpBQ,UACAR,OACmC;AACnC,KAAI,aAAa,OACf,QAAO;AAGT,QAAO,MAAM,cAAc,EACzB,SAAS,SACV,EAAC;AACH"}