@isdk/ai-tool-llm-llamacpp-pro/dist/index.js

llama.cpp LLM Provider

"use strict";var e,t=Object.defineProperty,o=Object.getOwnPropertyDescriptor,a=Object.getOwnPropertyNames,n=Object.prototype.hasOwnProperty,s={};((e,o)=>{for(var a in o)t(e,a,{get:o[a],enumerable:!0})})(s,{AILoadModelOptionsMap:()=>f,AIModelSettingsMap:()=>w,AIOptionsMap:()=>d,LlamaCppProvider:()=>g,LlamaCppProviderName:()=>v,LlamaLoadModelOptionsKeys:()=>p,LlamaVocabType:()=>c,LlmArch:()=>l,LlmSizeType:()=>i,ReverseAILoadModelOptions:()=>u,ReverseAIOptionsMap:()=>h,llamaCpp:()=>B,llamaCppToAIResult:()=>_,parseLlamaCppStream:()=>m}),module.exports=(e=s,((e,s,r,i)=>{if(s&&"object"==typeof s||"function"==typeof s)for(let l of a(s))n.call(e,l)||l===r||t(e,l,{get:()=>s[l],enumerable:!(i=o(s,l))||i.enumerable});return e})(t({},"__esModule",{value:!0}),e));var r=require("@isdk/ai-tool-llm"),i=(e=>(e[e.Unknown=0]="Unknown",e[e._17M=1]="_17M",e[e._22M=2]="_22M",e[e._33M=3]="_33M",e[e._109M=4]="_109M",e[e._137M=5]="_137M",e[e._335M=6]="_335M",e[e._0_5B=7]="_0_5B",e[e._1B=8]="_1B",e[e._2B=9]="_2B",e[e._3B=10]="_3B",e[e._4B=11]="_4B",e[e._7B=12]="_7B",e[e._8B=13]="_8B",e[e._13B=14]="_13B",e[e._14B=15]="_14B",e[e._15B=16]="_15B",e[e._20B=17]="_20B",e[e._30B=18]="_30B",e[e._34B=19]="_34B",e[e._35B=20]="_35B",e[e._40B=21]="_40B",e[e._65B=22]="_65B",e[e._70B=23]="_70B",e[e.Small=24]="Small",e[e.Medium=25]="Medium",e[e.Large=26]="Large",e[e.XL=27]="XL",e))(i||{}),l=(e=>(e[e.Llama=0]="Llama",e[e.Falcon=1]="Falcon",e[e.Baichuan=2]="Baichuan",e[e.Grok=3]="Grok",e[e.Gpt2=4]="Gpt2",e[e.Gptj=5]="Gptj",e[e.Gptneox=6]="Gptneox",e[e.Mpt=7]="Mpt",e[e.Starcoder=8]="Starcoder",e[e.Refact=9]="Refact",e[e.Bert=10]="Bert",e[e.Nomic_bert=11]="Nomic_bert",e[e.Jina_bert_v2=12]="Jina_bert_v2",e[e.Bloom=13]="Bloom",e[e.Stablelm=14]="Stablelm",e[e.Qwen=15]="Qwen",e[e.Qwen2=16]="Qwen2",e[e.Qwen2moe=17]="Qwen2moe",e[e.Phi2=18]="Phi2",e[e.Phi3=19]="Phi3",e[e.Plamo=20]="Plamo",e[e.Codeshell=21]="Codeshell",e[e.Orion=22]="Orion",e[e.Internlm2=23]="Internlm2",e[e.Minicpm=24]="Minicpm",e[e.Gemma=25]="Gemma",e[e.Starcoder2=26]="Starcoder2",e[e.Mamba=27]="Mamba",e[e.Xverse=28]="Xverse",e[e.Command_r=29]="Command_r",e[e.Dbrx=30]="Dbrx",e[e.Olmo=31]="Olmo",e[e.Arctic=32]="Arctic",e[e.Deepseek2=33]="Deepseek2",e[e.Unknown=34]="Unknown",e))(l||{}),c=(e=>(e[e.None=0]="None",e[e.Spm=1]="Spm",e[e.Bpe=2]="Bpe",e[e.Wpm=3]="Wpm",e))(c||{}),p=["mmproj","alias","gpu_layers","n_ctx","embedding","batch","parallel","threads","n_threads_batch","grp_attn_n","grp_attn_w","rope_scaling","rope_freq_base","rope_freq_scale","yarn_ext_factor","yarn_attn_factor","yarn_beta_fast","yarn_beta_slow","use_mmap","use_mlock","cont_batching","cache_type_k","cache_type_v","lora","lora_base"];function _(e){const t={content:e.content,options:e};if(e.stop&&(e.stopped_eos||e.stopped_word?t.finishReason="stop":e.stopped_limit?t.finishReason="length":t.finishReason=null),void 0!==e.stop&&(t.stop=e.stop),e.generation_settings){const t=e.generation_settings.max_tokens;e.generation_settings=(o=e.generation_settings,(0,r.mapApiOptions)(o,h)),t&&(e.generation_settings.max_tokens=t)}var o;return t}function m(e){return t=>{const o=JSON.parse(t);return e&&Object.assign(o,e),_(o)}}var d={max_tokens:"n_predict",stop_words:"stop",content_size:"n_ctx","response_format.type":"grammar_id","response_format.schema":"json_schema"},h=(0,r.flip)(d),f={content_size:"n_ctx",stop_words:"stop"},u=(0,r.flip)(f),w={n_params:"params_size",n_ctx_train:"content_size",model:"name"},y=require("@isdk/ai-tool"),M=require("@isdk/ai-tool-llm"),v="llamacppPro";function b(e){return(0,M.mapApiOptions)(e,d)}var g=class extends M.LLMProvider{constructor(){super(...arguments),this.rule=/.gguf$/,this.asyncFeatures=y.AsyncFeatures.MultiTask}async processModelOptions(e,t,o){const a="string"==typeof t;let n,s,r,i;if(t&&(a||Array.isArray(t))||(0,y.throwError)("missing prompt value","LlamaCppProvider"),(o=b(o)).stop&&"string"==typeof o.stop&&(o.stop=[o.stop]),!e&&this.model&&(e=this.model),e){e.endsWith(".gguf")&&(e=e.slice(0,-5)),e.startsWith(v+"://")&&(e=e.slice(v.length+3)),n=await this.getModelInfo();const t=n?.name;(t&&t.indexOf(e)<0||function(e,t){let o=!1;for(const a of e)if(o=t.includes(a),o)break;return o}(p,Object.keys(o)))&&(n=await this.getModelInfo(e),await this.loadModel({...o,model:e}))}else n=await this.getModelInfo();if(n||(0,y.throwError)("llamaCpp no current model",this.name),o.model=e,a)s=await this.getChatTemplate(n,o),s&&(r=O(s),s=s.prompt);else{const e=await this.formatPrompt(t,n,o);if(!e)throw new y.NotFoundError("ChatTemplate:"+n.name,this.name);t=e,o.chatTemplate&&(r=O(o.chatTemplate),s=o.chatTemplate.prompt)}if(s?.parameters){const e=this.getDefaultParameters(s,n.name);if(e){(e.eot_token||e.eos_token)&&(r||(r=[]),e.eot_token&&!r.includes(e.eot_token)&&r.push(e.eot_token),e.eos_token&&!r.includes(e.eos_token)&&r.push(e.eos_token));const t=b(e);Array.isArray(t.stop)&&Array.isArray(o.stop)&&(o.stop=[...t.stop,...o.stop].filter(((e,t,o)=>o.indexOf(e)===t))),o={...t,...o}}}return(n.eot_token||n.eos_token)&&(r||(r=[]),n.eot_token&&!r.includes(n.eot_token)&&r.push(n.eot_token),n.eos_token&&!r.includes(n.eos_token)&&r.push(n.eos_token)),r&&r.length&&(Array.isArray(o.stop)?o.stop=Array.from(new Set([...r,...o.stop])):o.stop=r),o.grammar_id&&o.json_schema&&delete o.grammar_id,o.chatTemplate&&(o.chatTemplate.prompt?._id&&(i={id:o.chatTemplate.prompt._id},o.chatTemplate.version&&(i.version=o.chatTemplate.version)),delete o.chatTemplate),i&&(o.chatTemplateId=i),o.value=t,o}func({model:e,value:t,options:o}){return this.runAsyncCancelableTask(o,(async a=>{const n=a.aborter.signal;delete(a={...a}).aborter,a=await this.processModelOptions(e,t,a);const s=this.apiUrl??"http://localhost:8080",r={"Content-Type":"application/json"},i=o.stream;i&&(r.Connection="keep-alive",r.Accept="text/event-stream");const l=a.chatTemplateId;l&&delete a.chatTemplateId,t=a.value,delete a.value;const c={...a,prompt:t},p=await fetch((0,M.joinUrl)(s,"/completion"),{method:"POST",headers:r,body:JSON.stringify(c),signal:n});let d;if(p.ok)if(i)d=(0,y.AIStream)(p,m({chatTemplateId:l}));else{const e=await p.text();d=_(JSON.parse(e)),d.chatTemplateId=l}else{const e=this.emitError?this.emitError(p,s,c):void 0;if(e)d=e;else{if(!i)throw new y.CommonError(await p.text(),"LlamaCppProvider");d=(0,y.getResponseErrorReadableStream)(p.body)}}return d}))}async loadModel(e){const t=this.apiUrl??"http://localhost:8080";var o;o=e,e=(0,M.mapApiOptions)(o,f);const a=await fetch((0,M.joinUrl)(t,"/v1/models"),{method:"POST",body:JSON.stringify(e),headers:{Connection:"keep-alive","Content-Type":"application/json",Accept:"application/json"}}),n=await a.json(),s=n.error;if(s)throw s.code?new y.CommonError(s.message,"loadModel",s.code):new y.CommonError(s,"loadModel");const r=function(e){return(0,M.mapApiOptions)(e,u)}(n);return r}async getModelInfo(e){const t=this.apiUrl??"http://localhost:8080";e=e?"?model="+e:"";const o=await fetch((0,M.joinUrl)(t,"/model/meta"+e)),a=await o.json(),n=a.error;if(n)throw n.code?new y.CommonError(n.message,"getModelInfo",n.code):new y.CommonError(n,"getModelInfo");const s=(0,M.mapApiOptions)(a,w);return s.params_size&&!s.scale&&(s.scale=(0,y.paramsSizeToScaleStr)(s.params_size)),s.name+=".gguf",s}async listModels(){const e=this.apiUrl??"http://localhost:8080",t=await fetch((0,M.joinUrl)(e,"/v1/models")),o=await t.json(),a=o.error;if(a)throw a.code?new y.CommonError(a.message,"listModels",a.code):new y.CommonError(a,"listModels");const n=o.data?.map((e=>e.name));return n}};(0,y.makeToolFuncCancelable)(g);var B=new g(v);function O(e){let t=[],o=e.version;Array.isArray(o)&&(o=o[0]);const a=e.prompt,n=a.version;if(o&&"@"!==o&&n?.[o]?.prompt){const e=n[o].prompt;e.eot_token&&t.push(e.eot_token),e.eos_token&&t.push(e.eos_token)}else if(a.prompt){const e=a.prompt;e.eot_token&&t.push(e.eot_token),e.eos_token&&t.push(e.eos_token)}return t.length?t:void 0}