@isdk/ai-tool-llm-llamacpp/dist/index.js

llama.cpp LLM Provider

"use strict";var e,t=Object.create,a=Object.defineProperty,o=Object.getOwnPropertyDescriptor,n=Object.getOwnPropertyNames,r=Object.getPrototypeOf,i=Object.prototype.hasOwnProperty,s=(e,t,r,s)=>{if(t&&"object"==typeof t||"function"==typeof t)for(let _ of n(t))i.call(e,_)||_===r||a(e,_,{get:()=>t[_],enumerable:!(s=o(t,_))||s.enumerable});return e},_={};((e,t)=>{for(var o in t)a(e,o,{get:t[o],enumerable:!0})})(_,{AILoadModelOptionsMap:()=>y,AIModelSettingsMap:()=>g,AIOptionsMap:()=>w,LlamaCppProvider:()=>A,LlamaCppProviderName:()=>O,LlamaLoadModelOptionsKeys:()=>h,LlamaSplitMode:()=>d,LlamaVocabType:()=>m,LlmArch:()=>p,LlmSizeType:()=>c,ReverseAILoadModelOptions:()=>B,ReverseAIOptionsMap:()=>M,llamaCpp:()=>S,llamaCppToAIResult:()=>u,parseLlamaCppStream:()=>f}),module.exports=(e=_,s(a({},"__esModule",{value:!0}),e));var l=require("@isdk/ai-tool-llm"),c=(e=>(e[e.Unknown=0]="Unknown",e[e._14M=1]="_14M",e[e._17M=2]="_17M",e[e._22M=3]="_22M",e[e._33M=4]="_33M",e[e._60M=5]="_60M",e[e._70M=6]="_70M",e[e._80M=7]="_80M",e[e._109M=8]="_109M",e[e._137M=9]="_137M",e[e._160M=10]="_160M",e[e._220M=11]="_220M",e[e._250M=12]="_250M",e[e._270M=13]="_270M",e[e._335M=14]="_335M",e[e._410M=15]="_410M",e[e._450M=16]="_450M",e[e._770M=17]="_770M",e[e._780M=18]="_780M",e[e._0_5B=19]="_0_5B",e[e._1B=20]="_1B",e[e._1_3B=21]="_1_3B",e[e._1_4B=22]="_1_4B",e[e._2B=23]="_2B",e[e._2_8B=24]="_2_8B",e[e._3B=25]="_3B",e[e._4B=26]="_4B",e[e._6B=27]="_6B",e[e._6_9B=28]="_6_9B",e[e._7B=29]="_7B",e[e._8B=30]="_8B",e[e._9B=31]="_9B",e[e._11B=32]="_11B",e[e._12B=33]="_12B",e[e._13B=34]="_13B",e[e._14B=35]="_14B",e[e._15B=36]="_15B",e[e._16B=37]="_16B",e[e._20B=38]="_20B",e[e._30B=39]="_30B",e[e._34B=40]="_34B",e[e._35B=41]="_35B",e[e._40B=42]="_40B",e[e._65B=43]="_65B",e[e._70B=44]="_70B",e[e._236B=45]="_236B",e[e._314B=46]="_314B",e[e._SMALL=47]="_SMALL",e[e._MEDIUM=48]="_MEDIUM",e[e._LARGE=49]="_LARGE",e[e._XL=50]="_XL",e[e._A2_7B=51]="_A2_7B",e[e._8x7B=52]="_8x7B",e[e._8x22B=53]="_8x22B",e[e._16x12B=54]="_16x12B",e[e._10B_128x3_66B=55]="_10B_128x3_66B",e[e._57B_A14B=56]="_57B_A14B",e[e._27B=57]="_27B",e))(c||{}),p=(e=>(e[e.Llama=0]="Llama",e[e.Falcon=1]="Falcon",e[e.Grok=2]="Grok",e[e.Gpt2=3]="Gpt2",e[e.Gptj=4]="Gptj",e[e.Gptneox=5]="Gptneox",e[e.Mpt=6]="Mpt",e[e.Baichuan=7]="Baichuan",e[e.Starcoder=8]="Starcoder",e[e.Refact=9]="Refact",e[e.Bert=10]="Bert",e[e.Nomic_bert=11]="Nomic_bert",e[e.Jina_bert_v2=12]="Jina_bert_v2",e[e.Bloom=13]="Bloom",e[e.Stablelm=14]="Stablelm",e[e.Qwen=15]="Qwen",e[e.Qwen2=16]="Qwen2",e[e.Qwen2moe=17]="Qwen2moe",e[e.Phi2=18]="Phi2",e[e.Phi3=19]="Phi3",e[e.Plamo=20]="Plamo",e[e.Codeshell=21]="Codeshell",e[e.Orion=22]="Orion",e[e.Internlm2=23]="Internlm2",e[e.Minicpm=24]="Minicpm",e[e.Gemma=25]="Gemma",e[e.Gemma2=26]="Gemma2",e[e.Starcoder2=27]="Starcoder2",e[e.Mamba=28]="Mamba",e[e.Xverse=29]="Xverse",e[e.Command_r=30]="Command_r",e[e.Dbrx=31]="Dbrx",e[e.Olmo=32]="Olmo",e[e.Openelm=33]="Openelm",e[e.Arctic=34]="Arctic",e[e.Deepseek2=35]="Deepseek2",e[e.Chatglm=36]="Chatglm",e[e.Bitnet=37]="Bitnet",e[e.T5=38]="T5",e[e.T5encoder=39]="T5encoder",e[e.Jais=40]="Jais",e[e.Nemotron=41]="Nemotron",e[e.Exaone=42]="Exaone",e[e.Unknown=43]="Unknown",e))(p||{}),m=(e=>(e[e.NONE=0]="NONE",e[e.SPM=1]="SPM",e[e.BPE=2]="BPE",e[e.WPM=3]="WPM",e[e.UGM=4]="UGM",e))(m||{}),d=(e=>(e[e.SPLIT_MODE_NONE=0]="SPLIT_MODE_NONE",e[e.SPLIT_MODE_LAYER=1]="SPLIT_MODE_LAYER",e[e.SPLIT_MODE_ROW=2]="SPLIT_MODE_ROW",e))(d||{}),h=["threads","threads_batch","threads_draft","threads_batch_draft","escape","path_prompt_cache","prompt_cache_all","prompt_cache_ro","predict","ctx_size","grp_attn_n","grp_attn_w","rope_freq_base","rope_freq_scale","rope_scaling","yarn_orig_ctx","yarn_ext_factor","yarn_attn_factor","yarn_beta_fast","yarn_beta_slow","pooling","attention_type","defrag_thold","samplers","samplers_sequence","batch_size","ubatch_size","keep","draft","chunks","parallel","sequences","p_split","model","model_draft","model_alias","model_url","hf_token","hf_repo","hf_file","lora","lora_init_without_apply","mmproj","image","special","embedding","embd_normalize","embd_output_format","embd_sep","conversation","infill","dump_kv_cache","no_kv_offload","cache_type_k","cache_type_v","multiline_input","cont_batching","flash_attn","use_mlock","gpu_layers","gpu_layers_draft","main_gpu","split_mode","tensor_split","rpc_servers","use_mmap","numa","verbosity","verbose_prompt","reverse_prompt","display_prompt","logdir","lookup_cache_static","lookup_cache_dynamic","logits_file","logits_all","check_tensors"];function u(e){const t={content:e.content,options:e};if(e.stop&&(e.stopped_eos||e.stopped_word||"eos"===e.stop_type||"word"===e.stop_type?t.finishReason="stop":e.stopped_limit||"limit"===e.stop_type?t.finishReason="length":t.finishReason=null),void 0!==e.stop&&(t.stop=e.stop),e.generation_settings){const t=e.generation_settings.max_tokens;e.generation_settings=(a=e.generation_settings,(0,l.mapApiOptions)(a,M)),e.generation_settings.model_id=e.model,t&&(e.generation_settings.max_tokens=t)}var a;return t}function f(e){return t=>{const a=JSON.parse(t);return u({...e,...a})}}var w={max_tokens:"n_predict",stop_words:"stop",content_size:"n_ctx","response_format.type":"grammar_id","response_format.schema":"json_schema"},M=(0,l.flip)(w),y={content_size:"n_ctx",stop_words:"stop"},B=(0,l.flip)(y),g={model:"name",n_ctx:"content_size"},b=((e,o,n)=>(n=null!=e?t(r(e)):{},s(!o&&e&&e.__esModule?n:a(n,"default",{value:e,enumerable:!0}),e)))(require("path")),v=require("@isdk/ai-tool"),k=require("@isdk/ai-tool-llm"),O="llamacpp";function L(e){return(0,k.mapApiOptions)(e,w)}var A=class extends k.LLMProvider{constructor(){super(...arguments),this.rule=/.gguf$/}async tryGetModelInfo(e,t){let a;if(!e&&this.model&&(e=this.model),e){if(e.endsWith(".gguf")&&(e=e.slice(0,-5)),e.startsWith(this.name+"://")&&(e=e.slice(this.name.length+3)),a=await this.getModelInfo(void 0,t),"."!==e){const o=a?.name;(!o||o.indexOf(e)<0||function(e,t){let a=!1;for(const o of e)if(a=t.includes(o),a)break;return a}(h,Object.keys(t)))&&(await this.loadModel({...t,model:e,currentModel:o}),a=await this.getModelInfo(e,t))}}else a=await this.getModelInfo();return a}async processModelOptions(e,t,a){const o="string"==typeof t;t&&(o||Array.isArray(t))||(0,v.throwError)("missing prompt value","LlamaCppProvider"),(a=L(a)).stop&&"string"==typeof a.stop&&(a.stop=[a.stop]);let n,r,i,s=await this.tryGetModelInfo(e,a);if(s||(0,v.throwError)("llamaCpp no current model",this.name),(!e||"."===e&&s.name)&&(e=s.name),a.model=e,o)n=await this.getChatTemplate(s,a),n&&(r=C(n),n=n.prompt);else{const e=await this.formatPrompt(t,s,a);if(!e)throw new v.NotFoundError("ChatTemplate:"+s.name,this.name);t=e,a.chatTemplate&&(r=C(a.chatTemplate),n=a.chatTemplate.prompt)}if(n?.parameters){const e=this.getDefaultParameters(n,s.name);if(e){(e.eot_token||e.eos_token)&&(r||(r=[]),e.eot_token&&!r.includes(e.eot_token)&&r.push(e.eot_token),e.eos_token&&!r.includes(e.eos_token)&&r.push(e.eos_token));const t=L(e);Array.isArray(t.stop)&&Array.isArray(a.stop)&&(a.stop=[...t.stop,...a.stop].filter(((e,t,a)=>a.indexOf(e)===t))),a={...t,...a}}}return(s.eot_token||s.eos_token)&&(r||(r=[]),s.eot_token&&!r.includes(s.eot_token)&&r.push(s.eot_token),s.eos_token&&!r.includes(s.eos_token)&&r.push(s.eos_token)),r&&r.length&&(Array.isArray(a.stop)?a.stop=Array.from(new Set([...r,...a.stop])):a.stop=r),a.grammar_id&&a.json_schema&&delete a.grammar_id,a.chatTemplate&&(a.chatTemplate.prompt?._id&&(i={id:a.chatTemplate.prompt._id},a.chatTemplate.version&&(i.version=a.chatTemplate.version),a.chatTemplate.prompt.shouldThink&&(i.shouldThink=a.chatTemplate.prompt.shouldThink)),delete a.chatTemplate),i&&(a.chatTemplateId=i),a.value=t,delete a.SystemTemplate,a}async fetch(e){const t=e.aborter.signal;delete e.aborter;const a=this.apiUrl??"http://localhost:8080",o={"Content-Type":"application/json"},n=e.stream;n&&(o.Connection="keep-alive",o.Accept="text/event-stream");const r=e.chatTemplateId;r&&delete e.chatTemplateId;const i=e.value;delete e.value;const s={...e,prompt:i},_=await fetch((0,k.joinUrl)(a,"/completion"),{method:"POST",headers:o,body:JSON.stringify(s),signal:t});let l;if(_.ok)if(n)l=(0,v.AIStream)(_,f({chatTemplateId:r}));else{const e=await _.text(),t=JSON.parse(e);t.chatTemplateId=r,l=u(t)}else{const e=this.emitError?this.emitError(_,a,s):void 0;if(e)l=e;else{if(!n)throw new v.CommonError(await _.text(),"LlamaCppProvider");l=(0,v.getResponseErrorReadableStream)(_.body)}}return l}func({model:e,value:t,options:a}){return this.runAsyncCancelableTask(a,(async a=>{a={...a},a=await this.processModelOptions(e,t,a);return await this.fetch(a)}))}async loadModel(e){const t=e.currentModel;throw new v.CommonError(`the model(${e.model}) is not the current llamaCpp running model(${t})`,this.name,v.ErrorCode.InvalidArgument)}async getModelInfo(e,t){const a=this.apiUrl??"http://localhost:8080",o=await fetch((0,k.joinUrl)(a,"/props")),n=await o.json(),r=n.error;if(r)throw r.code?new v.CommonError(r.message,"getModelInfo",r.code):new v.CommonError(r,"getModelInfo");const i=n.default_generation_settings;delete n.default_generation_settings;const s=(0,k.mapApiOptions)({...n,...i},g);return s.name=b.default.basename(s.name||s.model_path),s}async _tokenize(e,t={}){const{add_special:a,with_pieces:o}=t,n=this.apiUrl??"http://localhost:8080",r=await fetch((0,k.joinUrl)(n,"/tokenize"),{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({content:e,add_special:a,with_pieces:o})});return await r.json()}async tokenize(e,t={}){const a="string"==typeof e;if(e&&(a||Array.isArray(e))||(0,v.throwError)("missing content to tokenize","LlamaCppProvider"),t=L(t),!a){const a=await this.tryGetModelInfo(t.model,t),o=await this.formatPrompt(e,a,t);if(!o)throw new v.NotFoundError("ChatTemplate:"+a.name,"llamaCpp.tokenize");e=o}const o=await this._tokenize(e,t),n=o.error;if(n)throw n.code?new v.CommonError(n.message,"tokenize",n.code):new v.CommonError(n,"tokenize");return o.tokens}async countTokens(e,t){return(await this.tokenize(e,t)).length}};(0,v.makeToolFuncCancelable)(A,{asyncFeatures:v.AsyncFeatures.MultiTask});var S=new A(O);function C(e){let t=[],a=e.version;Array.isArray(a)&&(a=a[0]);const o=e.prompt,n=o.version;let r=n?.[a]?.prompt;return a&&"@"!==a&&r?(r.eot_token&&t.push(r.eot_token),r.eos_token&&t.push(r.eos_token)):o.prompt&&(r=o.prompt,r.eot_token&&t.push(r.eot_token),r.eos_token&&t.push(r.eos_token)),t.length?t:void 0}