@isdk/ai-tool-llm-llamacpp/dist/index.mjs

llama.cpp LLM Provider

import{flip as t,mapApiOptions as e}from"@isdk/ai-tool-llm";var o=(t=>(t[t.Unknown=0]="Unknown",t[t._14M=1]="_14M",t[t._17M=2]="_17M",t[t._22M=3]="_22M",t[t._33M=4]="_33M",t[t._60M=5]="_60M",t[t._70M=6]="_70M",t[t._80M=7]="_80M",t[t._109M=8]="_109M",t[t._137M=9]="_137M",t[t._160M=10]="_160M",t[t._220M=11]="_220M",t[t._250M=12]="_250M",t[t._270M=13]="_270M",t[t._335M=14]="_335M",t[t._410M=15]="_410M",t[t._450M=16]="_450M",t[t._770M=17]="_770M",t[t._780M=18]="_780M",t[t._0_5B=19]="_0_5B",t[t._1B=20]="_1B",t[t._1_3B=21]="_1_3B",t[t._1_4B=22]="_1_4B",t[t._2B=23]="_2B",t[t._2_8B=24]="_2_8B",t[t._3B=25]="_3B",t[t._4B=26]="_4B",t[t._6B=27]="_6B",t[t._6_9B=28]="_6_9B",t[t._7B=29]="_7B",t[t._8B=30]="_8B",t[t._9B=31]="_9B",t[t._11B=32]="_11B",t[t._12B=33]="_12B",t[t._13B=34]="_13B",t[t._14B=35]="_14B",t[t._15B=36]="_15B",t[t._16B=37]="_16B",t[t._20B=38]="_20B",t[t._30B=39]="_30B",t[t._34B=40]="_34B",t[t._35B=41]="_35B",t[t._40B=42]="_40B",t[t._65B=43]="_65B",t[t._70B=44]="_70B",t[t._236B=45]="_236B",t[t._314B=46]="_314B",t[t._SMALL=47]="_SMALL",t[t._MEDIUM=48]="_MEDIUM",t[t._LARGE=49]="_LARGE",t[t._XL=50]="_XL",t[t._A2_7B=51]="_A2_7B",t[t._8x7B=52]="_8x7B",t[t._8x22B=53]="_8x22B",t[t._16x12B=54]="_16x12B",t[t._10B_128x3_66B=55]="_10B_128x3_66B",t[t._57B_A14B=56]="_57B_A14B",t[t._27B=57]="_27B",t))(o||{}),a=(t=>(t[t.Llama=0]="Llama",t[t.Falcon=1]="Falcon",t[t.Grok=2]="Grok",t[t.Gpt2=3]="Gpt2",t[t.Gptj=4]="Gptj",t[t.Gptneox=5]="Gptneox",t[t.Mpt=6]="Mpt",t[t.Baichuan=7]="Baichuan",t[t.Starcoder=8]="Starcoder",t[t.Refact=9]="Refact",t[t.Bert=10]="Bert",t[t.Nomic_bert=11]="Nomic_bert",t[t.Jina_bert_v2=12]="Jina_bert_v2",t[t.Bloom=13]="Bloom",t[t.Stablelm=14]="Stablelm",t[t.Qwen=15]="Qwen",t[t.Qwen2=16]="Qwen2",t[t.Qwen2moe=17]="Qwen2moe",t[t.Phi2=18]="Phi2",t[t.Phi3=19]="Phi3",t[t.Plamo=20]="Plamo",t[t.Codeshell=21]="Codeshell",t[t.Orion=22]="Orion",t[t.Internlm2=23]="Internlm2",t[t.Minicpm=24]="Minicpm",t[t.Gemma=25]="Gemma",t[t.Gemma2=26]="Gemma2",t[t.Starcoder2=27]="Starcoder2",t[t.Mamba=28]="Mamba",t[t.Xverse=29]="Xverse",t[t.Command_r=30]="Command_r",t[t.Dbrx=31]="Dbrx",t[t.Olmo=32]="Olmo",t[t.Openelm=33]="Openelm",t[t.Arctic=34]="Arctic",t[t.Deepseek2=35]="Deepseek2",t[t.Chatglm=36]="Chatglm",t[t.Bitnet=37]="Bitnet",t[t.T5=38]="T5",t[t.T5encoder=39]="T5encoder",t[t.Jais=40]="Jais",t[t.Nemotron=41]="Nemotron",t[t.Exaone=42]="Exaone",t[t.Unknown=43]="Unknown",t))(a||{}),n=(t=>(t[t.NONE=0]="NONE",t[t.SPM=1]="SPM",t[t.BPE=2]="BPE",t[t.WPM=3]="WPM",t[t.UGM=4]="UGM",t))(n||{}),r=(t=>(t[t.SPLIT_MODE_NONE=0]="SPLIT_MODE_NONE",t[t.SPLIT_MODE_LAYER=1]="SPLIT_MODE_LAYER",t[t.SPLIT_MODE_ROW=2]="SPLIT_MODE_ROW",t))(r||{}),i=["threads","threads_batch","threads_draft","threads_batch_draft","escape","path_prompt_cache","prompt_cache_all","prompt_cache_ro","predict","ctx_size","grp_attn_n","grp_attn_w","rope_freq_base","rope_freq_scale","rope_scaling","yarn_orig_ctx","yarn_ext_factor","yarn_attn_factor","yarn_beta_fast","yarn_beta_slow","pooling","attention_type","defrag_thold","samplers","samplers_sequence","batch_size","ubatch_size","keep","draft","chunks","parallel","sequences","p_split","model","model_draft","model_alias","model_url","hf_token","hf_repo","hf_file","lora","lora_init_without_apply","mmproj","image","special","embedding","embd_normalize","embd_output_format","embd_sep","conversation","infill","dump_kv_cache","no_kv_offload","cache_type_k","cache_type_v","multiline_input","cont_batching","flash_attn","use_mlock","gpu_layers","gpu_layers_draft","main_gpu","split_mode","tensor_split","rpc_servers","use_mmap","numa","verbosity","verbose_prompt","reverse_prompt","display_prompt","logdir","lookup_cache_static","lookup_cache_dynamic","logits_file","logits_all","check_tensors"];function s(t){const o={content:t.content,options:t};if(t.stop&&(t.stopped_eos||t.stopped_word||"eos"===t.stop_type||"word"===t.stop_type?o.finishReason="stop":t.stopped_limit||"limit"===t.stop_type?o.finishReason="length":o.finishReason=null),void 0!==t.stop&&(o.stop=t.stop),t.generation_settings){const o=t.generation_settings.max_tokens;t.generation_settings=(a=t.generation_settings,e(a,c)),t.generation_settings.model_id=t.model,o&&(t.generation_settings.max_tokens=o)}var a;return o}function _(t){return e=>{const o=JSON.parse(e);return s({...t,...o})}}var l={max_tokens:"n_predict",stop_words:"stop",content_size:"n_ctx","response_format.type":"grammar_id","response_format.schema":"json_schema"},c=t(l),p={content_size:"n_ctx",stop_words:"stop"},m=t(p),h={model:"name",n_ctx:"content_size"};import d from"path";import{AIStream as f,AsyncFeatures as u,CommonError as w,ErrorCode as B,NotFoundError as y,getResponseErrorReadableStream as M,makeToolFuncCancelable as g,throwError as k}from"@isdk/ai-tool";import{LLMProvider as v,joinUrl as b,mapApiOptions as O}from"@isdk/ai-tool-llm";var x="llamacpp";function S(t){return O(t,l)}var z=class extends v{constructor(){super(...arguments),this.rule=/.gguf$/}async tryGetModelInfo(t,e){let o;if(!t&&this.model&&(t=this.model),t){if(t.endsWith(".gguf")&&(t=t.slice(0,-5)),t.startsWith(this.name+"://")&&(t=t.slice(this.name.length+3)),o=await this.getModelInfo(void 0,e),"."!==t){const a=o?.name;(!a||a.indexOf(t)<0||function(t,e){let o=!1;for(const a of t)if(o=e.includes(a),o)break;return o}(i,Object.keys(e)))&&(await this.loadModel({...e,model:t,currentModel:a}),o=await this.getModelInfo(t,e))}}else o=await this.getModelInfo();return o}async processModelOptions(t,e,o){const a="string"==typeof e;e&&(a||Array.isArray(e))||k("missing prompt value","LlamaCppProvider"),(o=S(o)).stop&&"string"==typeof o.stop&&(o.stop=[o.stop]);let n,r,i,s=await this.tryGetModelInfo(t,o);if(s||k("llamaCpp no current model",this.name),(!t||"."===t&&s.name)&&(t=s.name),o.model=t,a)n=await this.getChatTemplate(s,o),n&&(r=P(n),n=n.prompt);else{const t=await this.formatPrompt(e,s,o);if(!t)throw new y("ChatTemplate:"+s.name,this.name);e=t,o.chatTemplate&&(r=P(o.chatTemplate),n=o.chatTemplate.prompt)}if(n?.parameters){const t=this.getDefaultParameters(n,s.name);if(t){(t.eot_token||t.eos_token)&&(r||(r=[]),t.eot_token&&!r.includes(t.eot_token)&&r.push(t.eot_token),t.eos_token&&!r.includes(t.eos_token)&&r.push(t.eos_token));const e=S(t);Array.isArray(e.stop)&&Array.isArray(o.stop)&&(o.stop=[...e.stop,...o.stop].filter(((t,e,o)=>o.indexOf(t)===e))),o={...e,...o}}}return(s.eot_token||s.eos_token)&&(r||(r=[]),s.eot_token&&!r.includes(s.eot_token)&&r.push(s.eot_token),s.eos_token&&!r.includes(s.eos_token)&&r.push(s.eos_token)),r&&r.length&&(Array.isArray(o.stop)?o.stop=Array.from(new Set([...r,...o.stop])):o.stop=r),o.grammar_id&&o.json_schema&&delete o.grammar_id,o.chatTemplate&&(o.chatTemplate.prompt?._id&&(i={id:o.chatTemplate.prompt._id},o.chatTemplate.version&&(i.version=o.chatTemplate.version),o.chatTemplate.prompt.shouldThink&&(i.shouldThink=o.chatTemplate.prompt.shouldThink)),delete o.chatTemplate),i&&(o.chatTemplateId=i),o.value=e,delete o.SystemTemplate,o}async fetch(t){const e=t.aborter.signal;delete t.aborter;const o=this.apiUrl??"http://localhost:8080",a={"Content-Type":"application/json"},n=t.stream;n&&(a.Connection="keep-alive",a.Accept="text/event-stream");const r=t.chatTemplateId;r&&delete t.chatTemplateId;const i=t.value;delete t.value;const l={...t,prompt:i},c=await fetch(b(o,"/completion"),{method:"POST",headers:a,body:JSON.stringify(l),signal:e});let p;if(c.ok)if(n)p=f(c,_({chatTemplateId:r}));else{const t=await c.text(),e=JSON.parse(t);e.chatTemplateId=r,p=s(e)}else{const t=this.emitError?this.emitError(c,o,l):void 0;if(t)p=t;else{if(!n)throw new w(await c.text(),"LlamaCppProvider");p=M(c.body)}}return p}func({model:t,value:e,options:o}){return this.runAsyncCancelableTask(o,(async o=>{o={...o},o=await this.processModelOptions(t,e,o);return await this.fetch(o)}))}async loadModel(t){const e=t.currentModel;throw new w(`the model(${t.model}) is not the current llamaCpp running model(${e})`,this.name,B.InvalidArgument)}async getModelInfo(t,e){const o=this.apiUrl??"http://localhost:8080",a=await fetch(b(o,"/props")),n=await a.json(),r=n.error;if(r)throw r.code?new w(r.message,"getModelInfo",r.code):new w(r,"getModelInfo");const i=n.default_generation_settings;delete n.default_generation_settings;const s=O({...n,...i},h);return s.name=d.basename(s.name||s.model_path),s}async _tokenize(t,e={}){const{add_special:o,with_pieces:a}=e,n=this.apiUrl??"http://localhost:8080",r=await fetch(b(n,"/tokenize"),{method:"POST",headers:{"Content-Type":"application/json"},body:JSON.stringify({content:t,add_special:o,with_pieces:a})});return await r.json()}async tokenize(t,e={}){const o="string"==typeof t;if(t&&(o||Array.isArray(t))||k("missing content to tokenize","LlamaCppProvider"),e=S(e),!o){const o=await this.tryGetModelInfo(e.model,e),a=await this.formatPrompt(t,o,e);if(!a)throw new y("ChatTemplate:"+o.name,"llamaCpp.tokenize");t=a}const a=await this._tokenize(t,e),n=a.error;if(n)throw n.code?new w(n.message,"tokenize",n.code):new w(n,"tokenize");return a.tokens}async countTokens(t,e){return(await this.tokenize(t,e)).length}};g(z,{asyncFeatures:u.MultiTask});var A=new z(x);function P(t){let e=[],o=t.version;Array.isArray(o)&&(o=o[0]);const a=t.prompt,n=a.version;let r=n?.[o]?.prompt;return o&&"@"!==o&&r?(r.eot_token&&e.push(r.eot_token),r.eos_token&&e.push(r.eos_token)):a.prompt&&(r=a.prompt,r.eot_token&&e.push(r.eot_token),r.eos_token&&e.push(r.eos_token)),e.length?e:void 0}export{p as AILoadModelOptionsMap,h as AIModelSettingsMap,l as AIOptionsMap,z as LlamaCppProvider,x as LlamaCppProviderName,i as LlamaLoadModelOptionsKeys,r as LlamaSplitMode,n as LlamaVocabType,a as LlmArch,o as LlmSizeType,m as ReverseAILoadModelOptions,c as ReverseAIOptionsMap,A as llamaCpp,s as llamaCppToAIResult,_ as parseLlamaCppStream};