@isdk/ai-tool-llm-llamacpp-pro/dist/index.mjs

llama.cpp LLM Provider

import{flip as t,mapApiOptions as e}from"@isdk/ai-tool-llm";var o=(t=>(t[t.Unknown=0]="Unknown",t[t._17M=1]="_17M",t[t._22M=2]="_22M",t[t._33M=3]="_33M",t[t._109M=4]="_109M",t[t._137M=5]="_137M",t[t._335M=6]="_335M",t[t._0_5B=7]="_0_5B",t[t._1B=8]="_1B",t[t._2B=9]="_2B",t[t._3B=10]="_3B",t[t._4B=11]="_4B",t[t._7B=12]="_7B",t[t._8B=13]="_8B",t[t._13B=14]="_13B",t[t._14B=15]="_14B",t[t._15B=16]="_15B",t[t._20B=17]="_20B",t[t._30B=18]="_30B",t[t._34B=19]="_34B",t[t._35B=20]="_35B",t[t._40B=21]="_40B",t[t._65B=22]="_65B",t[t._70B=23]="_70B",t[t.Small=24]="Small",t[t.Medium=25]="Medium",t[t.Large=26]="Large",t[t.XL=27]="XL",t))(o||{}),a=(t=>(t[t.Llama=0]="Llama",t[t.Falcon=1]="Falcon",t[t.Baichuan=2]="Baichuan",t[t.Grok=3]="Grok",t[t.Gpt2=4]="Gpt2",t[t.Gptj=5]="Gptj",t[t.Gptneox=6]="Gptneox",t[t.Mpt=7]="Mpt",t[t.Starcoder=8]="Starcoder",t[t.Refact=9]="Refact",t[t.Bert=10]="Bert",t[t.Nomic_bert=11]="Nomic_bert",t[t.Jina_bert_v2=12]="Jina_bert_v2",t[t.Bloom=13]="Bloom",t[t.Stablelm=14]="Stablelm",t[t.Qwen=15]="Qwen",t[t.Qwen2=16]="Qwen2",t[t.Qwen2moe=17]="Qwen2moe",t[t.Phi2=18]="Phi2",t[t.Phi3=19]="Phi3",t[t.Plamo=20]="Plamo",t[t.Codeshell=21]="Codeshell",t[t.Orion=22]="Orion",t[t.Internlm2=23]="Internlm2",t[t.Minicpm=24]="Minicpm",t[t.Gemma=25]="Gemma",t[t.Starcoder2=26]="Starcoder2",t[t.Mamba=27]="Mamba",t[t.Xverse=28]="Xverse",t[t.Command_r=29]="Command_r",t[t.Dbrx=30]="Dbrx",t[t.Olmo=31]="Olmo",t[t.Arctic=32]="Arctic",t[t.Deepseek2=33]="Deepseek2",t[t.Unknown=34]="Unknown",t))(a||{}),n=(t=>(t[t.None=0]="None",t[t.Spm=1]="Spm",t[t.Bpe=2]="Bpe",t[t.Wpm=3]="Wpm",t))(n||{}),s=["mmproj","alias","gpu_layers","n_ctx","embedding","batch","parallel","threads","n_threads_batch","grp_attn_n","grp_attn_w","rope_scaling","rope_freq_base","rope_freq_scale","yarn_ext_factor","yarn_attn_factor","yarn_beta_fast","yarn_beta_slow","use_mmap","use_mlock","cont_batching","cache_type_k","cache_type_v","lora","lora_base"];function r(t){const o={content:t.content,options:t};if(t.stop&&(t.stopped_eos||t.stopped_word?o.finishReason="stop":t.stopped_limit?o.finishReason="length":o.finishReason=null),void 0!==t.stop&&(o.stop=t.stop),t.generation_settings){const o=t.generation_settings.max_tokens;t.generation_settings=(a=t.generation_settings,e(a,c)),o&&(t.generation_settings.max_tokens=o)}var a;return o}function i(t){return e=>{const o=JSON.parse(e);return t&&Object.assign(o,t),r(o)}}var l={max_tokens:"n_predict",stop_words:"stop",content_size:"n_ctx","response_format.type":"grammar_id","response_format.schema":"json_schema"},c=t(l),p={content_size:"n_ctx",stop_words:"stop"},_=t(p),m={n_params:"params_size",n_ctx_train:"content_size",model:"name"};import{AIStream as h,AsyncFeatures as d,CommonError as f,NotFoundError as w,getResponseErrorReadableStream as u,makeToolFuncCancelable as y,paramsSizeToScaleStr as g,throwError as v}from"@isdk/ai-tool";import{LLMProvider as B,joinUrl as M,mapApiOptions as b}from"@isdk/ai-tool-llm";var k="llamacppPro";function O(t){return b(t,l)}var S=class extends B{constructor(){super(...arguments),this.rule=/.gguf$/,this.asyncFeatures=d.MultiTask}async processModelOptions(t,e,o){const a="string"==typeof e;let n,r,i,l;if(e&&(a||Array.isArray(e))||v("missing prompt value","LlamaCppProvider"),(o=O(o)).stop&&"string"==typeof o.stop&&(o.stop=[o.stop]),!t&&this.model&&(t=this.model),t){t.endsWith(".gguf")&&(t=t.slice(0,-5)),t.startsWith(k+"://")&&(t=t.slice(14)),n=await this.getModelInfo();const e=n?.name;(e&&e.indexOf(t)<0||function(t,e){let o=!1;for(const a of t)if(o=e.includes(a),o)break;return o}(s,Object.keys(o)))&&(n=await this.getModelInfo(t),await this.loadModel({...o,model:t}))}else n=await this.getModelInfo();if(n||v("llamaCpp no current model",this.name),o.model=t,a)r=await this.getChatTemplate(n,o),r&&(i=A(r),r=r.prompt);else{const t=await this.formatPrompt(e,n,o);if(!t)throw new w("ChatTemplate:"+n.name,this.name);e=t,o.chatTemplate&&(i=A(o.chatTemplate),r=o.chatTemplate.prompt)}if(r?.parameters){const t=this.getDefaultParameters(r,n.name);if(t){(t.eot_token||t.eos_token)&&(i||(i=[]),t.eot_token&&!i.includes(t.eot_token)&&i.push(t.eot_token),t.eos_token&&!i.includes(t.eos_token)&&i.push(t.eos_token));const e=O(t);Array.isArray(e.stop)&&Array.isArray(o.stop)&&(o.stop=[...e.stop,...o.stop].filter(((t,e,o)=>o.indexOf(t)===e))),o={...e,...o}}}return(n.eot_token||n.eos_token)&&(i||(i=[]),n.eot_token&&!i.includes(n.eot_token)&&i.push(n.eot_token),n.eos_token&&!i.includes(n.eos_token)&&i.push(n.eos_token)),i&&i.length&&(Array.isArray(o.stop)?o.stop=Array.from(new Set([...i,...o.stop])):o.stop=i),o.grammar_id&&o.json_schema&&delete o.grammar_id,o.chatTemplate&&(o.chatTemplate.prompt?._id&&(l={id:o.chatTemplate.prompt._id},o.chatTemplate.version&&(l.version=o.chatTemplate.version)),delete o.chatTemplate),l&&(o.chatTemplateId=l),o.value=e,o}func({model:t,value:e,options:o}){return this.runAsyncCancelableTask(o,(async a=>{const n=a.aborter.signal;delete(a={...a}).aborter,a=await this.processModelOptions(t,e,a);const s=this.apiUrl??"http://localhost:8080",l={"Content-Type":"application/json"},c=o.stream;c&&(l.Connection="keep-alive",l.Accept="text/event-stream");const p=a.chatTemplateId;p&&delete a.chatTemplateId,e=a.value,delete a.value;const _={...a,prompt:e},m=await fetch(M(s,"/completion"),{method:"POST",headers:l,body:JSON.stringify(_),signal:n});let d;if(m.ok)if(c)d=h(m,i({chatTemplateId:p}));else{const t=await m.text();d=r(JSON.parse(t)),d.chatTemplateId=p}else{const t=this.emitError?this.emitError(m,s,_):void 0;if(t)d=t;else{if(!c)throw new f(await m.text(),"LlamaCppProvider");d=u(m.body)}}return d}))}async loadModel(t){const e=this.apiUrl??"http://localhost:8080";t=b(t,p);const o=await fetch(M(e,"/v1/models"),{method:"POST",body:JSON.stringify(t),headers:{Connection:"keep-alive","Content-Type":"application/json",Accept:"application/json"}}),a=await o.json(),n=a.error;if(n)throw n.code?new f(n.message,"loadModel",n.code):new f(n,"loadModel");const s=function(t){return b(t,_)}(a);return s}async getModelInfo(t){const e=this.apiUrl??"http://localhost:8080";t=t?"?model="+t:"";const o=await fetch(M(e,"/model/meta"+t)),a=await o.json(),n=a.error;if(n)throw n.code?new f(n.message,"getModelInfo",n.code):new f(n,"getModelInfo");const s=b(a,m);return s.params_size&&!s.scale&&(s.scale=g(s.params_size)),s.name+=".gguf",s}async listModels(){const t=this.apiUrl??"http://localhost:8080",e=await fetch(M(t,"/v1/models")),o=await e.json(),a=o.error;if(a)throw a.code?new f(a.message,"listModels",a.code):new f(a,"listModels");const n=o.data?.map((t=>t.name));return n}};y(S);var x=new S(k);function A(t){let e=[],o=t.version;Array.isArray(o)&&(o=o[0]);const a=t.prompt,n=a.version;if(o&&"@"!==o&&n?.[o]?.prompt){const t=n[o].prompt;t.eot_token&&e.push(t.eot_token),t.eos_token&&e.push(t.eos_token)}else if(a.prompt){const t=a.prompt;t.eot_token&&e.push(t.eot_token),t.eos_token&&e.push(t.eos_token)}return e.length?e:void 0}export{p as AILoadModelOptionsMap,m as AIModelSettingsMap,l as AIOptionsMap,S as LlamaCppProvider,k as LlamaCppProviderName,s as LlamaLoadModelOptionsKeys,n as LlamaVocabType,a as LlmArch,o as LlmSizeType,_ as ReverseAILoadModelOptions,c as ReverseAIOptionsMap,x as llamaCpp,r as llamaCppToAIResult,i as parseLlamaCppStream};