weaviate-ts-embedded/dist/index.js

Weaviate TypeScript EmbeddedDB

import o from"fs";import{get as h}from"https";import m from"net";import{spawn as v}from"child_process";import{dirname as d,basename as f}from"path/posix";import{homedir as u}from"os";import{join as l}from"path";import{extract as y}from"tar";import{createHash as E}from"crypto";import P from"adm-zip";var w=l(u(),".cache/weaviate-embedded"),g=l(u(),".local/share/weaviate"),S="latest",c=class{binaryPath;persistenceDataPath;host;port;version;binaryUrl;env;constructor(t){if(this.version&&this.binaryUrl)throw new Error("cannot provide both version and binaryUrl");this.host=t&&t.host?t.host:"127.0.0.1",this.port=t&&t.port?t.port:6666,this.binaryUrl=t?.binaryUrl,this.version=this.parseVersion(t),this.binaryPath=this.getBinaryPath(t),this.persistenceDataPath=this.getPersistenceDataPath(),this.env=this.parseEnv(t)}parseEnv(t){this.persistenceDataPath||(this.persistenceDataPath=this.getPersistenceDataPath());let e={AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED:"true",QUERY_DEFAULTS_LIMIT:"20",PERSISTENCE_DATA_PATH:this.persistenceDataPath,CLUSTER_HOSTNAME:`Embedded_at_${this.port}`,DEFAULT_VECTORIZER_MODULE:"none",ENABLE_MODULES:"text2vec-openai,text2vec-cohere,text2vec-huggingface,ref2vec-centroid,generative-openai,qna-openai",...process.env};return t&&t.env&&Object.entries(t.env).forEach(([i,r])=>{e[i]=r}),e}parseVersion(t){if(!(t&&t.binaryUrl)){if(!t||!t.version)return S;if(t.version=="latest")return"latest";if(t.version.match(/[1-9]\.[1-9]{2}\..*/g))return t.version;throw new Error(`invalid version: ${t.version}. version must resemble '{major}.{minor}.{patch}, or 'latest'`)}}getBinaryPath(t){let e=process.env.XDG_CACHE_HOME;if(e||(e=w),this.version||(this.version=this.parseVersion(t)),this.binaryUrl){let i=E("md5").update(this.binaryUrl).digest("base64url");return`${e}-${i}`}return`${e}-${this.version}`}getPersistenceDataPath(){let t=process.env.XDG_DATA_HOME;return t||(t=g),t}},p=class{options;pid;constructor(t){this.options=t,this.pid=0,this.ensurePathsExist(),$()}async start(){await this.isListening()&&console.log(`Embedded db already listening @ ${this.options.host}:${this.options.port}`),await this.resolveWeaviateVersion().then(async()=>{await this.ensureWeaviateBinaryExists()}),this.options.env.CLUSTER_GOSSIP_BIND_PORT||(this.options.env.CLUSTER_GOSSIP_BIND_PORT=await D());let t=v(this.options.binaryPath,["--host",this.options.host,"--port",`${this.options.port}`,"--scheme","http"],{env:this.options.env});t.on("error",e=>{console.log(`embedded db failed to start: ${JSON.stringify(e)}`)}),t.stdout.pipe(process.stdout),t.stderr.pipe(process.stderr),this.pid=t.pid,console.log(`Started ${this.options.binaryPath} @ ${this.options.host}:${this.options.port} -- process ID ${this.pid}`),await this.waitTillListening()}stop(){try{process.kill(this.pid,"SIGTERM"),console.log(`Embedded db @ PID ${this.pid} successfully stopped`)}catch{console.log(`Tried to stop embedded db @ PID ${this.pid}.`,"PID not found, so nothing will be done")}}resolveWeaviateVersion(){return new Promise((t,e)=>{this.options.version=="latest"?h("https://api.github.com/repos/weaviate/weaviate/releases/latest",{headers:{"User-Agent":"Weaviate-Embedded-DB"}},i=>{let r="";i.on("data",n=>{r+=n}),i.on("end",()=>{if(i.statusCode===200)try{let n=JSON.parse(r);this.options.version=n.tag_name.slice(1),t()}catch(n){e(new Error(`failed to parse latest binary version response: ${JSON.stringify(n)}`))}else e(new Error(`fetch latest binary version, unexpected status code ${i.statusCode}: ${r}`))})}).on("error",i=>{e(new Error(`failed to find latest binary version: ${JSON.stringify(i)}`))}):t()})}async ensureWeaviateBinaryExists(){o.existsSync(`${this.options.binaryPath}`)||(console.log(`Binary ${this.options.binaryPath} does not exist.`,`Downloading binary for version ${this.options.version||this.options.binaryPath}`),await this.downloadBinary().then(async t=>{t.endsWith("tgz")?await this.untarBinary(t):await this.unzipBinary(t)}))}ensurePathsExist(){let t=d(this.options.binaryPath);o.mkdirSync(t,{recursive:!0}),o.mkdirSync(this.options.persistenceDataPath,{recursive:!0})}downloadBinary(){let t=this.buildBinaryUrl(),e;t.endsWith(".zip")?e=`${this.options.binaryPath}.zip`:e=`${this.options.binaryPath}.tgz`;let i=o.createWriteStream(e);return new Promise((r,n)=>{h(t,s=>{s.statusCode==200?(s.pipe(i),i.on("finish",()=>{i.close(),r(e)})):s.statusCode==302&&s.headers.location?h(s.headers.location,b=>{b.pipe(i),i.on("finish",()=>{i.close(),r(e)})}):s.statusCode==404?n(new Error(`failed to download binary: not found. are you sure Weaviate version ${this.options.version} exists? note that embedded db for linux is only supported for versions >= 1.18.0, and embedded db for mac is only supported for versions >= 1.19.8`)):n(new Error(`failed to download binary: unexpected status code: ${s.statusCode}`))}).on("error",s=>{o.unlinkSync(e),n(new Error(`failed to download binary: ${s}`))})})}buildBinaryUrl(){if(this.options.binaryUrl)return this.options.binaryUrl;let t;switch(process.arch){case"arm64":t="arm64";break;case"x64":t="amd64";break;default:throw new Error(`Embedded DB unsupported architecture: ${process.arch}`)}let e="tar.gz";return process.platform=="darwin"&&(e="zip",t="all"),`https://github.com/weaviate/weaviate/releases/download/v${this.options.version}/weaviate-v${this.options.version}-${process.platform}-${t}.${e}`}untarBinary(t){let e=o.createReadStream(t);return new Promise((i,r)=>{e.pipe(y({cwd:d(t),strict:!0}).on("finish",()=>{e.close(),o.unlinkSync(t),o.renameSync(l(d(this.options.binaryPath),"weaviate"),this.options.binaryPath),i(null)}).on("error",n=>{this.options.binaryUrl&&r(new Error(`failed to untar binary: ${n}, are you sure binaryUrl points to a tar file?`)),r(new Error(`failed to untar binary: ${JSON.stringify(n)}`))}))})}unzipBinary(t){let e=new P(t),i=e.getEntries();return new Promise((r,n)=>{i.forEach(s=>{s.entryName=="weaviate"&&(e.extractEntryTo(s.entryName,d(this.options.binaryPath),!1,!0,!1,f(this.options.binaryPath)),o.unlinkSync(t),o.chmodSync(this.options.binaryPath,511),r(null))}),n(new Error("failed to find binary in zip"))})}waitTillListening(){return new Promise((t,e)=>{let i=setTimeout(()=>{clearTimeout(i),clearInterval(r),e(new Error(`failed to connect to embedded db @ ${this.options.host}:${this.options.port}`))},3e4),r=setInterval(()=>{this.isListening().then(n=>{n&&(clearTimeout(i),clearInterval(r),t(null))})},500)})}isListening(){let t=m.connect(this.options.port,this.options.host);return new Promise(e=>{t.on("connect",()=>{console.log("connected to embedded db!"),t.destroy(),e(!0)}).on("error",i=>{console.log("Trying to connect to embedded db...",JSON.stringify(i)),t.destroy(),e(!1)})})}};function $(){let a=process.platform;if(a!="linux"&&a!="darwin")throw new Error(`${a} is not supported with EmbeddedDB`)}function D(){return new Promise((a,t)=>{let e=m.createServer();e.listen(0,()=>{let{port:i}=e.address();i?e.close(()=>a(i.toString())):t(new Error("failed to find open port"))})})}import C from"weaviate-ts-client";var O={client:function(a,t){return t||(t={host:"127.0.0.1:6666",scheme:"http"}),{...C.client(t),embedded:new p(a)}}},V=O;export{p as EmbeddedDB,c as EmbeddedOptions,V as default};