closevector-web/src/manager.ts

CloseVector is fundamentally a vector database. We have made dedicated libraries available for both browsers and node.js, aiming for easy integration no matter your platform. One feature we've been working on is its potential for scalability. Instead of b

import { CloseVectorHNSWWeb } from './hnswlibWasm';
import { download } from './loader';
import { CloseVectorEmbeddings, CloseVectorDocument } from 'closevector-common';
import { createPublicGetFileOperationUrl } from './lib';
import { CloseVectorFreeEmbeddings } from './embeddings';

type TypeOfOnProgress = Parameters<typeof download>[0]["onProgress"];
type TypeOfFileUrlCreator = (...args: any[]) => Promise<string>;
type TypeOfCreateLib = ConstructorParameters<typeof CloseVectorHNSWWeb>[1];

export class CloseVectorManager {

    accessKey: string | undefined;
    uuid: string | undefined;
    customEmbeddings: CloseVectorEmbeddings | undefined;

    lib: CloseVectorHNSWWeb | null = null;
    urlCreator: TypeOfFileUrlCreator | null = null;
    onProgress: TypeOfOnProgress | null = null;

    constructor(options: {
        // To download CloseVector database from server, you need to provide accessKey and uuid
        accessKey?: string,
        uuid?: string,

        // If you want to use your own embeddings, you can provide it here. Otherwise, we will use free embeddings
        customEmbeddings?: CloseVectorEmbeddings,

        // If you want to use your own urlCreator, you can provide it here. Otherwise, we will use default urlCreator
        fileUrlCreator?: TypeOfFileUrlCreator,

        // Progress callback for downloading from cloud. If you want to use your own onProgress, you can provide it here. Otherwise, we will use default onProgress
        onProgress?: TypeOfOnProgress,

    }) {
        const { accessKey, uuid, customEmbeddings } = options;
        this.accessKey = accessKey;
        this.uuid = uuid;
        this.customEmbeddings = customEmbeddings;

        this.onProgress = options.onProgress || null;
        this.urlCreator = options.fileUrlCreator || null;

        if (!this.urlCreator) {
            this.urlCreator = async () => {
                if (!this.uuid || !this.accessKey) {
                    throw new Error("uuid and accessKey are required");
                }

                let resp = await createPublicGetFileOperationUrl({
                    uuid: this.uuid,
                    accessKey: this.accessKey
                });
                return resp.url;
            }
        }

        if (!this.customEmbeddings) {
            if (!this.accessKey) {
                throw new Error("accessKey is required when customEmbeddings is not provided");
            }
            this.customEmbeddings = new CloseVectorFreeEmbeddings({ key: this.accessKey });
        }
    }

    async loadFromCloud() {
        if (!this.accessKey || !this.uuid) {
            throw new Error("uuid and accessKey are required");
        }

        if (!this.customEmbeddings) {
            throw new Error("customEmbeddings is required");
        }

        if (!this.urlCreator) {
            throw new Error("urlCreator is required");
        }

        const url = await this.urlCreator();
        await download({
            url,
            onProgress: this.onProgress || undefined,
        });

        const lib = await CloseVectorHNSWWeb.load(`${this.uuid}.hnsw`, this.customEmbeddings);
        return lib;
    }

    async createNewCloseVector(args: Partial<TypeOfCreateLib>) {
        if (!this.customEmbeddings) {
            throw new Error("customEmbeddings is required");
        }

        if (!args.space) {
            args.space = 'cosine';
        }

        const lib = new CloseVectorHNSWWeb(this.customEmbeddings, args as TypeOfCreateLib);
        return lib;
    }

    async fromDocuments(documents: CloseVectorDocument[]) {
        if (!this.customEmbeddings) {
            throw new Error("customEmbeddings is required");
        }

        let lib = await CloseVectorHNSWWeb.fromDocuments(documents, this.customEmbeddings);
        this.lib = lib;
    }

    async fromTexts(texts: string[], metadatas: object[] | object) {
        if (!this.customEmbeddings) {
            throw new Error("customEmbeddings is required");
        }

        let lib = await CloseVectorHNSWWeb.fromTexts(texts, metadatas, this.customEmbeddings);
        this.lib = lib;
    } 
}