closevector-web/src/lib.ts

CloseVector is fundamentally a vector database. We have made dedicated libraries available for both browsers and node.js, aiming for easy integration no matter your platform. One feature we've been working on is its potential for scalability. Instead of b

import { HnswlibModule, loadHnswlib } from 'closevector-hnswlib-wasm';
import jwt from 'closevector-crypto-jwt'; // you can also use any other JWT library, for example: jsonwebtoken

export const END_POINT = 'https://vector-kv.mega-ug.uk';

export enum FileVisibility {
    Public = 0,
    Private = 1
}

export const chunkArray = <T>(arr: T[], chunkSize: number) =>
    arr.reduce((chunks, elem, index) => {
        const chunkIndex = Math.floor(index / chunkSize);
        const chunk = chunks[chunkIndex] || [];
        // eslint-disable-next-line no-param-reassign
        chunks[chunkIndex] = chunk.concat([elem]);
        return chunks;
    }, [] as T[][]);

export async function encryptToken(object: Record<string, any>, secret: string): Promise<string> {
    const token = await jwt.sign(
        {
            // 1 hour from now on
            ...object,
            iat: Date.now(),
            exp: Math.floor((Date.now() + 60 * 60 * 1000) / 1000),
        },
        secret
    );
    return token;
}

export async function createUploadFileOperationUrl(options: {
    uuid?: string;
    public?: boolean;
    description: string;
    accessKey: string;
    secret: string;
}) {
    const { accessKey, secret } = options;

    let resp = await fetchUsingToken(`${END_POINT}/file/url`, {
        accessKey,
        secret,
        method: 'POST',
        headers: {
            'Content-Type': 'application/json',
        },
        body: JSON.stringify({
            uuid: options.uuid,
            description: options.description,
            visibility: options.public ? FileVisibility.Public : FileVisibility.Private
        }),
    });

    const body = await resp.json();
    return body as {
        url: string;
        uuid: string;
    };
}

export async function createPublicGetFileOperationUrl(options: {
    uuid: string;
    accessKey: string;
}) {
    const { uuid, accessKey } = options;

    let resp = await fetch(
        `https://vector-kv.mega-ug.uk/public/file/${uuid}?accessKey=${accessKey}`,
        {
            method: 'GET',
        }
    );
    if (resp.status !== 200) {
        throw new Error('fetching file fail');
    }
    let json = await resp.json();
    return json as {
        url: string;
        uuid: string;
    };
}

export async function createGetFileOperationUrl(options: {
    uuid: string;
    accessKey: string;
    secret: string;
}) {
    const { accessKey, secret, uuid } = options;

    let resp = await fetchUsingToken(`${END_POINT}/file/url/${uuid}`, {
        accessKey,
        secret,
        method: 'GET',
        headers: {
            'Content-Type': 'application/json',
        },
    });

    const body = await resp.json();

    return body as {
        url: string;
    };
}

/**
 *
 * @param url
 * @param options
 * @returns response returned by fetch
 * @summary should not be used in web
 */
export const fetchUsingToken = async function fetchUsingToken(
    url: string,
    options: RequestInit & {
        accessKey: string;
        secret: string;
    }
) {
    const { accessKey, secret } = options;

    let res = await fetch(url, {
        ...options,
        headers: {
            ...options.headers,
            Authorization: `Bearer ${accessKey}:${await encryptToken({ accessKey }, secret)}`,
        },
    });

    if (res.status !== 200) {
        let json = await res.json();
        throw new Error(await (json as any).message);
    }

    return res;
};

let __lib: HnswlibModule | undefined = undefined;
async function getLib() {
    if (!__lib) {
        __lib = await loadHnswlib();
    }
    return __lib;
}

export class IDBFS {
    static async writeBufferToFile(path: string, buffer: Uint8Array) {
        return new Promise<void>(async (resolve, reject) => {
            const lib = await getLib();
            lib.EmscriptenFileSystemManager.writeBufferToFile(path, buffer, resolve);
        });
    }

    static async writeStringToFile(path: string, data: string) {
        return new Promise<void>(async (resolve, reject) => {
            const lib = await getLib();
            lib.EmscriptenFileSystemManager.writeStringToFile(path, data, resolve);
        });
    }

    static async getBufferFromFile(path: string) {
        const lib = await getLib();
        return await lib.EmscriptenFileSystemManager.getBufferFromFile(path);
    }

    static async getStringFromFile(path: string) {
        const lib = await getLib();
        return await lib.EmscriptenFileSystemManager.getStringFromFile(path);
    }

    static async writeFs() {
        const lib = await getLib();

        const pr = new Promise<void>((resolve, reject) => {
            lib.EmscriptenFileSystemManager.syncFS(false, () => {
                resolve();
            });
        });

        await pr;
    }
}

export function pathJoin(...parts: string[]) {
    return parts.join('/').replace(/\/+/g, '/');
}