closevector-web/dist/loader.js

CloseVector is fundamentally a vector database. We have made dedicated libraries available for both browsers and node.js, aiming for easy integration no matter your platform. One feature we've been working on is its potential for scalability. Instead of b

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.upload = exports.download = void 0;
const zlib = require("closevector-browserify-zlib");
const Stream = require("closevector-stream-browserify");
const tar = require("tar-stream");
const lib_1 = require("./lib");
class ReaderAsReadableStream extends (Stream.Readable || Stream.default?.Readable) {
    reader;
    constructor(reader) {
        super();
        this.reader = reader;
    }
    _read() {
        this.reader.read().then(({ value, done }) => {
            if (done) {
                this.push(null);
            }
            else {
                this.push(value);
            }
        });
    }
}
class Saver extends (Stream.Writable || Stream.default.Writable) {
    meta;
    buffer;
    offset = 0;
    constructor(meta) {
        super();
        this.meta = meta;
        this.offset = 0;
        this.buffer = new Uint8Array(meta.size);
    }
    _write(chunk, encoding, callback) {
        this.buffer.set(chunk, this.offset);
        this.offset += chunk.length;
        if (this.offset >= this.meta.size) {
            (async () => {
                try {
                    if (this.meta.type == 'file') {
                        await lib_1.IDBFS.writeBufferToFile(this.meta.name, this.buffer);
                    }
                }
                catch (e) {
                    return callback(e);
                }
            })();
        }
        callback(null);
    }
}
class ProgressTransformer extends (Stream.Transform || Stream.default.Transform) {
    onProgress;
    constructor(onProgress) {
        super();
        this.onProgress = onProgress;
    }
    _transform(chunk, encoding, callback) {
        this.onProgress(chunk.length);
        callback(null, chunk);
    }
}
class BufferCollector extends (Stream.Writable || Stream.default.Writable) {
    buffer;
    constructor() {
        super();
        this.buffer = new Uint8Array(0);
    }
    getBuffer() {
        return this.buffer;
    }
    _write(chunk, encoding, callback) {
        this.buffer = new Uint8Array([...this.buffer, ...chunk]);
        callback(null);
    }
}
async function download(options) {
    if (!window.ReadableStream) {
        throw new Error('ReadableStream not supported');
    }
    const extract = tar.extract();
    // download as stream
    const res = await fetch(options.url);
    if (!res.ok) {
        throw new Error(`unexpected response ${res.statusText}`);
    }
    let bodyReader = res.body.getReader();
    let totalBytes = +res.headers.get('Content-Length');
    let rs = new ReaderAsReadableStream(bodyReader);
    let totalBytesLoaded = 0;
    rs = rs.pipe(new ProgressTransformer((bytesLoaded) => {
        if (options.onProgress) {
            totalBytesLoaded += bytesLoaded;
            options.onProgress({
                loaded: totalBytesLoaded,
                total: totalBytes,
            });
        }
    }));
    rs = rs.pipe(zlib.createGunzip()).pipe(extract);
    return new Promise(async (resolve, reject) => {
        extract.on('entry', function (header, stream, next) {
            // header is the tar header
            // stream is the content body (might be an empty stream)
            // call next when you are done with this entry
            if (header.type == 'directory') {
                next();
                return;
            }
            stream.pipe(new Saver({
                ...header,
            }));
            stream.on('end', function () {
                next(); // ready for next entry
            });
            stream.resume(); // just auto drain the stream
        });
        extract.on('finish', function () { });
        rs.on('finish', () => {
            resolve();
        });
        rs.on('error', e => {
            reject(e);
        });
    });
}
exports.download = download;
async function upload(options) {
    const indexContent = await lib_1.IDBFS.getBufferFromFile(options.path + '/hnswlib.index');
    const argsContent = await lib_1.IDBFS.getBufferFromFile(options.path + '/args.json');
    const metaContent = await lib_1.IDBFS.getBufferFromFile(options.path + '/docstore.json');
    // pack files in path and upload as stream to url
    const pack = tar.pack();
    function packEntry(name, content) {
        // convert content to buffer
        // return new Promise<void>((resolve, reject) => {
        //     pack.entry({ name }, Uint8Array.from(content), err => {
        //         if (err) {
        //             reject(err);
        //         } else {
        //             resolve();
        //         }
        //     });
        // });
        pack.entry({ name }, Uint8Array.from(content));
    }
    packEntry(options.path + '/hnswlib.index', indexContent);
    packEntry(options.path + '/args.json', argsContent);
    packEntry(options.path + '/docstore.json', metaContent);
    pack.finalize();
    let bufferCollector = new BufferCollector();
    pack.pipe(zlib.createGzip()).pipe(bufferCollector);
    await new Promise((resolve, reject) => {
        // @ts-ignore
        bufferCollector.on('finish', async () => {
            const buffer = bufferCollector.getBuffer();
            const res = await fetch(options.url, {
                method: 'POST',
                body: buffer,
            });
            if (!res.ok) {
                return reject(new Error(`unexpected response ${res.statusText}`));
            }
            resolve();
        });
    });
}
exports.upload = upload;