@speckle/objectloader2/dist/esm/operations/downloaders/serverDownloader.js

This is an updated objectloader for the Speckle viewer written in typescript

import BatchedPool from '../../helpers/batchedPool.js';
import { ObjectLoaderRuntimeError } from '../../types/errors.js';
import { isBase, take } from '../../types/types.js';
export default class ServerDownloader {
    #requestUrlRootObj;
    #requestUrlChildren;
    #headers;
    #options;
    #fetch;
    #results;
    #downloadQueue;
    #decoder = new TextDecoder();
    constructor(options) {
        this.#options = options;
        this.#fetch =
            options.fetch ?? ((...args) => globalThis.fetch(...args));
        this.#headers = {};
        if (options.headers) {
            for (const header of options.headers.entries()) {
                this.#headers[header[0]] = header[1];
            }
        }
        this.#headers['Accept'] = `text/plain`;
        if (this.#options.token) {
            this.#headers['Authorization'] = `Bearer ${this.#options.token}`;
        }
        this.#requestUrlChildren = `${this.#options.serverUrl}/api/getobjects/${this.#options.streamId}`;
        this.#requestUrlRootObj = `${this.#options.serverUrl}/objects/${this.#options.streamId}/${this.#options.objectId}/single`;
    }
    #getDownloadCountAndSizes(total) {
        if (total <= 50) {
            return [total];
        }
        return [10000, 25000, 10000, 1000];
    }
    initializePool(params) {
        const { results, total } = params;
        this.#results = results;
        this.#downloadQueue = new BatchedPool({
            concurrencyAndSizes: this.#getDownloadCountAndSizes(total),
            maxWaitTime: params.maxDownloadBatchWait,
            processFunction: (batch) => this.downloadBatch({
                batch,
                url: this.#requestUrlChildren,
                headers: this.#headers
            })
        });
    }
    #getPool() {
        if (this.#downloadQueue) {
            return this.#downloadQueue;
        }
        throw new Error('Download pool is not initialized');
    }
    add(id) {
        this.#getPool().add(id);
    }
    async disposeAsync() {
        await this.#downloadQueue?.disposeAsync();
    }
    #processJson(baseId, unparsedBase) {
        let base;
        try {
            base = JSON.parse(unparsedBase);
        }
        catch (e) {
            throw new Error(`Error parsing object ${baseId}: ${e.message}`);
        }
        if (isBase(base)) {
            return { baseId, base };
        }
        else {
            throw new ObjectLoaderRuntimeError(`${baseId} is not a base`);
        }
    }
    async downloadBatch(params) {
        const { batch, url, headers } = params;
        const keys = new Set(batch);
        const response = await this.#fetch(url, {
            method: 'POST',
            headers: { ...headers, 'Content-Type': 'application/json' },
            body: JSON.stringify({ objects: JSON.stringify(batch) })
        });
        this.#validateResponse(response);
        if (!response.body) {
            throw new Error('ReadableStream not supported or response has no body.');
        }
        const reader = response.body.getReader();
        let leftover = new Uint8Array(0);
        let count = 0;
        while (true) {
            const { done, value } = await reader.read();
            if (done)
                break;
            leftover = await this.processArray(leftover, value, keys, async () => {
                count++;
                if (count % 1000 === 0) {
                    await new Promise((resolve) => setTimeout(resolve, 100)); //allow other stuff to happen
                }
            });
        }
        if (keys.size > 0) {
            throw new Error('Items requested were not downloaded: ' + take(keys.values(), 10).join(','));
        }
    }
    async processArray(leftover, value, keys, callback) {
        //this concat will allocate a new array
        const combined = this.concatUint8Arrays(leftover, value);
        let start = 0;
        //subarray doesn't allocate
        for (let i = 0; i < combined.length; i++) {
            if (combined[i] === 0x0a) {
                const line = combined.subarray(start, i); // line without \n
                //strings are allocated here
                const item = this.processLine(line);
                this.#results?.add(item);
                start = i + 1;
                await callback();
                keys.delete(item.baseId);
            }
        }
        return combined.subarray(start); // carry over remainder
    }
    processLine(line) {
        for (let i = 0; i < line.length; i++) {
            if (line[i] === 0x09) {
                //this is a tab
                const baseId = this.#decoder.decode(line.subarray(0, i));
                const json = line.subarray(i + 1);
                const base = this.#decoder.decode(json);
                const item = this.#processJson(baseId, base);
                item.size = json.length;
                return item;
            }
        }
        throw new ObjectLoaderRuntimeError('Invalid line format: ' + this.#decoder.decode(line));
    }
    concatUint8Arrays(a, b) {
        const c = new Uint8Array(a.length + b.length);
        c.set(a, 0);
        c.set(b, a.length);
        return c;
    }
    async downloadSingle() {
        const response = await this.#fetch(this.#requestUrlRootObj, {
            headers: this.#headers
        });
        this.#validateResponse(response);
        const responseText = await response.text();
        const item = this.#processJson(this.#options.objectId, responseText);
        item.size = 0;
        return item;
    }
    #validateResponse(response) {
        if (!response.ok) {
            if ([401, 403].includes(response.status)) {
                throw new ObjectLoaderRuntimeError('You do not have access!');
            }
            throw new ObjectLoaderRuntimeError(`Failed to fetch objects: ${response.status} ${response.statusText})`);
        }
    }
}
//# sourceMappingURL=serverDownloader.js.map