@micrio/tiler-base/dist/lib/uploader.js

The base Micrio client-side tiler package used by the Micrio CLI and GUI tools

import { promises as fs } from 'node:fs';
import https from 'https';
import { NUM_UPLOAD_TRIES, SIGNED_URIS, UPLOAD_THREADS } from '../globals.js';
import { api } from './micrioApi.js';
import { sanitize } from './utils.js';
/** A session-wide multithreaded uploader to Micrio */
export class Uploader {
    agent;
    state;
    folder;
    format;
    outDir;
    jobs = [];
    oncomplete;
    uris = {};
    running = new Map();
    errored = new Map();
    constructor(agent, state, folder, format, outDir) {
        this.agent = agent;
        this.state = state;
        this.folder = folder;
        this.format = format;
        this.outDir = outDir;
        this.outDir = sanitize(outDir, outDir);
    }
    /** This is called for each individual resulting tile of an image operation
     * Or the final function to send the succesful status to Micrio after all tiles
     * of an image have been uploaded.
     */
    add(jobs) {
        this.jobs.push(...jobs);
        if (this.state.job)
            this.state.update?.(this.state.job.numUploads += jobs.length);
        this.nextBatch();
    }
    /** Async function to await queue upload completion */
    complete() {
        return new Promise(ok => {
            if (this.jobs.length + this.running.size == 0)
                return ok();
            this.oncomplete = ok;
        });
    }
    /** Get signed R2 upload URLs for the next batch of queued file uploads */
    getUploadUris(first) {
        const files = this.jobs.filter(t => !(t instanceof Function || this.uris[typeof t == 'string' ? t : t.path])).slice(0, SIGNED_URIS - (first ? 1 : 0));
        if (first)
            files.unshift(first);
        if (!files.length)
            return;
        const call = api(this.state.account, this.agent, `/../${this.folder.split('/')[1]}/store`, { files: files.map(f => sanitize(f, this.outDir)) })
            .catch(e => { throw new Error('Upload error: ' + (e.message ?? 'Upload permission denied')); })
            .then(r => {
            if (!r)
                throw new Error('Upload permission denied.');
            // After the request is completed, assign each file its signed upload URL
            r.keys.forEach((sig, i) => this.uris[files[i]] = `https://${r.r2Base}.r2.cloudflarestorage.com/${sanitize(files[i], this.outDir)}?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=${r.key}%2F${r.time.slice(0, 8)}%2Fauto%2Fs3%2Faws4_request&X-Amz-Date=${r.time}&X-Amz-Expires=300&X-Amz-Signature=${sig}&X-Amz-SignedHeaders=host&x-id=PutObject`);
        });
        // Until finished, assign the running promise as the upload url
        files.forEach(f => this.uris[f] = call);
    }
    /** Get an indivual file/tile uploadUrl */
    async getUploadUri(f) {
        if (!this.uris[f])
            await this.getUploadUris(f);
        if (this.uris[f] instanceof Promise)
            await this.uris[f];
        return this.uris[f];
    }
    /** This makes sure all upload threads are always filled */
    nextBatch() {
        let r = UPLOAD_THREADS - this.running.size;
        while (--r > 0)
            this.next();
    }
    /** Do the next upload thread */
    async next() {
        if (this.running.size >= UPLOAD_THREADS)
            return;
        const job = this.jobs.shift();
        if (!job)
            return;
        this.running.set(job, (job instanceof Function ? job()
            : typeof job == 'string' ? this.getUploadUri(job).then(uri => this.upload(uri, job))
                : this.getUploadUri(job.path).then(uri => this.upload(uri, job.path, job.blob))).catch((e) => {
            // Could not get upload URLs -- immediately kill
            if (e?.message?.startsWith('Upload error: '))
                throw e;
            const numErrored = (this.errored.get(job) ?? 0) + 1;
            this.errored.set(job, numErrored);
            if (numErrored > NUM_UPLOAD_TRIES)
                throw new Error(`Fatal error: could not ${job instanceof Function ? 'finalize upload' : `upload ${job}`} after ${NUM_UPLOAD_TRIES} tries. (${e?.message ?? 'Error'})`);
            // Try again
            this.jobs.push(job);
        }).then(() => {
            this.running.delete(job);
            if (typeof job == 'string')
                delete this.uris[job];
            const remaining = this.jobs.length + this.running.size;
            if (this.state?.job)
                this.state.update?.(this.state.job.numUploaded = this.state.job.numUploads - remaining);
            if (this.oncomplete)
                this.state?.log(`Remaining uploads: ${remaining}...`, true);
            if (this.jobs.length)
                this.nextBatch();
            else if (!remaining)
                this.oncomplete?.();
        }));
    }
    /** Individual file / tile upload */
    async upload(_url, path, _blob) {
        return new Promise(async (ok, err) => {
            const url = new URL(_url);
            const blob = _blob ?? await fs.readFile(path);
            if (this.state?.job)
                this.state.job.bytesResult += blob.byteLength;
            const req = https.request({
                host: url.host,
                path: url.pathname + url.search,
                method: 'PUT',
                agent: this.agent,
                headers: {
                    'Content-Type': _blob ? 'application/octet-stream' : `image/${this.format}`,
                    'Content-Length': blob.byteLength,
                }
            }, res => {
                req.destroy();
                if (res.statusCode == 200)
                    ok();
                else
                    err(new Error(res.statusCode + ': ' + res.statusMessage));
            });
            req.on('error', (e) => {
                req.destroy();
                err(e);
            });
            req.write(blob);
            req.end();
        });
    }
}