@micrio/tiler-base
Version:
The base Micrio client-side tiler package used by the Micrio CLI and GUI tools
163 lines (162 loc) • 7.79 kB
JavaScript
import { pdf } from 'pdf-to-img';
import { promises as fs } from 'node:fs';
import os from 'os';
import path from 'path';
import https from 'https';
import { OMNI_PROCESSING_THREADS, PROCESSING_THREADS } from '../globals.js';
import { api } from '../lib/micrioApi.js';
import { fsExists, jdToTime } from '../lib/utils.js';
import { Uploader } from '../lib/uploader.js';
import { process } from '../lib/tiler.js';
import { getArchiveBin } from '../lib/archive.js';
const setStatus = (state, status, override, noLog) => {
if (!state)
return;
if (state.job)
state?.update?.(state.job.status = status);
if (!noLog)
state.log(status, override);
};
/** Process all specified and upload them to Micrio */
export async function upload(files, opts, state) {
if (!state?.account?.email)
throw new Error(`Not logged in. Run 'micrio login' first`);
let url;
try {
url = new URL(opts.destination);
}
catch (e) {
throw new Error('Invalid target URL. This has to be the full URL of the target folder of the Micrio dashboard (https://dash.micr.io/...)');
}
const folder = url.pathname;
const httpAgent = new https.Agent({
rejectUnauthorized: true,
keepAlive: true,
timeout: 3000
});
// Check if user has access to the requested destination folder
// This automatically throws an error if the user doesn't have access or the folder doesn't exist
await api(state.account, httpAgent, `${folder}/access`);
const start = Date.now();
if (!files.length)
throw new Error('No images to process');
let origImageNum = files.length;
const tmpDir = path.join(os.tmpdir(), '_micrio');
if (!await fsExists(tmpDir))
await fs.mkdir(tmpDir);
const outDir = path.join(tmpDir, Math.floor(Math.random() * 10000000) + '');
if (!await fsExists(outDir))
await fs.mkdir(outDir);
// TS is weird here -- if this can be undefined, compilation messes up
let omni = {};
const uploader = new Uploader(httpAgent, state, folder, opts.format, outDir);
const hQueue = {};
// Omni images start with single image to create main ID
// After the initial image is created, it has ID, and processing threads will be maximized
let threads = opts.type == 'omni' ? 1 : PROCESSING_THREADS;
let totalJobs = 0;
let numProcessed = 0;
let singleImageResultId;
// Original files mapped to output directories
const fileOutputDirs = new Map();
// Process and upload an original image file while there are available threads
const addToQueue = async (fileName, saveId, _opts = {}) => {
const queue = Object.values(hQueue);
if (queue.length >= threads)
await Promise.any(queue);
hQueue[fileName] = process(state, uploader, fileName, outDir, folder + (_opts.pdfAlbum ? '/' + _opts.pdfAlbum.slug : ''), opts.format, opts.type, {
omniId: omni?.id,
omniFrame: _opts.omniFrameIdx,
omniTotalFrames: totalJobs,
}).then(r => {
delete hQueue[fileName];
numProcessed++;
if (state.job)
state.update?.(state.job.numProcessed = numProcessed);
if (numProcessed == totalJobs)
setStatus(state, 'Uploading...', false, true);
if (opts.type == 'omni' && !omni.id) {
fileOutputDirs.set(r.id, []);
omni = r;
threads = OMNI_PROCESSING_THREADS;
}
// When the user uploaded a single omni, pdf or image, return the view URL after completion
if (saveId)
singleImageResultId = r.id;
// When reading to create preview archive, add the image ID there
fileOutputDirs.get(omni?.id || _opts.pdfAlbum?.id)?.push([fileName, r]);
}, e => {
// If one omni frame or pdf page fails, everything fails
if (opts.type == 'omni' || _opts.pdfAlbum)
throw e;
const message = e && typeof e == 'string' ? e : e?.message;
state.log(`Error: Could not tile ${fileName}: ${message?.trim() ?? 'Unknown error'}`);
origImageNum--;
});
};
const pdfAlbums = [];
// PDF parser
for (let i = 0; i < files.length; i++) {
const f = files[i];
if (f.endsWith('.pdf')) {
state?.log(`Parsing PDF file ${f}...`);
let counter = 1;
const document = await pdf(f, { scale: parseInt(opts.pdfScale || '4') })
.catch(e => { throw new Error(`PDF reading error: ${e.toString()}`); });
totalJobs += document.length;
// Create a new Micrio PDF album in the specified folder
const pdfAlbum = await api(state.account, uploader.agent, `${folder}/create`, {
name: encodeURIComponent(f),
type: 'pdf'
});
fileOutputDirs.set(pdfAlbum.id, []);
pdfAlbums.push(pdfAlbum);
for await (const image of document) {
state.log(`Processing page ${counter} / ${document.length}...`, true);
const fName = `${f}.${(counter).toString().padStart(4, '0')}.png`;
// Not using the async method here corrupts the written image -_-
// Took a while to figure that out.
await fs.writeFile(fName, image);
// Already start uploading and processing while parsing
await addToQueue(fName, files.length == 1, { pdfAlbum });
counter++;
}
files.splice(i--, 1);
}
}
// Regular image files
if (files.length) {
totalJobs += files.length;
setStatus(state, 'Processing...', false, true);
if (files.length == 1)
state?.log('Processing: ' + files[0]);
for (let i = 0; i < files.length; i++)
await addToQueue(files[i], opts.type == 'omni' || files.length == 1, { omniFrameIdx: i });
}
// Wait for all images to finish processing
await Promise.all(Object.values(hQueue));
// Wait until the Uploader has finished all of its individual upload threads
await uploader.complete();
// In case of an omni object or PDF file, create the pregenerated optimized viewing package
// which contains thumbnails of each individual frame/page
for (let [containerId, entries] of Array.from(fileOutputDirs.entries())) {
const album = pdfAlbums.find(a => a.id == containerId);
// Generate and upload the file and tell Micrio that the omni object or album is published
uploader.add([
{
path: omni?.id ? `${omni.id}/base.bin` : `g/${containerId}.${Math.floor(jdToTime(album.created) / 1000)}.bin`,
blob: await getArchiveBin(outDir, opts.format, containerId, entries, omni?.id)
},
() => api(state.account, uploader.agent, `${folder}${album ? '/' + album.slug : ''}/@${omni.id || 'album'}/status`, { status: 4, albumVersion: album?.created })
]);
}
// Wait until the Uploader has finished all of its individual upload threads
await uploader.complete();
state?.log();
setStatus(state, 'Finalizing...');
// Remove the entire original directory containing all tile results
await fs.rm(outDir, { recursive: true, force: true });
setStatus(state, `${origImageNum ? 'Succesfully a' : 'A'}dded ${opts.type == 'omni' ? `a 360 object image (${origImageNum} frames)` : `${origImageNum} file${origImageNum == 1 ? '' : 's'}`} in ${Math.round(Date.now() - start) / 1000}s.`);
if (singleImageResultId)
setStatus(state, 'Resulting viewable URL: https://i.micr.io/' + singleImageResultId);
}