bserializer/dist/serialize_parallel.js

A TypeScript-based serialization/deserialization library built for exploration and learning.

import { Buffer } from 'buffer';
import os from 'os';
import * as url from 'url';
import { TYPE_CODES } from "./typecodes.js";
export const __dirnameLocal = url.fileURLToPath(new URL('.', import.meta.url));
import workerpool from 'workerpool';
import { _serializeDataMultithread } from "./serialize.js";
import { BufferWriter } from "./bufferWriter.js";
const MAX_WORKERS = os.cpus().length;
const pool = workerpool.pool({
    maxWorkers: 4
});
export async function parallelSerializeArray(array) {
    const chunkSize = Math.ceil(array.length / MAX_WORKERS);
    const chunks = [];
    for (let i = 0; i < array.length; i += chunkSize) {
        chunks.push(array.slice(i, i + chunkSize));
    }
    const resultPromises = [];
    for (const chunk of chunks) {
        const buffer = new BufferWriter();
        resultPromises.push(pool.exec(_serializeDataMultithread, [chunk, buffer]));
    }
    const results = await Promise.all(resultPromises);
    // Prepend ARRAY type code and total array length
    const totalSize = 1 + // ARRAY type code
        4 + // length
        results.reduce((sum, buf) => sum + buf.length, 0);
    const finalBuffer = Buffer.allocUnsafe(totalSize);
    let offset = 0;
    finalBuffer.writeUInt8(TYPE_CODES.ARRAY, offset++);
    finalBuffer.writeUInt32LE(array.length, offset);
    offset += 4;
    for (const buf of results) {
        buf.copy(finalBuffer, offset);
        offset += buf.length;
    }
    return finalBuffer;
}