streaming-iterables/dist/index.mjs

A collection of utilities for async iterables. Designed to replace your streams.

async function* _batch(size, iterable) {
    let dataBatch = [];
    for await (const data of iterable) {
        dataBatch.push(data);
        if (dataBatch.length === size) {
            yield dataBatch;
            dataBatch = [];
        }
    }
    if (dataBatch.length > 0) {
        yield dataBatch;
    }
}
function* _syncBatch(size, iterable) {
    let dataBatch = [];
    for (const data of iterable) {
        dataBatch.push(data);
        if (dataBatch.length === size) {
            yield dataBatch;
            dataBatch = [];
        }
    }
    if (dataBatch.length > 0) {
        yield dataBatch;
    }
}
function batch(size, iterable) {
    if (iterable === undefined) {
        return curriedIterable => batch(size, curriedIterable);
    }
    if (iterable[Symbol.asyncIterator]) {
        return _batch(size, iterable);
    }
    return _syncBatch(size, iterable);
}

function getIterator(iterable) {
    if (typeof iterable.next === 'function') {
        return iterable;
    }
    if (typeof iterable[Symbol.iterator] === 'function') {
        return iterable[Symbol.iterator]();
    }
    if (typeof iterable[Symbol.asyncIterator] === 'function') {
        return iterable[Symbol.asyncIterator]();
    }
    throw new TypeError('"values" does not to conform to any of the iterator or iterable protocols');
}

function defer() {
    let reject;
    let resolve;
    const promise = new Promise((resolveFunc, rejectFunc) => {
        resolve = resolveFunc;
        reject = rejectFunc;
    });
    return {
        promise,
        reject,
        resolve,
    };
}

/// <reference lib="esnext.asynciterable" />
function _buffer(size, iterable) {
    const iterator = getIterator(iterable);
    const resultQueue = [];
    const readQueue = [];
    let reading = false;
    let ended = false;
    function fulfillReadQueue() {
        while (readQueue.length > 0 && resultQueue.length > 0) {
            const readDeferred = readQueue.shift();
            const { error, value } = resultQueue.shift();
            if (error) {
                readDeferred.reject(error);
            }
            else {
                readDeferred.resolve({ done: false, value });
            }
        }
        while (readQueue.length > 0 && ended) {
            const { resolve } = readQueue.shift();
            resolve({ done: true, value: undefined });
        }
    }
    async function fillQueue() {
        if (ended) {
            return;
        }
        if (reading) {
            return;
        }
        if (resultQueue.length >= size) {
            return;
        }
        reading = true;
        try {
            const { done, value } = await iterator.next();
            if (done) {
                ended = true;
            }
            else {
                resultQueue.push({ value });
            }
        }
        catch (error) {
            ended = true;
            resultQueue.push({ error });
        }
        fulfillReadQueue();
        reading = false;
        fillQueue();
    }
    async function next() {
        if (resultQueue.length > 0) {
            const { error, value } = resultQueue.shift();
            if (error) {
                throw error;
            }
            fillQueue();
            return { done: false, value };
        }
        if (ended) {
            return { done: true, value: undefined }; // stupid ts
        }
        const deferred = defer();
        readQueue.push(deferred);
        fillQueue();
        return deferred.promise;
    }
    const asyncIterableIterator = {
        next,
        [Symbol.asyncIterator]: () => asyncIterableIterator,
    };
    return asyncIterableIterator;
}
function* syncBuffer(size, iterable) {
    const valueQueue = [];
    let e;
    try {
        for (const value of iterable) {
            valueQueue.push(value);
            if (valueQueue.length <= size) {
                continue;
            }
            yield valueQueue.shift();
        }
    }
    catch (error) {
        e = error;
    }
    for (const value of valueQueue) {
        yield value;
    }
    if (e) {
        throw e;
    }
}
function buffer(size, iterable) {
    if (iterable === undefined) {
        return curriedIterable => buffer(size, curriedIterable);
    }
    if (size === 0) {
        return iterable;
    }
    if (iterable[Symbol.asyncIterator]) {
        return _buffer(size, iterable);
    }
    return syncBuffer(size, iterable);
}

async function _collect(iterable) {
    const values = [];
    for await (const value of iterable) {
        values.push(value);
    }
    return values;
}
function collect(iterable) {
    if (iterable[Symbol.asyncIterator]) {
        return _collect(iterable);
    }
    return Array.from(iterable);
}

async function* _concat(iterables) {
    for await (const iterable of iterables) {
        yield* iterable;
    }
}
function* _syncConcat(iterables) {
    for (const iterable of iterables) {
        yield* iterable;
    }
}
function concat(...iterables) {
    const hasAnyAsync = iterables.find(itr => itr[Symbol.asyncIterator] !== undefined);
    if (hasAnyAsync) {
        return _concat(iterables);
    }
    else {
        return _syncConcat(iterables);
    }
}

async function _consume(iterable) {
    for await (const val of iterable) {
        // do nothing
    }
}
function consume(iterable) {
    if (iterable[Symbol.asyncIterator]) {
        return _consume(iterable);
    }
    for (const val of iterable) {
        // do nothing
    }
}

async function* _filter(filterFunc, iterable) {
    for await (const data of iterable) {
        if (await filterFunc(data)) {
            yield data;
        }
    }
}
function filter(filterFunc, iterable) {
    if (iterable === undefined) {
        return (curriedIterable) => _filter(filterFunc, curriedIterable);
    }
    return _filter(filterFunc, iterable);
}

async function* flatten(iterable) {
    for await (const maybeItr of iterable) {
        if (maybeItr && typeof maybeItr !== 'string' && (maybeItr[Symbol.iterator] || maybeItr[Symbol.asyncIterator])) {
            yield* flatten(maybeItr);
        }
        else {
            yield maybeItr;
        }
    }
}

async function* _map(func, iterable) {
    for await (const val of iterable) {
        yield await func(val);
    }
}
function map(func, iterable) {
    if (iterable === undefined) {
        return curriedIterable => _map(func, curriedIterable);
    }
    return _map(func, iterable);
}

function flatMap(func, iterable) {
    if (iterable === undefined) {
        return curriedIterable => flatMap(func, curriedIterable);
    }
    return filter(i => i !== undefined && i !== null, flatten(map(func, iterable)));
}

function _flatTransform(concurrency, func, iterable) {
    const iterator = getIterator(iterable);
    const resultQueue = [];
    const readQueue = [];
    let ended = false;
    let reading = false;
    let inflightCount = 0;
    let lastError = null;
    function fulfillReadQueue() {
        while (readQueue.length > 0 && resultQueue.length > 0) {
            const { resolve } = readQueue.shift();
            const value = resultQueue.shift();
            resolve({ done: false, value });
        }
        while (readQueue.length > 0 && inflightCount === 0 && ended) {
            const { resolve, reject } = readQueue.shift();
            if (lastError) {
                reject(lastError);
                lastError = null;
            }
            else {
                resolve({ done: true, value: undefined });
            }
        }
    }
    async function fillQueue() {
        if (ended) {
            fulfillReadQueue();
            return;
        }
        if (reading) {
            return;
        }
        if (inflightCount + resultQueue.length >= concurrency) {
            return;
        }
        reading = true;
        inflightCount++;
        try {
            const { done, value } = await iterator.next();
            if (done) {
                ended = true;
                inflightCount--;
                fulfillReadQueue();
            }
            else {
                mapAndQueue(value);
            }
        }
        catch (error) {
            ended = true;
            inflightCount--;
            lastError = error;
            fulfillReadQueue();
        }
        reading = false;
        fillQueue();
    }
    async function mapAndQueue(itrValue) {
        try {
            const value = await func(itrValue);
            if (value && value[Symbol.asyncIterator]) {
                for await (const asyncVal of value) {
                    resultQueue.push(asyncVal);
                }
            }
            else {
                resultQueue.push(value);
            }
        }
        catch (error) {
            ended = true;
            lastError = error;
        }
        inflightCount--;
        fulfillReadQueue();
        fillQueue();
    }
    async function next() {
        if (resultQueue.length === 0) {
            const deferred = defer();
            readQueue.push(deferred);
            fillQueue();
            return deferred.promise;
        }
        const value = resultQueue.shift();
        fillQueue();
        return { done: false, value };
    }
    const asyncIterableIterator = {
        next,
        [Symbol.asyncIterator]: () => asyncIterableIterator,
    };
    return asyncIterableIterator;
}
function flatTransform(concurrency, func, iterable) {
    if (func === undefined) {
        return (curriedFunc, curriedIterable) => curriedIterable
            ? flatTransform(concurrency, curriedFunc, curriedIterable)
            : flatTransform(concurrency, curriedFunc);
    }
    if (iterable === undefined) {
        return (curriedIterable) => flatTransform(concurrency, func, curriedIterable);
    }
    return filter(i => i !== undefined && i !== null, flatten(_flatTransform(concurrency, func, iterable)));
}

async function onceReadable(stream) {
    return new Promise(resolve => {
        stream.once('readable', () => {
            resolve();
        });
    });
}
async function* _fromStream(stream) {
    while (true) {
        const data = stream.read();
        if (data !== null) {
            yield data;
            continue;
        }
        if (stream._readableState.ended) {
            break;
        }
        await onceReadable(stream);
    }
}
function fromStream(stream) {
    if (typeof stream[Symbol.asyncIterator] === 'function') {
        return stream;
    }
    return _fromStream(stream);
}

async function* merge(...iterables) {
    const sources = new Set(iterables.map(getIterator));
    while (sources.size > 0) {
        for (const iterator of sources) {
            const nextVal = await iterator.next();
            if (nextVal.done) {
                sources.delete(iterator);
            }
            else {
                yield nextVal.value;
            }
        }
    }
}

function pipeline(firstFn, ...fns) {
    let previousFn = firstFn();
    for (const func of fns) {
        previousFn = func(previousFn);
    }
    return previousFn;
}

async function* _parallelMap(concurrency, func, iterable) {
    let transformError = null;
    const wrapFunc = value => ({
        value: func(value),
    });
    const stopOnError = async function* (source) {
        for await (const value of source) {
            if (transformError) {
                return;
            }
            yield value;
        }
    };
    const output = pipeline(() => iterable, buffer(1), stopOnError, map(wrapFunc), buffer(concurrency - 1));
    const itr = getIterator(output);
    while (true) {
        const { value, done } = await itr.next();
        if (done) {
            break;
        }
        try {
            const val = await value.value;
            if (!transformError) {
                yield val;
            }
        }
        catch (error) {
            transformError = error;
        }
    }
    if (transformError) {
        throw transformError;
    }
}
function parallelMap(concurrency, func, iterable) {
    if (func === undefined) {
        return (curriedFunc, curriedIterable) => parallelMap(concurrency, curriedFunc, curriedIterable);
    }
    if (iterable === undefined) {
        return curriedIterable => parallelMap(concurrency, func, curriedIterable);
    }
    if (concurrency === 1) {
        return map(func, iterable);
    }
    return _parallelMap(concurrency, func, iterable);
}

function parallelFlatMap(concurrency, func, iterable) {
    if (func === undefined) {
        return (curriedFunc, curriedIterable) => curriedIterable
            ? parallelFlatMap(concurrency, curriedFunc, curriedIterable)
            : parallelFlatMap(concurrency, curriedFunc);
    }
    if (iterable === undefined) {
        return (curriedIterable) => parallelFlatMap(concurrency, func, curriedIterable);
    }
    return filter(i => i !== undefined && i !== null, flatten(parallelMap(concurrency, func, iterable)));
}

/// <reference lib="esnext.asynciterable" />
async function* parallelMerge(...iterables) {
    const inputs = iterables.map(getIterator);
    const concurrentWork = new Set();
    const values = new Map();
    let lastError = null;
    let errCb = null;
    let valueCb = null;
    const notifyError = err => {
        lastError = err;
        if (errCb) {
            errCb(err);
        }
    };
    const notifyDone = value => {
        if (valueCb) {
            valueCb(value);
        }
    };
    const waitForQueue = () => new Promise((resolve, reject) => {
        if (lastError) {
            reject(lastError);
        }
        if (values.size > 0) {
            return resolve();
        }
        valueCb = resolve;
        errCb = reject;
    });
    const queueNext = input => {
        const nextVal = Promise.resolve(input.next()).then(async ({ done, value }) => {
            if (!done) {
                values.set(input, value);
            }
            concurrentWork.delete(nextVal);
        });
        concurrentWork.add(nextVal);
        nextVal.then(notifyDone, notifyError);
    };
    for (const input of inputs) {
        queueNext(input);
    }
    while (true) {
        // We technically don't have to check `values.size` as the for loop should have emptied it
        // However I haven't yet found specs verifying that behavior, only tests
        // the guard in waitForQueue() checking for values is in place for the same reason
        if (concurrentWork.size === 0 && values.size === 0) {
            return;
        }
        await waitForQueue();
        for (const [input, value] of values) {
            values.delete(input);
            yield value;
            queueNext(input);
        }
    }
}

async function _reduce(func, start, iterable) {
    let value = start;
    for await (const nextItem of iterable) {
        value = await func(value, nextItem);
    }
    return value;
}
function reduce(func, start, iterable) {
    if (start === undefined) {
        return (curriedStart, curriedIterable) => curriedIterable ? _reduce(func, curriedStart, curriedIterable) : reduce(func, curriedStart);
    }
    if (iterable === undefined) {
        return (curriedIterable) => reduce(func, start, curriedIterable);
    }
    return _reduce(func, start, iterable);
}

async function* _take(count, iterable) {
    let taken = 0;
    for await (const val of iterable) {
        yield await val;
        taken++;
        if (taken >= count) {
            break;
        }
    }
}
function* _syncTake(count, iterable) {
    let taken = 0;
    for (const val of iterable) {
        yield val;
        taken++;
        if (taken >= count) {
            break;
        }
    }
}
function take(count, iterable) {
    if (iterable === undefined) {
        return curriedIterable => take(count, curriedIterable);
    }
    if (iterable[Symbol.asyncIterator]) {
        return _take(count, iterable);
    }
    return _syncTake(count, iterable);
}

async function* _asyncTap(func, iterable) {
    for await (const val of iterable) {
        await func(val);
        yield val;
    }
}
function tap(func, iterable) {
    if (iterable === undefined) {
        return (curriedIterable) => _asyncTap(func, curriedIterable);
    }
    return _asyncTap(func, iterable);
}

function addTime(a, b) {
    let seconds = a[0] + b[0];
    let nanoseconds = a[1] + b[1];
    if (nanoseconds >= 1000000000) {
        const remainder = nanoseconds % 1000000000;
        seconds += (nanoseconds - remainder) / 1000000000;
        nanoseconds = remainder;
    }
    return [seconds, nanoseconds];
}
async function* _asyncTime(config, iterable) {
    const itr = iterable[Symbol.asyncIterator]();
    let total = [0, 0];
    while (true) {
        const start = process.hrtime();
        const { value, done } = await itr.next();
        const delta = process.hrtime(start);
        total = addTime(total, delta);
        if (config.progress) {
            config.progress(delta, total);
        }
        if (done) {
            if (config.total) {
                config.total(total);
            }
            return value;
        }
        yield value;
    }
}
function* _syncTime(config, iterable) {
    const itr = iterable[Symbol.iterator]();
    let total = [0, 0];
    while (true) {
        const start = process.hrtime();
        const { value, done } = itr.next();
        const delta = process.hrtime(start);
        total = addTime(total, delta);
        if (config.progress) {
            config.progress(delta, total);
        }
        if (done) {
            if (config.total) {
                config.total(total);
            }
            return value;
        }
        yield value;
    }
}
function time(config = {}, iterable) {
    if (iterable === undefined) {
        return curriedIterable => time(config, curriedIterable);
    }
    if (iterable[Symbol.asyncIterator] !== undefined) {
        return _asyncTime(config, iterable);
    }
    else {
        return _syncTime(config, iterable);
    }
}

function _transform(concurrency, func, iterable) {
    const iterator = getIterator(iterable);
    const resultQueue = [];
    const readQueue = [];
    let ended = false;
    let reading = false;
    let inflightCount = 0;
    let lastError = null;
    function fulfillReadQueue() {
        while (readQueue.length > 0 && resultQueue.length > 0) {
            const { resolve } = readQueue.shift();
            const value = resultQueue.shift();
            resolve({ done: false, value });
        }
        while (readQueue.length > 0 && inflightCount === 0 && ended) {
            const { resolve, reject } = readQueue.shift();
            if (lastError) {
                reject(lastError);
                lastError = null;
            }
            else {
                resolve({ done: true, value: undefined });
            }
        }
    }
    async function fillQueue() {
        if (ended) {
            fulfillReadQueue();
            return;
        }
        if (reading) {
            return;
        }
        if (inflightCount + resultQueue.length >= concurrency) {
            return;
        }
        reading = true;
        inflightCount++;
        try {
            const { done, value } = await iterator.next();
            if (done) {
                ended = true;
                inflightCount--;
                fulfillReadQueue();
            }
            else {
                mapAndQueue(value);
            }
        }
        catch (error) {
            ended = true;
            inflightCount--;
            lastError = error;
            fulfillReadQueue();
        }
        reading = false;
        fillQueue();
    }
    async function mapAndQueue(itrValue) {
        try {
            const value = await func(itrValue);
            resultQueue.push(value);
        }
        catch (error) {
            ended = true;
            lastError = error;
        }
        inflightCount--;
        fulfillReadQueue();
        fillQueue();
    }
    async function next() {
        if (resultQueue.length === 0) {
            const deferred = defer();
            readQueue.push(deferred);
            fillQueue();
            return deferred.promise;
        }
        const value = resultQueue.shift();
        fillQueue();
        return { done: false, value };
    }
    const asyncIterableIterator = {
        next,
        [Symbol.asyncIterator]: () => asyncIterableIterator,
    };
    return asyncIterableIterator;
}
function transform(concurrency, func, iterable) {
    if (func === undefined) {
        return (curriedFunc, curriedIterable) => curriedIterable
            ? transform(concurrency, curriedFunc, curriedIterable)
            : transform(concurrency, curriedFunc);
    }
    if (iterable === undefined) {
        return (curriedIterable) => transform(concurrency, func, curriedIterable);
    }
    return _transform(concurrency, func, iterable);
}

async function _writeToStream(stream, iterable) {
    let lastError = null;
    let errCb = null;
    let drainCb = null;
    const notifyError = err => {
        lastError = err;
        if (errCb) {
            errCb(err);
        }
    };
    const notifyDrain = () => {
        if (drainCb) {
            drainCb();
        }
    };
    const cleanup = () => {
        stream.removeListener('error', notifyError);
        stream.removeListener('drain', notifyDrain);
    };
    stream.once('error', notifyError);
    const waitForDrain = () => new Promise((resolve, reject) => {
        if (lastError) {
            return reject(lastError);
        }
        stream.once('drain', notifyDrain);
        drainCb = resolve;
        errCb = reject;
    });
    for await (const value of iterable) {
        if (stream.write(value) === false) {
            await waitForDrain();
        }
        if (lastError) {
            break;
        }
    }
    cleanup();
    if (lastError) {
        throw lastError;
    }
}
function writeToStream(stream, iterable) {
    if (iterable === undefined) {
        return (curriedIterable) => _writeToStream(stream, curriedIterable);
    }
    return _writeToStream(stream, iterable);
}

export { batch, buffer, collect, concat, consume, filter, flatMap, flatTransform, flatten, fromStream, getIterator, map, merge, parallelFlatMap, parallelMap, parallelMerge, pipeline, reduce, take, tap, time, transform, writeToStream };