@loaders.gl/json/dist/lib/parsers/parse-ndjson-in-batches.js

Framework-independent loader for JSON and streaming JSON formats

// loaders.gl
// SPDX-License-Identifier: MIT
// Copyright (c) vis.gl contributors
import { TableBatchBuilder } from '@loaders.gl/schema';
import { makeLineIterator, makeNumberedLineIterator, makeTextDecoderIterator } from '@loaders.gl/loader-utils';
export async function* parseNDJSONInBatches(binaryAsyncIterator, options) {
    const textIterator = makeTextDecoderIterator(binaryAsyncIterator);
    const lineIterator = makeLineIterator(textIterator);
    const numberedLineIterator = makeNumberedLineIterator(lineIterator);
    const schema = null;
    const shape = 'row-table';
    // @ts-ignore
    const tableBatchBuilder = new TableBatchBuilder(schema, {
        ...options,
        shape
    });
    for await (const { counter, line } of numberedLineIterator) {
        try {
            const row = JSON.parse(line);
            tableBatchBuilder.addRow(row);
            tableBatchBuilder.chunkComplete(line);
            const batch = tableBatchBuilder.getFullBatch();
            if (batch) {
                yield batch;
            }
        }
        catch (error) {
            throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);
        }
    }
    const batch = tableBatchBuilder.getFinalBatch();
    if (batch) {
        yield batch;
    }
}