@loaders.gl/json
Version:
Framework-independent loader for JSON and streaming JSON formats
1,644 lines (1,623 loc) • 96.6 kB
JavaScript
(function webpackUniversalModuleDefinition(root, factory) {
if (typeof exports === 'object' && typeof module === 'object')
module.exports = factory();
else if (typeof define === 'function' && define.amd) define([], factory);
else if (typeof exports === 'object') exports['loaders'] = factory();
else root['loaders'] = factory();})(globalThis, function () {
"use strict";
var __exports__ = (() => {
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
var __commonJS = (cb, mod) => function __require() {
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
};
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var __publicField = (obj, key, value) => {
__defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
return value;
};
// external-global-plugin:@loaders.gl/core
var require_core = __commonJS({
"external-global-plugin:@loaders.gl/core"(exports, module) {
module.exports = globalThis.loaders;
}
});
// bundle.ts
var bundle_exports = {};
__export(bundle_exports, {
JSONLoader: () => JSONLoader,
JSONWriter: () => JSONWriter,
NDJSONLoader: () => NDJSONLoader,
_ClarinetParser: () => ClarinetParser,
_GeoJSONLoader: () => GeoJSONLoader,
_GeoJSONWorkerLoader: () => GeoJSONWorkerLoader,
_GeoJSONWriter: () => GeoJSONWriter,
_JSONPath: () => JSONPath,
_rebuildJsonObject: () => rebuildJsonObject
});
__reExport(bundle_exports, __toESM(require_core(), 1));
// ../schema/src/lib/table/simple-table/data-type.ts
function getDataTypeFromValue(value, defaultNumberType = "float32") {
if (value instanceof Date) {
return "date-millisecond";
}
if (value instanceof Number) {
return defaultNumberType;
}
if (typeof value === "string") {
return "utf8";
}
if (value === null || value === "undefined") {
return "null";
}
return "null";
}
function getDataTypeFromArray(array) {
let type = getDataTypeFromTypedArray(array);
if (type !== "null") {
return { type, nullable: false };
}
if (array.length > 0) {
type = getDataTypeFromValue(array[0]);
return { type, nullable: true };
}
return { type: "null", nullable: true };
}
function getDataTypeFromTypedArray(array) {
switch (array.constructor) {
case Int8Array:
return "int8";
case Uint8Array:
case Uint8ClampedArray:
return "uint8";
case Int16Array:
return "int16";
case Uint16Array:
return "uint16";
case Int32Array:
return "int32";
case Uint32Array:
return "uint32";
case Float32Array:
return "float32";
case Float64Array:
return "float64";
default:
return "null";
}
}
// ../schema/src/lib/table/batches/base-table-batch-aggregator.ts
var DEFAULT_ROW_COUNT = 100;
var BaseTableBatchAggregator = class {
schema;
options;
shape;
length = 0;
rows = null;
cursor = 0;
_headers = [];
constructor(schema, options) {
this.options = options;
this.schema = schema;
if (!Array.isArray(schema)) {
this._headers = [];
for (const key in schema) {
this._headers[schema[key].index] = schema[key].name;
}
}
}
rowCount() {
return this.length;
}
addArrayRow(row, cursor) {
if (Number.isFinite(cursor)) {
this.cursor = cursor;
}
this.shape = "array-row-table";
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
this.rows[this.length] = row;
this.length++;
}
addObjectRow(row, cursor) {
if (Number.isFinite(cursor)) {
this.cursor = cursor;
}
this.shape = "object-row-table";
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
this.rows[this.length] = row;
this.length++;
}
getBatch() {
let rows = this.rows;
if (!rows) {
return null;
}
rows = rows.slice(0, this.length);
this.rows = null;
const batch = {
shape: this.shape || "array-row-table",
batchType: "data",
data: rows,
length: this.length,
schema: this.schema,
cursor: this.cursor
};
return batch;
}
};
// ../schema/src/lib/table/simple-table/row-utils.ts
function convertToObjectRow(arrayRow, headers) {
if (!arrayRow) {
throw new Error("null row");
}
const objectRow = {};
if (headers) {
for (let i = 0; i < headers.length; i++) {
objectRow[headers[i]] = arrayRow[i];
}
} else {
for (let i = 0; i < arrayRow.length; i++) {
const columnName = `column-${i}`;
objectRow[columnName] = arrayRow[i];
}
}
return objectRow;
}
function convertToArrayRow(objectRow, headers) {
if (!objectRow) {
throw new Error("null row");
}
if (headers) {
const arrayRow = new Array(headers.length);
for (let i = 0; i < headers.length; i++) {
arrayRow[i] = objectRow[headers[i]];
}
return arrayRow;
}
return Object.values(objectRow);
}
function inferHeadersFromArrayRow(arrayRow) {
const headers = [];
for (let i = 0; i < arrayRow.length; i++) {
const columnName = `column-${i}`;
headers.push(columnName);
}
return headers;
}
function inferHeadersFromObjectRow(row) {
return Object.keys(row);
}
// ../schema/src/lib/table/batches/row-table-batch-aggregator.ts
var DEFAULT_ROW_COUNT2 = 100;
var RowTableBatchAggregator = class {
schema;
options;
length = 0;
objectRows = null;
arrayRows = null;
cursor = 0;
_headers = null;
constructor(schema, options) {
this.options = options;
this.schema = schema;
if (schema) {
this._headers = [];
for (const key in schema) {
this._headers[schema[key].index] = schema[key].name;
}
}
}
rowCount() {
return this.length;
}
addArrayRow(row, cursor) {
if (Number.isFinite(cursor)) {
this.cursor = cursor;
}
this._headers ||= inferHeadersFromArrayRow(row);
switch (this.options.shape) {
case "object-row-table":
const rowObject = convertToObjectRow(row, this._headers);
this.addObjectRow(rowObject, cursor);
break;
case "array-row-table":
this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT2);
this.arrayRows[this.length] = row;
this.length++;
break;
}
}
addObjectRow(row, cursor) {
if (Number.isFinite(cursor)) {
this.cursor = cursor;
}
this._headers ||= inferHeadersFromObjectRow(row);
switch (this.options.shape) {
case "array-row-table":
const rowArray = convertToArrayRow(row, this._headers);
this.addArrayRow(rowArray, cursor);
break;
case "object-row-table":
this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT2);
this.objectRows[this.length] = row;
this.length++;
break;
}
}
getBatch() {
let rows = this.arrayRows || this.objectRows;
if (!rows) {
return null;
}
rows = rows.slice(0, this.length);
this.arrayRows = null;
this.objectRows = null;
return {
shape: this.options.shape,
batchType: "data",
data: rows,
length: this.length,
// @ts-expect-error we should infer a schema
schema: this.schema,
cursor: this.cursor
};
}
};
// ../schema/src/lib/table/batches/columnar-table-batch-aggregator.ts
var DEFAULT_ROW_COUNT3 = 100;
var ColumnarTableBatchAggregator = class {
schema;
length = 0;
allocated = 0;
columns = {};
constructor(schema, options) {
this.schema = schema;
this._reallocateColumns();
}
rowCount() {
return this.length;
}
addArrayRow(row) {
this._reallocateColumns();
let i = 0;
for (const fieldName in this.columns) {
this.columns[fieldName][this.length] = row[i++];
}
this.length++;
}
addObjectRow(row) {
this._reallocateColumns();
for (const fieldName in row) {
this.columns[fieldName][this.length] = row[fieldName];
}
this.length++;
}
getBatch() {
this._pruneColumns();
const columns = Array.isArray(this.schema) ? this.columns : {};
if (!Array.isArray(this.schema)) {
for (const fieldName in this.schema) {
const field = this.schema[fieldName];
columns[field.name] = this.columns[field.index];
}
}
this.columns = {};
const batch = {
shape: "columnar-table",
batchType: "data",
data: columns,
schema: this.schema,
length: this.length
};
return batch;
}
// HELPERS
_reallocateColumns() {
if (this.length < this.allocated) {
return;
}
this.allocated = this.allocated > 0 ? this.allocated *= 2 : DEFAULT_ROW_COUNT3;
this.columns = {};
for (const fieldName in this.schema) {
const field = this.schema[fieldName];
const ArrayType = field.type || Float32Array;
const oldColumn = this.columns[field.index];
if (oldColumn && ArrayBuffer.isView(oldColumn)) {
const typedArray = new ArrayType(this.allocated);
typedArray.set(oldColumn);
this.columns[field.index] = typedArray;
} else if (oldColumn) {
oldColumn.length = this.allocated;
this.columns[field.index] = oldColumn;
} else {
this.columns[field.index] = new ArrayType(this.allocated);
}
}
}
_pruneColumns() {
for (const [columnName, column] of Object.entries(this.columns)) {
this.columns[columnName] = column.slice(0, this.length);
}
}
};
// ../schema/src/lib/table/batches/table-batch-builder.ts
var DEFAULT_OPTIONS = {
shape: void 0,
batchSize: "auto",
batchDebounceMs: 0,
limit: 0,
_limitMB: 0
};
var ERR_MESSAGE = "TableBatchBuilder";
var _TableBatchBuilder = class {
schema;
options;
aggregator = null;
batchCount = 0;
bytesUsed = 0;
isChunkComplete = false;
lastBatchEmittedMs = Date.now();
totalLength = 0;
totalBytes = 0;
rowBytes = 0;
constructor(schema, options) {
this.schema = schema;
this.options = { ...DEFAULT_OPTIONS, ...options };
}
limitReached() {
if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {
return true;
}
if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
return true;
}
return false;
}
/** @deprecated Use addArrayRow or addObjectRow */
addRow(row) {
if (this.limitReached()) {
return;
}
this.totalLength++;
this.rowBytes = this.rowBytes || this._estimateRowMB(row);
this.totalBytes += this.rowBytes;
if (Array.isArray(row)) {
this.addArrayRow(row);
} else {
this.addObjectRow(row);
}
}
/** Add one row to the batch */
addArrayRow(row) {
if (!this.aggregator) {
const TableBatchType = this._getTableBatchType();
this.aggregator = new TableBatchType(this.schema, this.options);
}
this.aggregator.addArrayRow(row);
}
/** Add one row to the batch */
addObjectRow(row) {
if (!this.aggregator) {
const TableBatchType = this._getTableBatchType();
this.aggregator = new TableBatchType(this.schema, this.options);
}
this.aggregator.addObjectRow(row);
}
/** Mark an incoming raw memory chunk has completed */
chunkComplete(chunk) {
if (chunk instanceof ArrayBuffer) {
this.bytesUsed += chunk.byteLength;
}
if (typeof chunk === "string") {
this.bytesUsed += chunk.length;
}
this.isChunkComplete = true;
}
getFullBatch(options) {
return this._isFull() ? this._getBatch(options) : null;
}
getFinalBatch(options) {
return this._getBatch(options);
}
// INTERNAL
_estimateRowMB(row) {
return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
}
_isFull() {
if (!this.aggregator || this.aggregator.rowCount() === 0) {
return false;
}
if (this.options.batchSize === "auto") {
if (!this.isChunkComplete) {
return false;
}
} else if (this.options.batchSize > this.aggregator.rowCount()) {
return false;
}
if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
return false;
}
this.isChunkComplete = false;
this.lastBatchEmittedMs = Date.now();
return true;
}
/**
* bytesUsed can be set via chunkComplete or via getBatch*
*/
_getBatch(options) {
if (!this.aggregator) {
return null;
}
if (options?.bytesUsed) {
this.bytesUsed = options.bytesUsed;
}
const normalizedBatch = this.aggregator.getBatch();
normalizedBatch.count = this.batchCount;
normalizedBatch.bytesUsed = this.bytesUsed;
Object.assign(normalizedBatch, options);
this.batchCount++;
this.aggregator = null;
return normalizedBatch;
}
_getTableBatchType() {
switch (this.options.shape) {
case "array-row-table":
case "object-row-table":
return RowTableBatchAggregator;
case "columnar-table":
return ColumnarTableBatchAggregator;
case "arrow-table":
if (!_TableBatchBuilder.ArrowBatch) {
throw new Error(ERR_MESSAGE);
}
return _TableBatchBuilder.ArrowBatch;
default:
return BaseTableBatchAggregator;
}
}
};
var TableBatchBuilder = _TableBatchBuilder;
__publicField(TableBatchBuilder, "ArrowBatch");
// ../schema/src/lib/table/simple-table/table-accessors.ts
function getTableLength(table) {
switch (table.shape) {
case "array-row-table":
case "object-row-table":
return table.data.length;
case "geojson-table":
return table.features.length;
case "arrow-table":
const arrowTable = table.data;
return arrowTable.numRows;
case "columnar-table":
for (const column of Object.values(table.data)) {
return column.length || 0;
}
return 0;
default:
throw new Error("table");
}
}
function getTableNumCols(table) {
if (table.schema) {
return table.schema.fields.length;
}
if (getTableLength(table) === 0) {
throw new Error("empty table");
}
switch (table.shape) {
case "array-row-table":
return table.data[0].length;
case "object-row-table":
return Object.keys(table.data[0]).length;
case "geojson-table":
return Object.keys(table.features[0]).length;
case "columnar-table":
return Object.keys(table.data).length;
case "arrow-table":
const arrowTable = table.data;
return arrowTable.numCols;
default:
throw new Error("table");
}
}
function getTableRowAsObject(table, rowIndex, target, copy2) {
switch (table.shape) {
case "object-row-table":
return copy2 ? Object.fromEntries(Object.entries(table.data[rowIndex])) : table.data[rowIndex];
case "array-row-table":
if (table.schema) {
const objectRow2 = target || {};
for (let i = 0; i < table.schema.fields.length; i++) {
objectRow2[table.schema.fields[i].name] = table.data[rowIndex][i];
}
return objectRow2;
}
throw new Error("no schema");
case "geojson-table":
if (table.schema) {
const objectRow2 = target || {};
for (let i = 0; i < table.schema.fields.length; i++) {
objectRow2[table.schema.fields[i].name] = table.features[rowIndex][i];
}
return objectRow2;
}
throw new Error("no schema");
case "columnar-table":
if (table.schema) {
const objectRow2 = target || {};
for (let i = 0; i < table.schema.fields.length; i++) {
objectRow2[table.schema.fields[i].name] = table.data[table.schema.fields[i].name][rowIndex];
}
return objectRow2;
} else {
const objectRow2 = target || {};
for (const [name, column] of Object.entries(table.data)) {
objectRow2[name] = column[rowIndex];
}
return objectRow2;
}
case "arrow-table":
const arrowTable = table.data;
const objectRow = target || {};
const row = arrowTable.get(rowIndex);
const schema = arrowTable.schema;
for (let i = 0; i < schema.fields.length; i++) {
objectRow[schema.fields[i].name] = row?.[schema.fields[i].name];
}
return objectRow;
default:
throw new Error("shape");
}
}
function getTableRowAsArray(table, rowIndex, target, copy2) {
switch (table.shape) {
case "array-row-table":
return copy2 ? Array.from(table.data[rowIndex]) : table.data[rowIndex];
case "object-row-table":
if (table.schema) {
const arrayRow2 = target || [];
for (let i = 0; i < table.schema.fields.length; i++) {
arrayRow2[i] = table.data[rowIndex][table.schema.fields[i].name];
}
return arrayRow2;
}
return Object.values(table.data[rowIndex]);
case "geojson-table":
if (table.schema) {
const arrayRow2 = target || [];
for (let i = 0; i < table.schema.fields.length; i++) {
arrayRow2[i] = table.features[rowIndex][table.schema.fields[i].name];
}
return arrayRow2;
}
return Object.values(table.features[rowIndex]);
case "columnar-table":
if (table.schema) {
const arrayRow2 = target || [];
for (let i = 0; i < table.schema.fields.length; i++) {
arrayRow2[i] = table.data[table.schema.fields[i].name][rowIndex];
}
return arrayRow2;
} else {
const arrayRow2 = target || [];
let i = 0;
for (const column of Object.values(table.data)) {
arrayRow2[i] = column[rowIndex];
i++;
}
return arrayRow2;
}
case "arrow-table":
const arrowTable = table.data;
const arrayRow = target || [];
const row = arrowTable.get(rowIndex);
const schema = arrowTable.schema;
for (let i = 0; i < schema.fields.length; i++) {
arrayRow[i] = row?.[schema.fields[i].name];
}
return arrayRow;
default:
throw new Error("shape");
}
}
function* makeRowIterator(table, shape) {
switch (shape) {
case "array-row-table":
yield* makeArrayRowIterator(table);
break;
case "object-row-table":
yield* makeObjectRowIterator(table);
break;
default:
throw new Error(`Unknown row type ${shape}`);
}
}
function* makeArrayRowIterator(table, target = []) {
const length = getTableLength(table);
for (let rowIndex = 0; rowIndex < length; rowIndex++) {
yield getTableRowAsArray(table, rowIndex, target);
}
}
function* makeObjectRowIterator(table, target = {}) {
const length = getTableLength(table);
for (let rowIndex = 0; rowIndex < length; rowIndex++) {
yield getTableRowAsObject(table, rowIndex, target);
}
}
// ../schema/src/lib/table/simple-table/table-schema.ts
function deduceTableSchema(table) {
switch (table.shape) {
case "array-row-table":
case "object-row-table":
return deduceSchemaFromRows(table.data);
case "geojson-table":
return deduceSchemaFromGeoJSON(table.features);
case "columnar-table":
return deduceSchemaFromColumns(table.data);
case "arrow-table":
default:
throw new Error("Deduce schema");
}
}
function deduceSchemaFromColumns(columnarTable) {
const fields = [];
for (const [columnName, column] of Object.entries(columnarTable)) {
const field = deduceFieldFromColumn(column, columnName);
fields.push(field);
}
return { fields, metadata: {} };
}
function deduceSchemaFromRows(rowTable) {
if (!rowTable.length) {
throw new Error("deduce from empty table");
}
const fields = [];
const row0 = rowTable[0];
for (const [columnName, value] of Object.entries(row0)) {
fields.push(deduceFieldFromValue(value, columnName));
}
return { fields, metadata: {} };
}
function deduceSchemaFromGeoJSON(features) {
if (!features.length) {
throw new Error("deduce from empty table");
}
const fields = [];
const row0 = features[0].properties || {};
for (const [columnName, value] of Object.entries(row0)) {
fields.push(deduceFieldFromValue(value, columnName));
}
return { fields, metadata: {} };
}
function deduceFieldFromColumn(column, name) {
if (ArrayBuffer.isView(column)) {
const type = getDataTypeFromArray(column);
return {
name,
type: type.type || "null",
nullable: type.nullable
// metadata: {}
};
}
if (Array.isArray(column) && column.length > 0) {
const value = column[0];
const type = getDataTypeFromValue(value);
return {
name,
type,
nullable: true
// metadata: {},
};
}
throw new Error("empty table");
}
function deduceFieldFromValue(value, name) {
const type = getDataTypeFromValue(value);
return {
name,
type,
nullable: true
// metadata: {}
};
}
// ../schema/src/lib/table/simple-table/make-table.ts
function makeTableFromData(data) {
let table;
switch (getTableShapeFromData(data)) {
case "array-row-table":
table = { shape: "array-row-table", data };
break;
case "object-row-table":
table = { shape: "object-row-table", data };
break;
case "columnar-table":
table = { shape: "columnar-table", data };
break;
default:
throw new Error("table");
}
const schema = deduceTableSchema(table);
return { ...table, schema };
}
function getTableShapeFromData(data) {
if (Array.isArray(data)) {
if (data.length === 0) {
throw new Error("cannot deduce type of empty table");
}
const firstRow = data[0];
if (Array.isArray(firstRow)) {
return "array-row-table";
}
if (firstRow && typeof firstRow === "object") {
return "object-row-table";
}
}
if (data && typeof data === "object") {
return "columnar-table";
}
throw new Error("invalid table");
}
// src/lib/parsers/parse-json.ts
function parseJSONSync(jsonText, options) {
try {
const json = JSON.parse(jsonText);
if (options.json?.table) {
const data = getFirstArray(json) || json;
return makeTableFromData(data);
}
return json;
} catch (error) {
throw new Error("JSONLoader: failed to parse JSON");
}
}
function getFirstArray(json) {
if (Array.isArray(json)) {
return json;
}
if (json && typeof json === "object") {
for (const value of Object.values(json)) {
const array = getFirstArray(value);
if (array) {
return array;
}
}
}
return null;
}
// ../loader-utils/src/lib/env-utils/assert.ts
function assert(condition, message) {
if (!condition) {
throw new Error(message || "loader assertion failed.");
}
}
// ../loader-utils/src/lib/binary-utils/array-buffer-utils.ts
function concatenateArrayBuffers(...sources) {
return concatenateArrayBuffersFromArray(sources);
}
function concatenateArrayBuffersFromArray(sources) {
const sourceArrays = sources.map(
(source2) => source2 instanceof ArrayBuffer ? new Uint8Array(source2) : source2
);
const byteLength = sourceArrays.reduce((length, typedArray) => length + typedArray.byteLength, 0);
const result = new Uint8Array(byteLength);
let offset = 0;
for (const sourceArray of sourceArrays) {
result.set(sourceArray, offset);
offset += sourceArray.byteLength;
}
return result.buffer;
}
// ../loader-utils/src/lib/iterators/text-iterators.ts
async function* makeTextDecoderIterator(arrayBufferIterator, options = {}) {
const textDecoder = new TextDecoder(void 0, options);
for await (const arrayBuffer of arrayBufferIterator) {
yield typeof arrayBuffer === "string" ? arrayBuffer : textDecoder.decode(arrayBuffer, { stream: true });
}
}
async function* makeLineIterator(textIterator) {
let previous = "";
for await (const textChunk of textIterator) {
previous += textChunk;
let eolIndex;
while ((eolIndex = previous.indexOf("\n")) >= 0) {
const line = previous.slice(0, eolIndex + 1);
previous = previous.slice(eolIndex + 1);
yield line;
}
}
if (previous.length > 0) {
yield previous;
}
}
async function* makeNumberedLineIterator(lineIterator) {
let counter = 1;
for await (const line of lineIterator) {
yield { counter, line };
counter++;
}
}
// ../loader-utils/src/lib/iterators/async-iteration.ts
async function concatenateArrayBuffersAsync(asyncIterator) {
const arrayBuffers = [];
for await (const chunk of asyncIterator) {
arrayBuffers.push(chunk);
}
return concatenateArrayBuffers(...arrayBuffers);
}
// src/lib/clarinet/clarinet.ts
var MAX_BUFFER_LENGTH = Number.MAX_SAFE_INTEGER;
var Char = {
tab: 9,
// \t
lineFeed: 10,
// \n
carriageReturn: 13,
// \r
space: 32,
// " "
doubleQuote: 34,
// "
plus: 43,
// +
comma: 44,
// ,
minus: 45,
// -
period: 46,
// .
_0: 48,
// 0
_9: 57,
// 9
colon: 58,
// :
E: 69,
// E
openBracket: 91,
// [
backslash: 92,
// \
closeBracket: 93,
// ]
a: 97,
// a
b: 98,
// b
e: 101,
// e
f: 102,
// f
l: 108,
// l
n: 110,
// n
r: 114,
// r
s: 115,
// s
t: 116,
// t
u: 117,
// u
openBrace: 123,
// {
closeBrace: 125
// }
};
var stringTokenPattern = /[\\"\n]/g;
var DEFAULT_OPTIONS2 = {
onready: () => {
},
onopenobject: () => {
},
onkey: () => {
},
oncloseobject: () => {
},
onopenarray: () => {
},
onclosearray: () => {
},
onvalue: () => {
},
onerror: () => {
},
onend: () => {
},
onchunkparsed: () => {
}
};
var ClarinetParser = class {
options = DEFAULT_OPTIONS2;
bufferCheckPosition = MAX_BUFFER_LENGTH;
q = "";
c = "";
p = "";
closed = false;
closedRoot = false;
sawRoot = false;
// tag = null;
error = null;
state = 0 /* BEGIN */;
stack = [];
// mostly just for error reporting
position = 0;
column = 0;
line = 1;
slashed = false;
unicodeI = 0;
unicodeS = null;
depth = 0;
textNode;
numberNode;
constructor(options = {}) {
this.options = { ...DEFAULT_OPTIONS2, ...options };
this.textNode = void 0;
this.numberNode = "";
this.emit("onready");
}
end() {
if (this.state !== 1 /* VALUE */ || this.depth !== 0)
this._error("Unexpected end");
this._closeValue();
this.c = "";
this.closed = true;
this.emit("onend");
return this;
}
resume() {
this.error = null;
return this;
}
close() {
return this.write(null);
}
// protected
emit(event, data) {
this.options[event]?.(data, this);
}
emitNode(event, data) {
this._closeValue();
this.emit(event, data);
}
/* eslint-disable no-continue */
// eslint-disable-next-line complexity, max-statements
write(chunk) {
if (this.error) {
throw this.error;
}
if (this.closed) {
return this._error("Cannot write after close. Assign an onready handler.");
}
if (chunk === null) {
return this.end();
}
let i = 0;
let c = chunk.charCodeAt(0);
let p = this.p;
while (c) {
p = c;
this.c = c = chunk.charCodeAt(i++);
if (p !== c) {
this.p = p;
} else {
p = this.p;
}
if (!c)
break;
this.position++;
if (c === Char.lineFeed) {
this.line++;
this.column = 0;
} else
this.column++;
switch (this.state) {
case 0 /* BEGIN */:
if (c === Char.openBrace)
this.state = 2 /* OPEN_OBJECT */;
else if (c === Char.openBracket)
this.state = 4 /* OPEN_ARRAY */;
else if (!isWhitespace(c)) {
this._error("Non-whitespace before {[.");
}
continue;
case 10 /* OPEN_KEY */:
case 2 /* OPEN_OBJECT */:
if (isWhitespace(c))
continue;
if (this.state === 10 /* OPEN_KEY */)
this.stack.push(11 /* CLOSE_KEY */);
else if (c === Char.closeBrace) {
this.emit("onopenobject");
this.depth++;
this.emit("oncloseobject");
this.depth--;
this.state = this.stack.pop() || 1 /* VALUE */;
continue;
} else
this.stack.push(3 /* CLOSE_OBJECT */);
if (c === Char.doubleQuote)
this.state = 7 /* STRING */;
else
this._error('Malformed object key should start with "');
continue;
case 11 /* CLOSE_KEY */:
case 3 /* CLOSE_OBJECT */:
if (isWhitespace(c))
continue;
if (c === Char.colon) {
if (this.state === 3 /* CLOSE_OBJECT */) {
this.stack.push(3 /* CLOSE_OBJECT */);
this._closeValue("onopenobject");
this.depth++;
} else
this._closeValue("onkey");
this.state = 1 /* VALUE */;
} else if (c === Char.closeBrace) {
this.emitNode("oncloseobject");
this.depth--;
this.state = this.stack.pop() || 1 /* VALUE */;
} else if (c === Char.comma) {
if (this.state === 3 /* CLOSE_OBJECT */)
this.stack.push(3 /* CLOSE_OBJECT */);
this._closeValue();
this.state = 10 /* OPEN_KEY */;
} else
this._error("Bad object");
continue;
case 4 /* OPEN_ARRAY */:
case 1 /* VALUE */:
if (isWhitespace(c))
continue;
if (this.state === 4 /* OPEN_ARRAY */) {
this.emit("onopenarray");
this.depth++;
this.state = 1 /* VALUE */;
if (c === Char.closeBracket) {
this.emit("onclosearray");
this.depth--;
this.state = this.stack.pop() || 1 /* VALUE */;
continue;
} else {
this.stack.push(5 /* CLOSE_ARRAY */);
}
}
if (c === Char.doubleQuote)
this.state = 7 /* STRING */;
else if (c === Char.openBrace)
this.state = 2 /* OPEN_OBJECT */;
else if (c === Char.openBracket)
this.state = 4 /* OPEN_ARRAY */;
else if (c === Char.t)
this.state = 12 /* TRUE */;
else if (c === Char.f)
this.state = 15 /* FALSE */;
else if (c === Char.n)
this.state = 19 /* NULL */;
else if (c === Char.minus) {
this.numberNode += "-";
} else if (Char._0 <= c && c <= Char._9) {
this.numberNode += String.fromCharCode(c);
this.state = 23 /* NUMBER_DIGIT */;
} else
this._error("Bad value");
continue;
case 5 /* CLOSE_ARRAY */:
if (c === Char.comma) {
this.stack.push(5 /* CLOSE_ARRAY */);
this._closeValue("onvalue");
this.state = 1 /* VALUE */;
} else if (c === Char.closeBracket) {
this.emitNode("onclosearray");
this.depth--;
this.state = this.stack.pop() || 1 /* VALUE */;
} else if (isWhitespace(c))
continue;
else
this._error("Bad array");
continue;
case 7 /* STRING */:
if (this.textNode === void 0) {
this.textNode = "";
}
let starti = i - 1;
let slashed = this.slashed;
let unicodeI = this.unicodeI;
STRING_BIGLOOP:
while (true) {
while (unicodeI > 0) {
this.unicodeS += String.fromCharCode(c);
c = chunk.charCodeAt(i++);
this.position++;
if (unicodeI === 4) {
this.textNode += String.fromCharCode(parseInt(this.unicodeS, 16));
unicodeI = 0;
starti = i - 1;
} else {
unicodeI++;
}
if (!c)
break STRING_BIGLOOP;
}
if (c === Char.doubleQuote && !slashed) {
this.state = this.stack.pop() || 1 /* VALUE */;
this.textNode += chunk.substring(starti, i - 1);
this.position += i - 1 - starti;
break;
}
if (c === Char.backslash && !slashed) {
slashed = true;
this.textNode += chunk.substring(starti, i - 1);
this.position += i - 1 - starti;
c = chunk.charCodeAt(i++);
this.position++;
if (!c)
break;
}
if (slashed) {
slashed = false;
if (c === Char.n) {
this.textNode += "\n";
} else if (c === Char.r) {
this.textNode += "\r";
} else if (c === Char.t) {
this.textNode += " ";
} else if (c === Char.f) {
this.textNode += "\f";
} else if (c === Char.b) {
this.textNode += "\b";
} else if (c === Char.u) {
unicodeI = 1;
this.unicodeS = "";
} else {
this.textNode += String.fromCharCode(c);
}
c = chunk.charCodeAt(i++);
this.position++;
starti = i - 1;
if (!c)
break;
else
continue;
}
stringTokenPattern.lastIndex = i;
const reResult = stringTokenPattern.exec(chunk);
if (reResult === null) {
i = chunk.length + 1;
this.textNode += chunk.substring(starti, i - 1);
this.position += i - 1 - starti;
break;
}
i = reResult.index + 1;
c = chunk.charCodeAt(reResult.index);
if (!c) {
this.textNode += chunk.substring(starti, i - 1);
this.position += i - 1 - starti;
break;
}
}
this.slashed = slashed;
this.unicodeI = unicodeI;
continue;
case 12 /* TRUE */:
if (c === Char.r)
this.state = 13 /* TRUE2 */;
else
this._error(`Invalid true started with t${c}`);
continue;
case 13 /* TRUE2 */:
if (c === Char.u)
this.state = 14 /* TRUE3 */;
else
this._error(`Invalid true started with tr${c}`);
continue;
case 14 /* TRUE3 */:
if (c === Char.e) {
this.emit("onvalue", true);
this.state = this.stack.pop() || 1 /* VALUE */;
} else
this._error(`Invalid true started with tru${c}`);
continue;
case 15 /* FALSE */:
if (c === Char.a)
this.state = 16 /* FALSE2 */;
else
this._error(`Invalid false started with f${c}`);
continue;
case 16 /* FALSE2 */:
if (c === Char.l)
this.state = 17 /* FALSE3 */;
else
this._error(`Invalid false started with fa${c}`);
continue;
case 17 /* FALSE3 */:
if (c === Char.s)
this.state = 18 /* FALSE4 */;
else
this._error(`Invalid false started with fal${c}`);
continue;
case 18 /* FALSE4 */:
if (c === Char.e) {
this.emit("onvalue", false);
this.state = this.stack.pop() || 1 /* VALUE */;
} else
this._error(`Invalid false started with fals${c}`);
continue;
case 19 /* NULL */:
if (c === Char.u)
this.state = 20 /* NULL2 */;
else
this._error(`Invalid null started with n${c}`);
continue;
case 20 /* NULL2 */:
if (c === Char.l)
this.state = 21 /* NULL3 */;
else
this._error(`Invalid null started with nu${c}`);
continue;
case 21 /* NULL3 */:
if (c === Char.l) {
this.emit("onvalue", null);
this.state = this.stack.pop() || 1 /* VALUE */;
} else
this._error(`Invalid null started with nul${c}`);
continue;
case 22 /* NUMBER_DECIMAL_POINT */:
if (c === Char.period) {
this.numberNode += ".";
this.state = 23 /* NUMBER_DIGIT */;
} else
this._error("Leading zero not followed by .");
continue;
case 23 /* NUMBER_DIGIT */:
if (Char._0 <= c && c <= Char._9)
this.numberNode += String.fromCharCode(c);
else if (c === Char.period) {
if (this.numberNode.indexOf(".") !== -1)
this._error("Invalid number has two dots");
this.numberNode += ".";
} else if (c === Char.e || c === Char.E) {
if (this.numberNode.indexOf("e") !== -1 || this.numberNode.indexOf("E") !== -1)
this._error("Invalid number has two exponential");
this.numberNode += "e";
} else if (c === Char.plus || c === Char.minus) {
if (!(p === Char.e || p === Char.E))
this._error("Invalid symbol in number");
this.numberNode += String.fromCharCode(c);
} else {
this._closeNumber();
i--;
this.state = this.stack.pop() || 1 /* VALUE */;
}
continue;
default:
this._error(`Unknown state: ${this.state}`);
}
}
if (this.position >= this.bufferCheckPosition) {
checkBufferLength(this);
}
this.emit("onchunkparsed");
return this;
}
_closeValue(event = "onvalue") {
if (this.textNode !== void 0) {
this.emit(event, this.textNode);
}
this.textNode = void 0;
}
_closeNumber() {
if (this.numberNode)
this.emit("onvalue", parseFloat(this.numberNode));
this.numberNode = "";
}
_error(message = "") {
this._closeValue();
message += `
Line: ${this.line}
Column: ${this.column}
Char: ${this.c}`;
const error = new Error(message);
this.error = error;
this.emit("onerror", error);
}
};
function isWhitespace(c) {
return c === Char.carriageReturn || c === Char.lineFeed || c === Char.space || c === Char.tab;
}
function checkBufferLength(parser) {
const maxAllowed = Math.max(MAX_BUFFER_LENGTH, 10);
let maxActual = 0;
for (const buffer of ["textNode", "numberNode"]) {
const len = parser[buffer] === void 0 ? 0 : parser[buffer].length;
if (len > maxAllowed) {
switch (buffer) {
case "text":
break;
default:
parser._error(`Max buffer length exceeded: ${buffer}`);
}
}
maxActual = Math.max(maxActual, len);
}
parser.bufferCheckPosition = MAX_BUFFER_LENGTH - maxActual + parser.position;
}
// src/lib/jsonpath/jsonpath.ts
var JSONPath = class {
path;
constructor(path = null) {
this.path = ["$"];
if (path instanceof JSONPath) {
this.path = [...path.path];
return;
}
if (Array.isArray(path)) {
this.path.push(...path);
return;
}
if (typeof path === "string") {
this.path = path.split(".");
if (this.path[0] !== "$") {
throw new Error("JSONPaths must start with $");
}
}
}
clone() {
return new JSONPath(this);
}
toString() {
return this.path.join(".");
}
push(name) {
this.path.push(name);
}
pop() {
return this.path.pop();
}
set(name) {
this.path[this.path.length - 1] = name;
}
equals(other) {
if (!this || !other || this.path.length !== other.path.length) {
return false;
}
for (let i = 0; i < this.path.length; ++i) {
if (this.path[i] !== other.path[i]) {
return false;
}
}
return true;
}
/**
* Sets the value pointed at by path
* TODO - handle root path
* @param object
* @param value
*/
setFieldAtPath(object, value) {
const path = [...this.path];
path.shift();
const field = path.pop();
for (const component of path) {
object = object[component];
}
object[field] = value;
}
/**
* Gets the value pointed at by path
* TODO - handle root path
* @param object
*/
getFieldAtPath(object) {
const path = [...this.path];
path.shift();
const field = path.pop();
for (const component of path) {
object = object[component];
}
return object[field];
}
};
// src/lib/json-parser/json-parser.ts
var JSONParser = class {
parser;
result = void 0;
previousStates = [];
currentState = Object.freeze({ container: [], key: null });
jsonpath = new JSONPath();
constructor(options) {
this.reset();
this.parser = new ClarinetParser({
onready: () => {
this.jsonpath = new JSONPath();
this.previousStates.length = 0;
this.currentState.container.length = 0;
},
onopenobject: (name) => {
this._openObject({});
if (typeof name !== "undefined") {
this.parser.emit("onkey", name);
}
},
onkey: (name) => {
this.jsonpath.set(name);
this.currentState.key = name;
},
oncloseobject: () => {
this._closeObject();
},
onopenarray: () => {
this._openArray();
},
onclosearray: () => {
this._closeArray();
},
onvalue: (value) => {
this._pushOrSet(value);
},
onerror: (error) => {
throw error;
},
onend: () => {
this.result = this.currentState.container.pop();
},
...options
});
}
reset() {
this.result = void 0;
this.previousStates = [];
this.currentState = Object.freeze({ container: [], key: null });
this.jsonpath = new JSONPath();
}
write(chunk) {
this.parser.write(chunk);
}
close() {
this.parser.close();
}
// PRIVATE METHODS
_pushOrSet(value) {
const { container, key } = this.currentState;
if (key !== null) {
container[key] = value;
this.currentState.key = null;
} else {
container.push(value);
}
}
_openArray(newContainer = []) {
this.jsonpath.push(null);
this._pushOrSet(newContainer);
this.previousStates.push(this.currentState);
this.currentState = { container: newContainer, isArray: true, key: null };
}
_closeArray() {
this.jsonpath.pop();
this.currentState = this.previousStates.pop();
}
_openObject(newContainer = {}) {
this.jsonpath.push(null);
this._pushOrSet(newContainer);
this.previousStates.push(this.currentState);
this.currentState = { container: newContainer, isArray: false, key: null };
}
_closeObject() {
this.jsonpath.pop();
this.currentState = this.previousStates.pop();
}
};
// src/lib/json-parser/streaming-json-parser.ts
var StreamingJSONParser = class extends JSONParser {
jsonPaths;
streamingJsonPath = null;
streamingArray = null;
topLevelObject = null;
constructor(options = {}) {
super({
onopenarray: () => {
if (!this.streamingArray) {
if (this._matchJSONPath()) {
this.streamingJsonPath = this.getJsonPath().clone();
this.streamingArray = [];
this._openArray(this.streamingArray);
return;
}
}
this._openArray();
},
// Redefine onopenarray to inject value for top-level object
onopenobject: (name) => {
if (!this.topLevelObject) {
this.topLevelObject = {};
this._openObject(this.topLevelObject);
} else {
this._openObject({});
}
if (typeof name !== "undefined") {
this.parser.emit("onkey", name);
}
}
});
const jsonpaths = options.jsonpaths || [];
this.jsonPaths = jsonpaths.map((jsonpath) => new JSONPath(jsonpath));
}
/**
* write REDEFINITION
* - super.write() chunk to parser
* - get the contents (so far) of "topmost-level" array as batch of rows
* - clear top-level array
* - return the batch of rows\
*/
write(chunk) {
super.write(chunk);
let array = [];
if (this.streamingArray) {
array = [...this.streamingArray];
this.streamingArray.length = 0;
}
return array;
}
/**
* Returns a partially formed result object
* Useful for returning the "wrapper" object when array is not top level
* e.g. GeoJSON
*/
getPartialResult() {
return this.topLevelObject;
}
getStreamingJsonPath() {
return this.streamingJsonPath;
}
getStreamingJsonPathAsString() {
return this.streamingJsonPath && this.streamingJsonPath.toString();
}
getJsonPath() {
return this.jsonpath;
}
// PRIVATE METHODS
/**
* Checks is this.getJsonPath matches the jsonpaths provided in options
*/
_matchJSONPath() {
const currentPath = this.getJsonPath();
if (this.jsonPaths.length === 0) {
return true;
}
for (const jsonPath of this.jsonPaths) {
if (jsonPath.equals(currentPath)) {
return true;
}
}
return false;
}
};
// src/lib/parsers/parse-json-in-batches.ts
async function* parseJSONInBatches(binaryAsyncIterat