@loaders.gl/json
Version:
Framework-independent loader for JSON and streaming JSON formats
1,304 lines (1,281 loc) • 38.3 kB
JavaScript
;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// dist/index.js
var dist_exports = {};
__export(dist_exports, {
JSONLoader: () => JSONLoader,
JSONWriter: () => JSONWriter,
NDJSONLoader: () => NDJSONLoader,
_ClarinetParser: () => ClarinetParser,
_GeoJSONLoader: () => GeoJSONLoader,
_GeoJSONWorkerLoader: () => GeoJSONWorkerLoader,
_GeoJSONWriter: () => GeoJSONWriter,
_JSONPath: () => JSONPath,
_rebuildJsonObject: () => rebuildJsonObject
});
module.exports = __toCommonJS(dist_exports);
// dist/lib/parsers/parse-json.js
var import_schema = require("@loaders.gl/schema");
function parseJSONSync(jsonText, options) {
var _a;
try {
const json = JSON.parse(jsonText);
if ((_a = options.json) == null ? void 0 : _a.table) {
const data = getFirstArray(json) || json;
return (0, import_schema.makeTableFromData)(data);
}
return json;
} catch (error) {
throw new Error("JSONLoader: failed to parse JSON");
}
}
function getFirstArray(json) {
if (Array.isArray(json)) {
return json;
}
if (json && typeof json === "object") {
for (const value of Object.values(json)) {
const array = getFirstArray(value);
if (array) {
return array;
}
}
}
return null;
}
// dist/lib/parsers/parse-json-in-batches.js
var import_schema2 = require("@loaders.gl/schema");
var import_loader_utils = require("@loaders.gl/loader-utils");
// dist/lib/clarinet/clarinet.js
var MAX_BUFFER_LENGTH = Number.MAX_SAFE_INTEGER;
var STATE;
(function(STATE2) {
STATE2[STATE2["BEGIN"] = 0] = "BEGIN";
STATE2[STATE2["VALUE"] = 1] = "VALUE";
STATE2[STATE2["OPEN_OBJECT"] = 2] = "OPEN_OBJECT";
STATE2[STATE2["CLOSE_OBJECT"] = 3] = "CLOSE_OBJECT";
STATE2[STATE2["OPEN_ARRAY"] = 4] = "OPEN_ARRAY";
STATE2[STATE2["CLOSE_ARRAY"] = 5] = "CLOSE_ARRAY";
STATE2[STATE2["TEXT_ESCAPE"] = 6] = "TEXT_ESCAPE";
STATE2[STATE2["STRING"] = 7] = "STRING";
STATE2[STATE2["BACKSLASH"] = 8] = "BACKSLASH";
STATE2[STATE2["END"] = 9] = "END";
STATE2[STATE2["OPEN_KEY"] = 10] = "OPEN_KEY";
STATE2[STATE2["CLOSE_KEY"] = 11] = "CLOSE_KEY";
STATE2[STATE2["TRUE"] = 12] = "TRUE";
STATE2[STATE2["TRUE2"] = 13] = "TRUE2";
STATE2[STATE2["TRUE3"] = 14] = "TRUE3";
STATE2[STATE2["FALSE"] = 15] = "FALSE";
STATE2[STATE2["FALSE2"] = 16] = "FALSE2";
STATE2[STATE2["FALSE3"] = 17] = "FALSE3";
STATE2[STATE2["FALSE4"] = 18] = "FALSE4";
STATE2[STATE2["NULL"] = 19] = "NULL";
STATE2[STATE2["NULL2"] = 20] = "NULL2";
STATE2[STATE2["NULL3"] = 21] = "NULL3";
STATE2[STATE2["NUMBER_DECIMAL_POINT"] = 22] = "NUMBER_DECIMAL_POINT";
STATE2[STATE2["NUMBER_DIGIT"] = 23] = "NUMBER_DIGIT";
})(STATE || (STATE = {}));
var Char = {
tab: 9,
// \t
lineFeed: 10,
// \n
carriageReturn: 13,
// \r
space: 32,
// " "
doubleQuote: 34,
// "
plus: 43,
// +
comma: 44,
// ,
minus: 45,
// -
period: 46,
// .
_0: 48,
// 0
_9: 57,
// 9
colon: 58,
// :
E: 69,
// E
openBracket: 91,
// [
backslash: 92,
// \
closeBracket: 93,
// ]
a: 97,
// a
b: 98,
// b
e: 101,
// e
f: 102,
// f
l: 108,
// l
n: 110,
// n
r: 114,
// r
s: 115,
// s
t: 116,
// t
u: 117,
// u
openBrace: 123,
// {
closeBrace: 125
// }
};
var stringTokenPattern = /[\\"\n]/g;
var DEFAULT_OPTIONS = {
onready: () => {
},
onopenobject: () => {
},
onkey: () => {
},
oncloseobject: () => {
},
onopenarray: () => {
},
onclosearray: () => {
},
onvalue: () => {
},
onerror: () => {
},
onend: () => {
},
onchunkparsed: () => {
}
};
var ClarinetParser = class {
options = DEFAULT_OPTIONS;
bufferCheckPosition = MAX_BUFFER_LENGTH;
q = "";
c = "";
p = "";
closed = false;
closedRoot = false;
sawRoot = false;
// tag = null;
error = null;
state = STATE.BEGIN;
stack = [];
// mostly just for error reporting
position = 0;
column = 0;
line = 1;
slashed = false;
unicodeI = 0;
unicodeS = null;
depth = 0;
textNode;
numberNode;
constructor(options = {}) {
this.options = { ...DEFAULT_OPTIONS, ...options };
this.textNode = void 0;
this.numberNode = "";
this.emit("onready");
}
end() {
if (this.state !== STATE.VALUE || this.depth !== 0)
this._error("Unexpected end");
this._closeValue();
this.c = "";
this.closed = true;
this.emit("onend");
return this;
}
resume() {
this.error = null;
return this;
}
close() {
return this.write(null);
}
// protected
emit(event, data) {
var _a, _b;
(_b = (_a = this.options)[event]) == null ? void 0 : _b.call(_a, data, this);
}
emitNode(event, data) {
this._closeValue();
this.emit(event, data);
}
/* eslint-disable no-continue */
// eslint-disable-next-line complexity, max-statements
write(chunk) {
if (this.error) {
throw this.error;
}
if (this.closed) {
return this._error("Cannot write after close. Assign an onready handler.");
}
if (chunk === null) {
return this.end();
}
let i = 0;
let c = chunk.charCodeAt(0);
let p = this.p;
while (c) {
p = c;
this.c = c = chunk.charCodeAt(i++);
if (p !== c) {
this.p = p;
} else {
p = this.p;
}
if (!c)
break;
this.position++;
if (c === Char.lineFeed) {
this.line++;
this.column = 0;
} else
this.column++;
switch (this.state) {
case STATE.BEGIN:
if (c === Char.openBrace)
this.state = STATE.OPEN_OBJECT;
else if (c === Char.openBracket)
this.state = STATE.OPEN_ARRAY;
else if (!isWhitespace(c)) {
this._error("Non-whitespace before {[.");
}
continue;
case STATE.OPEN_KEY:
case STATE.OPEN_OBJECT:
if (isWhitespace(c))
continue;
if (this.state === STATE.OPEN_KEY)
this.stack.push(STATE.CLOSE_KEY);
else if (c === Char.closeBrace) {
this.emit("onopenobject");
this.depth++;
this.emit("oncloseobject");
this.depth--;
this.state = this.stack.pop() || STATE.VALUE;
continue;
} else
this.stack.push(STATE.CLOSE_OBJECT);
if (c === Char.doubleQuote)
this.state = STATE.STRING;
else
this._error('Malformed object key should start with "');
continue;
case STATE.CLOSE_KEY:
case STATE.CLOSE_OBJECT:
if (isWhitespace(c))
continue;
if (c === Char.colon) {
if (this.state === STATE.CLOSE_OBJECT) {
this.stack.push(STATE.CLOSE_OBJECT);
this._closeValue("onopenobject");
this.depth++;
} else
this._closeValue("onkey");
this.state = STATE.VALUE;
} else if (c === Char.closeBrace) {
this.emitNode("oncloseobject");
this.depth--;
this.state = this.stack.pop() || STATE.VALUE;
} else if (c === Char.comma) {
if (this.state === STATE.CLOSE_OBJECT)
this.stack.push(STATE.CLOSE_OBJECT);
this._closeValue();
this.state = STATE.OPEN_KEY;
} else
this._error("Bad object");
continue;
case STATE.OPEN_ARRAY:
case STATE.VALUE:
if (isWhitespace(c))
continue;
if (this.state === STATE.OPEN_ARRAY) {
this.emit("onopenarray");
this.depth++;
this.state = STATE.VALUE;
if (c === Char.closeBracket) {
this.emit("onclosearray");
this.depth--;
this.state = this.stack.pop() || STATE.VALUE;
continue;
} else {
this.stack.push(STATE.CLOSE_ARRAY);
}
}
if (c === Char.doubleQuote)
this.state = STATE.STRING;
else if (c === Char.openBrace)
this.state = STATE.OPEN_OBJECT;
else if (c === Char.openBracket)
this.state = STATE.OPEN_ARRAY;
else if (c === Char.t)
this.state = STATE.TRUE;
else if (c === Char.f)
this.state = STATE.FALSE;
else if (c === Char.n)
this.state = STATE.NULL;
else if (c === Char.minus) {
this.numberNode += "-";
} else if (Char._0 <= c && c <= Char._9) {
this.numberNode += String.fromCharCode(c);
this.state = STATE.NUMBER_DIGIT;
} else
this._error("Bad value");
continue;
case STATE.CLOSE_ARRAY:
if (c === Char.comma) {
this.stack.push(STATE.CLOSE_ARRAY);
this._closeValue("onvalue");
this.state = STATE.VALUE;
} else if (c === Char.closeBracket) {
this.emitNode("onclosearray");
this.depth--;
this.state = this.stack.pop() || STATE.VALUE;
} else if (isWhitespace(c))
continue;
else
this._error("Bad array");
continue;
case STATE.STRING:
if (this.textNode === void 0) {
this.textNode = "";
}
let starti = i - 1;
let slashed = this.slashed;
let unicodeI = this.unicodeI;
STRING_BIGLOOP:
while (true) {
while (unicodeI > 0) {
this.unicodeS += String.fromCharCode(c);
c = chunk.charCodeAt(i++);
this.position++;
if (unicodeI === 4) {
this.textNode += String.fromCharCode(parseInt(this.unicodeS, 16));
unicodeI = 0;
starti = i - 1;
} else {
unicodeI++;
}
if (!c)
break STRING_BIGLOOP;
}
if (c === Char.doubleQuote && !slashed) {
this.state = this.stack.pop() || STATE.VALUE;
this.textNode += chunk.substring(starti, i - 1);
this.position += i - 1 - starti;
break;
}
if (c === Char.backslash && !slashed) {
slashed = true;
this.textNode += chunk.substring(starti, i - 1);
this.position += i - 1 - starti;
c = chunk.charCodeAt(i++);
this.position++;
if (!c)
break;
}
if (slashed) {
slashed = false;
if (c === Char.n) {
this.textNode += "\n";
} else if (c === Char.r) {
this.textNode += "\r";
} else if (c === Char.t) {
this.textNode += " ";
} else if (c === Char.f) {
this.textNode += "\f";
} else if (c === Char.b) {
this.textNode += "\b";
} else if (c === Char.u) {
unicodeI = 1;
this.unicodeS = "";
} else {
this.textNode += String.fromCharCode(c);
}
c = chunk.charCodeAt(i++);
this.position++;
starti = i - 1;
if (!c)
break;
else
continue;
}
stringTokenPattern.lastIndex = i;
const reResult = stringTokenPattern.exec(chunk);
if (reResult === null) {
i = chunk.length + 1;
this.textNode += chunk.substring(starti, i - 1);
this.position += i - 1 - starti;
break;
}
i = reResult.index + 1;
c = chunk.charCodeAt(reResult.index);
if (!c) {
this.textNode += chunk.substring(starti, i - 1);
this.position += i - 1 - starti;
break;
}
}
this.slashed = slashed;
this.unicodeI = unicodeI;
continue;
case STATE.TRUE:
if (c === Char.r)
this.state = STATE.TRUE2;
else
this._error(`Invalid true started with t${c}`);
continue;
case STATE.TRUE2:
if (c === Char.u)
this.state = STATE.TRUE3;
else
this._error(`Invalid true started with tr${c}`);
continue;
case STATE.TRUE3:
if (c === Char.e) {
this.emit("onvalue", true);
this.state = this.stack.pop() || STATE.VALUE;
} else
this._error(`Invalid true started with tru${c}`);
continue;
case STATE.FALSE:
if (c === Char.a)
this.state = STATE.FALSE2;
else
this._error(`Invalid false started with f${c}`);
continue;
case STATE.FALSE2:
if (c === Char.l)
this.state = STATE.FALSE3;
else
this._error(`Invalid false started with fa${c}`);
continue;
case STATE.FALSE3:
if (c === Char.s)
this.state = STATE.FALSE4;
else
this._error(`Invalid false started with fal${c}`);
continue;
case STATE.FALSE4:
if (c === Char.e) {
this.emit("onvalue", false);
this.state = this.stack.pop() || STATE.VALUE;
} else
this._error(`Invalid false started with fals${c}`);
continue;
case STATE.NULL:
if (c === Char.u)
this.state = STATE.NULL2;
else
this._error(`Invalid null started with n${c}`);
continue;
case STATE.NULL2:
if (c === Char.l)
this.state = STATE.NULL3;
else
this._error(`Invalid null started with nu${c}`);
continue;
case STATE.NULL3:
if (c === Char.l) {
this.emit("onvalue", null);
this.state = this.stack.pop() || STATE.VALUE;
} else
this._error(`Invalid null started with nul${c}`);
continue;
case STATE.NUMBER_DECIMAL_POINT:
if (c === Char.period) {
this.numberNode += ".";
this.state = STATE.NUMBER_DIGIT;
} else
this._error("Leading zero not followed by .");
continue;
case STATE.NUMBER_DIGIT:
if (Char._0 <= c && c <= Char._9)
this.numberNode += String.fromCharCode(c);
else if (c === Char.period) {
if (this.numberNode.indexOf(".") !== -1)
this._error("Invalid number has two dots");
this.numberNode += ".";
} else if (c === Char.e || c === Char.E) {
if (this.numberNode.indexOf("e") !== -1 || this.numberNode.indexOf("E") !== -1)
this._error("Invalid number has two exponential");
this.numberNode += "e";
} else if (c === Char.plus || c === Char.minus) {
if (!(p === Char.e || p === Char.E))
this._error("Invalid symbol in number");
this.numberNode += String.fromCharCode(c);
} else {
this._closeNumber();
i--;
this.state = this.stack.pop() || STATE.VALUE;
}
continue;
default:
this._error(`Unknown state: ${this.state}`);
}
}
if (this.position >= this.bufferCheckPosition) {
checkBufferLength(this);
}
this.emit("onchunkparsed");
return this;
}
_closeValue(event = "onvalue") {
if (this.textNode !== void 0) {
this.emit(event, this.textNode);
}
this.textNode = void 0;
}
_closeNumber() {
if (this.numberNode)
this.emit("onvalue", parseFloat(this.numberNode));
this.numberNode = "";
}
_error(message = "") {
this._closeValue();
message += `
Line: ${this.line}
Column: ${this.column}
Char: ${this.c}`;
const error = new Error(message);
this.error = error;
this.emit("onerror", error);
}
};
function isWhitespace(c) {
return c === Char.carriageReturn || c === Char.lineFeed || c === Char.space || c === Char.tab;
}
function checkBufferLength(parser) {
const maxAllowed = Math.max(MAX_BUFFER_LENGTH, 10);
let maxActual = 0;
for (const buffer of ["textNode", "numberNode"]) {
const len = parser[buffer] === void 0 ? 0 : parser[buffer].length;
if (len > maxAllowed) {
switch (buffer) {
case "text":
break;
default:
parser._error(`Max buffer length exceeded: ${buffer}`);
}
}
maxActual = Math.max(maxActual, len);
}
parser.bufferCheckPosition = MAX_BUFFER_LENGTH - maxActual + parser.position;
}
// dist/lib/jsonpath/jsonpath.js
var JSONPath = class {
path;
constructor(path = null) {
this.path = ["$"];
if (path instanceof JSONPath) {
this.path = [...path.path];
return;
}
if (Array.isArray(path)) {
this.path.push(...path);
return;
}
if (typeof path === "string") {
this.path = path.split(".");
if (this.path[0] !== "$") {
throw new Error("JSONPaths must start with $");
}
}
}
clone() {
return new JSONPath(this);
}
toString() {
return this.path.join(".");
}
push(name) {
this.path.push(name);
}
pop() {
return this.path.pop();
}
set(name) {
this.path[this.path.length - 1] = name;
}
equals(other) {
if (!this || !other || this.path.length !== other.path.length) {
return false;
}
for (let i = 0; i < this.path.length; ++i) {
if (this.path[i] !== other.path[i]) {
return false;
}
}
return true;
}
/**
* Sets the value pointed at by path
* TODO - handle root path
* @param object
* @param value
*/
setFieldAtPath(object, value) {
const path = [...this.path];
path.shift();
const field = path.pop();
for (const component of path) {
object = object[component];
}
object[field] = value;
}
/**
* Gets the value pointed at by path
* TODO - handle root path
* @param object
*/
getFieldAtPath(object) {
const path = [...this.path];
path.shift();
const field = path.pop();
for (const component of path) {
object = object[component];
}
return object[field];
}
};
// dist/lib/json-parser/json-parser.js
var JSONParser = class {
parser;
result = void 0;
previousStates = [];
currentState = Object.freeze({ container: [], key: null });
jsonpath = new JSONPath();
constructor(options) {
this.reset();
this.parser = new ClarinetParser({
onready: () => {
this.jsonpath = new JSONPath();
this.previousStates.length = 0;
this.currentState.container.length = 0;
},
onopenobject: (name) => {
this._openObject({});
if (typeof name !== "undefined") {
this.parser.emit("onkey", name);
}
},
onkey: (name) => {
this.jsonpath.set(name);
this.currentState.key = name;
},
oncloseobject: () => {
this._closeObject();
},
onopenarray: () => {
this._openArray();
},
onclosearray: () => {
this._closeArray();
},
onvalue: (value) => {
this._pushOrSet(value);
},
onerror: (error) => {
throw error;
},
onend: () => {
this.result = this.currentState.container.pop();
},
...options
});
}
reset() {
this.result = void 0;
this.previousStates = [];
this.currentState = Object.freeze({ container: [], key: null });
this.jsonpath = new JSONPath();
}
write(chunk) {
this.parser.write(chunk);
}
close() {
this.parser.close();
}
// PRIVATE METHODS
_pushOrSet(value) {
const { container, key } = this.currentState;
if (key !== null) {
container[key] = value;
this.currentState.key = null;
} else {
container.push(value);
}
}
_openArray(newContainer = []) {
this.jsonpath.push(null);
this._pushOrSet(newContainer);
this.previousStates.push(this.currentState);
this.currentState = { container: newContainer, isArray: true, key: null };
}
_closeArray() {
this.jsonpath.pop();
this.currentState = this.previousStates.pop();
}
_openObject(newContainer = {}) {
this.jsonpath.push(null);
this._pushOrSet(newContainer);
this.previousStates.push(this.currentState);
this.currentState = { container: newContainer, isArray: false, key: null };
}
_closeObject() {
this.jsonpath.pop();
this.currentState = this.previousStates.pop();
}
};
// dist/lib/json-parser/streaming-json-parser.js
var StreamingJSONParser = class extends JSONParser {
jsonPaths;
streamingJsonPath = null;
streamingArray = null;
topLevelObject = null;
constructor(options = {}) {
super({
onopenarray: () => {
if (!this.streamingArray) {
if (this._matchJSONPath()) {
this.streamingJsonPath = this.getJsonPath().clone();
this.streamingArray = [];
this._openArray(this.streamingArray);
return;
}
}
this._openArray();
},
// Redefine onopenarray to inject value for top-level object
onopenobject: (name) => {
if (!this.topLevelObject) {
this.topLevelObject = {};
this._openObject(this.topLevelObject);
} else {
this._openObject({});
}
if (typeof name !== "undefined") {
this.parser.emit("onkey", name);
}
}
});
const jsonpaths = options.jsonpaths || [];
this.jsonPaths = jsonpaths.map((jsonpath) => new JSONPath(jsonpath));
}
/**
* write REDEFINITION
* - super.write() chunk to parser
* - get the contents (so far) of "topmost-level" array as batch of rows
* - clear top-level array
* - return the batch of rows\
*/
write(chunk) {
super.write(chunk);
let array = [];
if (this.streamingArray) {
array = [...this.streamingArray];
this.streamingArray.length = 0;
}
return array;
}
/**
* Returns a partially formed result object
* Useful for returning the "wrapper" object when array is not top level
* e.g. GeoJSON
*/
getPartialResult() {
return this.topLevelObject;
}
getStreamingJsonPath() {
return this.streamingJsonPath;
}
getStreamingJsonPathAsString() {
return this.streamingJsonPath && this.streamingJsonPath.toString();
}
getJsonPath() {
return this.jsonpath;
}
// PRIVATE METHODS
/**
* Checks is this.getJsonPath matches the jsonpaths provided in options
*/
_matchJSONPath() {
const currentPath = this.getJsonPath();
if (this.jsonPaths.length === 0) {
return true;
}
for (const jsonPath of this.jsonPaths) {
if (jsonPath.equals(currentPath)) {
return true;
}
}
return false;
}
};
// dist/lib/parsers/parse-json-in-batches.js
async function* parseJSONInBatches(binaryAsyncIterator, options) {
var _a;
const asyncIterator = (0, import_loader_utils.makeTextDecoderIterator)(binaryAsyncIterator);
const { metadata } = options;
const { jsonpaths } = options.json || {};
let isFirstChunk = true;
const schema = null;
const tableBatchBuilder = new import_schema2.TableBatchBuilder(schema, options);
const parser = new StreamingJSONParser({ jsonpaths });
for await (const chunk of asyncIterator) {
const rows = parser.write(chunk);
const jsonpath2 = rows.length > 0 && parser.getStreamingJsonPathAsString();
if (rows.length > 0 && isFirstChunk) {
if (metadata) {
const initialBatch = {
// Common fields
shape: ((_a = options == null ? void 0 : options.json) == null ? void 0 : _a.shape) || "array-row-table",
batchType: "partial-result",
data: [],
length: 0,
bytesUsed: 0,
// JSON additions
container: parser.getPartialResult(),
jsonpath: jsonpath2
};
yield initialBatch;
}
isFirstChunk = false;
}
for (const row of rows) {
tableBatchBuilder.addRow(row);
const batch3 = tableBatchBuilder.getFullBatch({ jsonpath: jsonpath2 });
if (batch3) {
yield batch3;
}
}
tableBatchBuilder.chunkComplete(chunk);
const batch2 = tableBatchBuilder.getFullBatch({ jsonpath: jsonpath2 });
if (batch2) {
yield batch2;
}
}
const jsonpath = parser.getStreamingJsonPathAsString();
const batch = tableBatchBuilder.getFinalBatch({ jsonpath });
if (batch) {
yield batch;
}
if (metadata) {
const finalBatch = {
shape: "json",
batchType: "final-result",
container: parser.getPartialResult(),
jsonpath: parser.getStreamingJsonPathAsString(),
/** Data Just to avoid crashing? */
data: [],
length: 0
// schema: null
};
yield finalBatch;
}
}
function rebuildJsonObject(batch, data) {
(0, import_loader_utils.assert)(batch.batchType === "final-result");
if (batch.jsonpath === "$") {
return data;
}
if (batch.jsonpath && batch.jsonpath.length > 1) {
const topLevelObject = batch.container;
const streamingPath = new JSONPath(batch.jsonpath);
streamingPath.setFieldAtPath(topLevelObject, data);
return topLevelObject;
}
return batch.container;
}
// dist/json-loader.js
var VERSION = true ? "4.3.2" : "latest";
var JSONLoader = {
dataType: null,
batchType: null,
name: "JSON",
id: "json",
module: "json",
version: VERSION,
extensions: ["json", "geojson"],
mimeTypes: ["application/json"],
category: "table",
text: true,
options: {
json: {
shape: void 0,
table: false,
jsonpaths: []
// batchSize: 'auto'
}
},
parse,
parseTextSync,
parseInBatches
};
async function parse(arrayBuffer, options) {
return parseTextSync(new TextDecoder().decode(arrayBuffer), options);
}
function parseTextSync(text, options) {
const jsonOptions = { ...options, json: { ...JSONLoader.options.json, ...options == null ? void 0 : options.json } };
return parseJSONSync(text, jsonOptions);
}
function parseInBatches(asyncIterator, options) {
const jsonOptions = { ...options, json: { ...JSONLoader.options.json, ...options == null ? void 0 : options.json } };
return parseJSONInBatches(asyncIterator, jsonOptions);
}
// dist/lib/parsers/parse-ndjson.js
var import_schema3 = require("@loaders.gl/schema");
function parseNDJSONSync(ndjsonText) {
const lines = ndjsonText.trim().split("\n");
const parsedLines = lines.map((line, counter) => {
try {
return JSON.parse(line);
} catch (error) {
throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);
}
});
return (0, import_schema3.makeTableFromData)(parsedLines);
}
// dist/lib/parsers/parse-ndjson-in-batches.js
var import_schema4 = require("@loaders.gl/schema");
var import_loader_utils2 = require("@loaders.gl/loader-utils");
async function* parseNDJSONInBatches(binaryAsyncIterator, options) {
const textIterator = (0, import_loader_utils2.makeTextDecoderIterator)(binaryAsyncIterator);
const lineIterator = (0, import_loader_utils2.makeLineIterator)(textIterator);
const numberedLineIterator = (0, import_loader_utils2.makeNumberedLineIterator)(lineIterator);
const schema = null;
const shape = "row-table";
const tableBatchBuilder = new import_schema4.TableBatchBuilder(schema, {
...options,
shape
});
for await (const { counter, line } of numberedLineIterator) {
try {
const row = JSON.parse(line);
tableBatchBuilder.addRow(row);
tableBatchBuilder.chunkComplete(line);
const batch2 = tableBatchBuilder.getFullBatch();
if (batch2) {
yield batch2;
}
} catch (error) {
throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);
}
}
const batch = tableBatchBuilder.getFinalBatch();
if (batch) {
yield batch;
}
}
// dist/ndjson-loader.js
var VERSION2 = true ? "4.3.2" : "latest";
var NDJSONLoader = {
dataType: null,
batchType: null,
name: "NDJSON",
id: "ndjson",
module: "json",
version: VERSION2,
extensions: ["ndjson", "jsonl"],
mimeTypes: [
"application/x-ndjson",
"application/jsonlines",
// https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#cm-batch
"application/json-seq"
],
category: "table",
text: true,
parse: async (arrayBuffer) => parseNDJSONSync(new TextDecoder().decode(arrayBuffer)),
parseTextSync: parseNDJSONSync,
parseInBatches: parseNDJSONInBatches,
options: {}
};
// dist/lib/encoders/json-encoder.js
var import_schema5 = require("@loaders.gl/schema");
function encodeTableAsJSON(table, options) {
var _a;
const shape = ((_a = options == null ? void 0 : options.json) == null ? void 0 : _a.shape) || "object-row-table";
const strings = [];
const rowIterator = (0, import_schema5.makeRowIterator)(table, shape);
for (const row of rowIterator) {
strings.push(JSON.stringify(row));
}
return `[${strings.join(",")}]`;
}
// dist/json-writer.js
var JSONWriter = {
id: "json",
version: "latest",
module: "json",
name: "JSON",
extensions: ["json"],
mimeTypes: ["application/json"],
options: {},
text: true,
encode: async (table, options) => new TextEncoder().encode(encodeTableAsJSON(table, options)).buffer,
encodeTextSync: (table, options) => encodeTableAsJSON(table, options)
};
// dist/geojson-loader.js
var import_gis = require("@loaders.gl/gis");
var VERSION3 = true ? "4.3.2" : "latest";
var GeoJSONWorkerLoader = {
dataType: null,
batchType: null,
name: "GeoJSON",
id: "geojson",
module: "geojson",
version: VERSION3,
worker: true,
extensions: ["geojson"],
mimeTypes: ["application/geo+json"],
category: "geometry",
text: true,
options: {
geojson: {
shape: "geojson-table"
},
json: {
shape: "object-row-table",
jsonpaths: ["$", "$.features"]
},
gis: {
format: "geojson"
}
}
};
var GeoJSONLoader = {
...GeoJSONWorkerLoader,
// @ts-expect-error
parse: parse2,
// @ts-expect-error
parseTextSync: parseTextSync2,
parseInBatches: parseInBatches2
};
async function parse2(arrayBuffer, options) {
return parseTextSync2(new TextDecoder().decode(arrayBuffer), options);
}
function parseTextSync2(text, options) {
options = { ...GeoJSONLoader.options, ...options };
options.geojson = { ...GeoJSONLoader.options.geojson, ...options.geojson };
options.gis = options.gis || {};
let geojson;
try {
geojson = JSON.parse(text);
} catch {
geojson = {};
}
const table = {
shape: "geojson-table",
// TODO - deduce schema from geojson
// TODO check that parsed data is of type FeatureCollection
type: "FeatureCollection",
features: (geojson == null ? void 0 : geojson.features) || []
};
switch (options.gis.format) {
case "binary":
return (0, import_gis.geojsonToBinary)(table.features);
default:
return table;
}
}
function parseInBatches2(asyncIterator, options) {
options = { ...GeoJSONLoader.options, ...options };
options.json = { ...GeoJSONLoader.options.geojson, ...options.geojson };
const geojsonIterator = parseJSONInBatches(asyncIterator, options);
switch (options.gis.format) {
case "binary":
return makeBinaryGeometryIterator(geojsonIterator);
default:
return geojsonIterator;
}
}
async function* makeBinaryGeometryIterator(geojsonIterator) {
for await (const batch of geojsonIterator) {
batch.data = (0, import_gis.geojsonToBinary)(batch.data);
yield batch;
}
}
// dist/geojson-writer.js
var import_loader_utils3 = require("@loaders.gl/loader-utils");
// dist/lib/encoders/geojson-encoder.js
var import_schema8 = require("@loaders.gl/schema");
// dist/lib/encoder-utils/encode-utils.js
var import_schema6 = require("@loaders.gl/schema");
function detectGeometryColumnIndex(table) {
var _a;
const geometryIndex = ((_a = table.schema) == null ? void 0 : _a.fields.findIndex((field) => field.name === "geometry")) ?? -1;
if (geometryIndex > -1) {
return geometryIndex;
}
if ((0, import_schema6.getTableLength)(table) > 0) {
const row = (0, import_schema6.getTableRowAsArray)(table, 0);
for (let columnIndex = 0; columnIndex < (0, import_schema6.getTableNumCols)(table); columnIndex++) {
const value = row == null ? void 0 : row[columnIndex];
if (value && typeof value === "object") {
return columnIndex;
}
}
}
throw new Error("Failed to detect geometry column");
}
function getRowPropertyObject(table, row, excludeColumnIndices = []) {
var _a;
const properties = {};
for (let columnIndex = 0; columnIndex < (0, import_schema6.getTableNumCols)(table); ++columnIndex) {
const columnName = (_a = table.schema) == null ? void 0 : _a.fields[columnIndex].name;
if (columnName && !excludeColumnIndices.includes(columnIndex)) {
properties[columnName] = row[columnName];
}
}
return properties;
}
// dist/lib/encoder-utils/encode-table-row.js
var import_schema7 = require("@loaders.gl/schema");
function encodeTableRow(table, rowIndex, geometryColumnIndex, utf8Encoder) {
const row = (0, import_schema7.getTableRowAsObject)(table, rowIndex);
if (!row)
return;
const featureWithProperties = getFeatureFromRow(table, row, geometryColumnIndex);
const featureString = JSON.stringify(featureWithProperties);
utf8Encoder.push(featureString);
}
function getFeatureFromRow(table, row, geometryColumnIndex) {
var _a;
const properties = getRowPropertyObject(table, row, [geometryColumnIndex]);
const columnName = (_a = table.schema) == null ? void 0 : _a.fields[geometryColumnIndex].name;
let featureOrGeometry = columnName && row[columnName];
if (!featureOrGeometry) {
return { type: "Feature", geometry: null, properties };
}
if (typeof featureOrGeometry === "string") {
try {
featureOrGeometry = JSON.parse(featureOrGeometry);
} catch (err) {
throw new Error("Invalid string geometry");
}
}
if (typeof featureOrGeometry !== "object" || typeof (featureOrGeometry == null ? void 0 : featureOrGeometry.type) !== "string") {
throw new Error("invalid geometry column value");
}
if ((featureOrGeometry == null ? void 0 : featureOrGeometry.type) === "Feature") {
return { ...featureOrGeometry, properties };
}
return { type: "Feature", geometry: featureOrGeometry, properties };
}
// dist/lib/encoder-utils/utf8-encoder.js
var Utf8ArrayBufferEncoder = class {
chunkSize;
strings = [];
totalLength = 0;
textEncoder = new TextEncoder();
constructor(chunkSize) {
this.chunkSize = chunkSize;
}
push(...strings) {
for (const string of strings) {
this.strings.push(string);
this.totalLength += string.length;
}
}
isFull() {
return this.totalLength >= this.chunkSize;
}
getArrayBufferBatch() {
return this.textEncoder.encode(this.getStringBatch()).buffer;
}
getStringBatch() {
const stringChunk = this.strings.join("");
this.strings = [];
this.totalLength = 0;
return stringChunk;
}
};
// dist/lib/encoders/geojson-encoder.js
async function* encodeTableAsGeojsonInBatches(batchIterator, inputOpts = {}) {
const options = { geojson: {}, chunkSize: 1e4, ...inputOpts };
const utf8Encoder = new Utf8ArrayBufferEncoder(options.chunkSize);
if (!options.geojson.featureArray) {
utf8Encoder.push("{\n", '"type": "FeatureCollection",\n', '"features":\n');
}
utf8Encoder.push("[");
let geometryColumn = options.geojson.geometryColumn;
let isFirstLine = true;
let start = 0;
for await (const tableBatch of batchIterator) {
const end = start + (0, import_schema8.getTableLength)(tableBatch);
if (!geometryColumn) {
geometryColumn = geometryColumn || detectGeometryColumnIndex(tableBatch);
}
for (let rowIndex = start; rowIndex < end; ++rowIndex) {
if (!isFirstLine) {
utf8Encoder.push(",");
}
utf8Encoder.push("\n");
isFirstLine = false;
encodeTableRow(tableBatch, rowIndex, geometryColumn, utf8Encoder);
if (utf8Encoder.isFull()) {
yield utf8Encoder.getArrayBufferBatch();
}
start = end;
}
const arrayBufferBatch = utf8Encoder.getArrayBufferBatch();
if (arrayBufferBatch.byteLength > 0) {
yield arrayBufferBatch;
}
}
utf8Encoder.push("\n");
utf8Encoder.push("]\n");
if (!options.geojson.featureArray) {
utf8Encoder.push("}");
}
yield utf8Encoder.getArrayBufferBatch();
}
// dist/geojson-writer.js
var GeoJSONWriter = {
id: "geojson",
version: "latest",
module: "geojson",
name: "GeoJSON",
extensions: ["geojson"],
mimeTypes: ["application/geo+json"],
text: true,
options: {
geojson: {
featureArray: false,
geometryColumn: null
}
},
async encode(table, options) {
const tableIterator = [table];
const batches = encodeTableAsGeojsonInBatches(tableIterator, options);
return await (0, import_loader_utils3.concatenateArrayBuffersAsync)(batches);
},
encodeInBatches: (tableIterator, options) => encodeTableAsGeojsonInBatches(tableIterator, options)
};
//# sourceMappingURL=index.cjs.map