@loaders.gl/csv
Version:
Framework-independent loader for CSV and DSV table formats
1,482 lines (1,469 loc) • 63.3 kB
JavaScript
(function webpackUniversalModuleDefinition(root, factory) {
if (typeof exports === 'object' && typeof module === 'object')
module.exports = factory();
else if (typeof define === 'function' && define.amd) define([], factory);
else if (typeof exports === 'object') exports['loaders'] = factory();
else root['loaders'] = factory();})(globalThis, function () {
"use strict";
var __exports__ = (() => {
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __defNormalProp = (obj, key, value) => key in obj ? __defProp(obj, key, { enumerable: true, configurable: true, writable: true, value }) : obj[key] = value;
var __commonJS = (cb, mod) => function __require() {
return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
};
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __reExport = (target, mod, secondTarget) => (__copyProps(target, mod, "default"), secondTarget && __copyProps(secondTarget, mod, "default"));
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var __publicField = (obj, key, value) => {
__defNormalProp(obj, typeof key !== "symbol" ? key + "" : key, value);
return value;
};
// external-global-plugin:@loaders.gl/core
var require_core = __commonJS({
"external-global-plugin:@loaders.gl/core"(exports, module) {
module.exports = globalThis.loaders;
}
});
// bundle.ts
var bundle_exports = {};
__export(bundle_exports, {
CSVLoader: () => CSVLoader,
CSVWriter: () => CSVWriter
});
__reExport(bundle_exports, __toESM(require_core(), 1));
// ../schema/src/lib/table/batches/base-table-batch-aggregator.ts
var DEFAULT_ROW_COUNT = 100;
var BaseTableBatchAggregator = class {
schema;
options;
shape;
length = 0;
rows = null;
cursor = 0;
_headers = [];
constructor(schema, options) {
this.options = options;
this.schema = schema;
if (!Array.isArray(schema)) {
this._headers = [];
for (const key in schema) {
this._headers[schema[key].index] = schema[key].name;
}
}
}
rowCount() {
return this.length;
}
addArrayRow(row, cursor) {
if (Number.isFinite(cursor)) {
this.cursor = cursor;
}
this.shape = "array-row-table";
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
this.rows[this.length] = row;
this.length++;
}
addObjectRow(row, cursor) {
if (Number.isFinite(cursor)) {
this.cursor = cursor;
}
this.shape = "object-row-table";
this.rows = this.rows || new Array(DEFAULT_ROW_COUNT);
this.rows[this.length] = row;
this.length++;
}
getBatch() {
let rows = this.rows;
if (!rows) {
return null;
}
rows = rows.slice(0, this.length);
this.rows = null;
const batch = {
shape: this.shape || "array-row-table",
batchType: "data",
data: rows,
length: this.length,
schema: this.schema,
cursor: this.cursor
};
return batch;
}
};
// ../schema/src/lib/table/simple-table/row-utils.ts
function convertToObjectRow(arrayRow, headers) {
if (!arrayRow) {
throw new Error("null row");
}
const objectRow = {};
if (headers) {
for (let i = 0; i < headers.length; i++) {
objectRow[headers[i]] = arrayRow[i];
}
} else {
for (let i = 0; i < arrayRow.length; i++) {
const columnName = `column-${i}`;
objectRow[columnName] = arrayRow[i];
}
}
return objectRow;
}
function convertToArrayRow(objectRow, headers) {
if (!objectRow) {
throw new Error("null row");
}
if (headers) {
const arrayRow = new Array(headers.length);
for (let i = 0; i < headers.length; i++) {
arrayRow[i] = objectRow[headers[i]];
}
return arrayRow;
}
return Object.values(objectRow);
}
function inferHeadersFromArrayRow(arrayRow) {
const headers = [];
for (let i = 0; i < arrayRow.length; i++) {
const columnName = `column-${i}`;
headers.push(columnName);
}
return headers;
}
function inferHeadersFromObjectRow(row) {
return Object.keys(row);
}
// ../schema/src/lib/table/batches/row-table-batch-aggregator.ts
var DEFAULT_ROW_COUNT2 = 100;
var RowTableBatchAggregator = class {
schema;
options;
length = 0;
objectRows = null;
arrayRows = null;
cursor = 0;
_headers = null;
constructor(schema, options) {
this.options = options;
this.schema = schema;
if (schema) {
this._headers = [];
for (const key in schema) {
this._headers[schema[key].index] = schema[key].name;
}
}
}
rowCount() {
return this.length;
}
addArrayRow(row, cursor) {
if (Number.isFinite(cursor)) {
this.cursor = cursor;
}
this._headers ||= inferHeadersFromArrayRow(row);
switch (this.options.shape) {
case "object-row-table":
const rowObject = convertToObjectRow(row, this._headers);
this.addObjectRow(rowObject, cursor);
break;
case "array-row-table":
this.arrayRows = this.arrayRows || new Array(DEFAULT_ROW_COUNT2);
this.arrayRows[this.length] = row;
this.length++;
break;
}
}
addObjectRow(row, cursor) {
if (Number.isFinite(cursor)) {
this.cursor = cursor;
}
this._headers ||= inferHeadersFromObjectRow(row);
switch (this.options.shape) {
case "array-row-table":
const rowArray = convertToArrayRow(row, this._headers);
this.addArrayRow(rowArray, cursor);
break;
case "object-row-table":
this.objectRows = this.objectRows || new Array(DEFAULT_ROW_COUNT2);
this.objectRows[this.length] = row;
this.length++;
break;
}
}
getBatch() {
let rows = this.arrayRows || this.objectRows;
if (!rows) {
return null;
}
rows = rows.slice(0, this.length);
this.arrayRows = null;
this.objectRows = null;
return {
shape: this.options.shape,
batchType: "data",
data: rows,
length: this.length,
// @ts-expect-error we should infer a schema
schema: this.schema,
cursor: this.cursor
};
}
};
// ../schema/src/lib/table/batches/columnar-table-batch-aggregator.ts
var DEFAULT_ROW_COUNT3 = 100;
var ColumnarTableBatchAggregator = class {
schema;
length = 0;
allocated = 0;
columns = {};
constructor(schema, options) {
this.schema = schema;
this._reallocateColumns();
}
rowCount() {
return this.length;
}
addArrayRow(row) {
this._reallocateColumns();
let i = 0;
for (const fieldName in this.columns) {
this.columns[fieldName][this.length] = row[i++];
}
this.length++;
}
addObjectRow(row) {
this._reallocateColumns();
for (const fieldName in row) {
this.columns[fieldName][this.length] = row[fieldName];
}
this.length++;
}
getBatch() {
this._pruneColumns();
const columns = Array.isArray(this.schema) ? this.columns : {};
if (!Array.isArray(this.schema)) {
for (const fieldName in this.schema) {
const field = this.schema[fieldName];
columns[field.name] = this.columns[field.index];
}
}
this.columns = {};
const batch = {
shape: "columnar-table",
batchType: "data",
data: columns,
schema: this.schema,
length: this.length
};
return batch;
}
// HELPERS
_reallocateColumns() {
if (this.length < this.allocated) {
return;
}
this.allocated = this.allocated > 0 ? this.allocated *= 2 : DEFAULT_ROW_COUNT3;
this.columns = {};
for (const fieldName in this.schema) {
const field = this.schema[fieldName];
const ArrayType = field.type || Float32Array;
const oldColumn = this.columns[field.index];
if (oldColumn && ArrayBuffer.isView(oldColumn)) {
const typedArray = new ArrayType(this.allocated);
typedArray.set(oldColumn);
this.columns[field.index] = typedArray;
} else if (oldColumn) {
oldColumn.length = this.allocated;
this.columns[field.index] = oldColumn;
} else {
this.columns[field.index] = new ArrayType(this.allocated);
}
}
}
_pruneColumns() {
for (const [columnName, column] of Object.entries(this.columns)) {
this.columns[columnName] = column.slice(0, this.length);
}
}
};
// ../schema/src/lib/table/batches/table-batch-builder.ts
var DEFAULT_OPTIONS = {
shape: void 0,
batchSize: "auto",
batchDebounceMs: 0,
limit: 0,
_limitMB: 0
};
var ERR_MESSAGE = "TableBatchBuilder";
var _TableBatchBuilder = class {
schema;
options;
aggregator = null;
batchCount = 0;
bytesUsed = 0;
isChunkComplete = false;
lastBatchEmittedMs = Date.now();
totalLength = 0;
totalBytes = 0;
rowBytes = 0;
constructor(schema, options) {
this.schema = schema;
this.options = { ...DEFAULT_OPTIONS, ...options };
}
limitReached() {
if (Boolean(this.options?.limit) && this.totalLength >= this.options.limit) {
return true;
}
if (Boolean(this.options?._limitMB) && this.totalBytes / 1e6 >= this.options._limitMB) {
return true;
}
return false;
}
/** @deprecated Use addArrayRow or addObjectRow */
addRow(row) {
if (this.limitReached()) {
return;
}
this.totalLength++;
this.rowBytes = this.rowBytes || this._estimateRowMB(row);
this.totalBytes += this.rowBytes;
if (Array.isArray(row)) {
this.addArrayRow(row);
} else {
this.addObjectRow(row);
}
}
/** Add one row to the batch */
addArrayRow(row) {
if (!this.aggregator) {
const TableBatchType = this._getTableBatchType();
this.aggregator = new TableBatchType(this.schema, this.options);
}
this.aggregator.addArrayRow(row);
}
/** Add one row to the batch */
addObjectRow(row) {
if (!this.aggregator) {
const TableBatchType = this._getTableBatchType();
this.aggregator = new TableBatchType(this.schema, this.options);
}
this.aggregator.addObjectRow(row);
}
/** Mark an incoming raw memory chunk has completed */
chunkComplete(chunk) {
if (chunk instanceof ArrayBuffer) {
this.bytesUsed += chunk.byteLength;
}
if (typeof chunk === "string") {
this.bytesUsed += chunk.length;
}
this.isChunkComplete = true;
}
getFullBatch(options) {
return this._isFull() ? this._getBatch(options) : null;
}
getFinalBatch(options) {
return this._getBatch(options);
}
// INTERNAL
_estimateRowMB(row) {
return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
}
_isFull() {
if (!this.aggregator || this.aggregator.rowCount() === 0) {
return false;
}
if (this.options.batchSize === "auto") {
if (!this.isChunkComplete) {
return false;
}
} else if (this.options.batchSize > this.aggregator.rowCount()) {
return false;
}
if (this.options.batchDebounceMs > Date.now() - this.lastBatchEmittedMs) {
return false;
}
this.isChunkComplete = false;
this.lastBatchEmittedMs = Date.now();
return true;
}
/**
* bytesUsed can be set via chunkComplete or via getBatch*
*/
_getBatch(options) {
if (!this.aggregator) {
return null;
}
if (options?.bytesUsed) {
this.bytesUsed = options.bytesUsed;
}
const normalizedBatch = this.aggregator.getBatch();
normalizedBatch.count = this.batchCount;
normalizedBatch.bytesUsed = this.bytesUsed;
Object.assign(normalizedBatch, options);
this.batchCount++;
this.aggregator = null;
return normalizedBatch;
}
_getTableBatchType() {
switch (this.options.shape) {
case "array-row-table":
case "object-row-table":
return RowTableBatchAggregator;
case "columnar-table":
return ColumnarTableBatchAggregator;
case "arrow-table":
if (!_TableBatchBuilder.ArrowBatch) {
throw new Error(ERR_MESSAGE);
}
return _TableBatchBuilder.ArrowBatch;
default:
return BaseTableBatchAggregator;
}
}
};
var TableBatchBuilder = _TableBatchBuilder;
__publicField(TableBatchBuilder, "ArrowBatch");
// ../schema/src/lib/table/simple-table/table-accessors.ts
function getTableLength(table) {
switch (table.shape) {
case "array-row-table":
case "object-row-table":
return table.data.length;
case "geojson-table":
return table.features.length;
case "arrow-table":
const arrowTable = table.data;
return arrowTable.numRows;
case "columnar-table":
for (const column of Object.values(table.data)) {
return column.length || 0;
}
return 0;
default:
throw new Error("table");
}
}
function getTableNumCols(table) {
if (table.schema) {
return table.schema.fields.length;
}
if (getTableLength(table) === 0) {
throw new Error("empty table");
}
switch (table.shape) {
case "array-row-table":
return table.data[0].length;
case "object-row-table":
return Object.keys(table.data[0]).length;
case "geojson-table":
return Object.keys(table.features[0]).length;
case "columnar-table":
return Object.keys(table.data).length;
case "arrow-table":
const arrowTable = table.data;
return arrowTable.numCols;
default:
throw new Error("table");
}
}
function getTableRowAsArray(table, rowIndex, target, copy2) {
switch (table.shape) {
case "array-row-table":
return copy2 ? Array.from(table.data[rowIndex]) : table.data[rowIndex];
case "object-row-table":
if (table.schema) {
const arrayRow2 = target || [];
for (let i = 0; i < table.schema.fields.length; i++) {
arrayRow2[i] = table.data[rowIndex][table.schema.fields[i].name];
}
return arrayRow2;
}
return Object.values(table.data[rowIndex]);
case "geojson-table":
if (table.schema) {
const arrayRow2 = target || [];
for (let i = 0; i < table.schema.fields.length; i++) {
arrayRow2[i] = table.features[rowIndex][table.schema.fields[i].name];
}
return arrayRow2;
}
return Object.values(table.features[rowIndex]);
case "columnar-table":
if (table.schema) {
const arrayRow2 = target || [];
for (let i = 0; i < table.schema.fields.length; i++) {
arrayRow2[i] = table.data[table.schema.fields[i].name][rowIndex];
}
return arrayRow2;
} else {
const arrayRow2 = target || [];
let i = 0;
for (const column of Object.values(table.data)) {
arrayRow2[i] = column[rowIndex];
i++;
}
return arrayRow2;
}
case "arrow-table":
const arrowTable = table.data;
const arrayRow = target || [];
const row = arrowTable.get(rowIndex);
const schema = arrowTable.schema;
for (let i = 0; i < schema.fields.length; i++) {
arrayRow[i] = row?.[schema.fields[i].name];
}
return arrayRow;
default:
throw new Error("shape");
}
}
function* makeArrayRowIterator(table, target = []) {
const length = getTableLength(table);
for (let rowIndex = 0; rowIndex < length; rowIndex++) {
yield getTableRowAsArray(table, rowIndex, target);
}
}
// ../schema/src/lib/utils/async-queue.ts
var ArrayQueue = class extends Array {
enqueue(value) {
return this.push(value);
}
dequeue() {
return this.shift();
}
};
var AsyncQueue = class {
_values;
_settlers;
_closed;
constructor() {
this._values = new ArrayQueue();
this._settlers = new ArrayQueue();
this._closed = false;
}
close() {
while (this._settlers.length > 0) {
this._settlers.dequeue().resolve({ done: true });
}
this._closed = true;
}
[Symbol.asyncIterator]() {
return this;
}
enqueue(value) {
if (this._closed) {
throw new Error("Closed");
}
if (this._settlers.length > 0) {
if (this._values.length > 0) {
throw new Error("Illegal internal state");
}
const settler = this._settlers.dequeue();
if (value instanceof Error) {
settler.reject(value);
} else {
settler.resolve({ value });
}
} else {
this._values.enqueue(value);
}
}
/**
* @returns a Promise for an IteratorResult
*/
next() {
if (this._values.length > 0) {
const value = this._values.dequeue();
if (value instanceof Error) {
return Promise.reject(value);
}
return Promise.resolve({ value });
}
if (this._closed) {
if (this._settlers.length > 0) {
throw new Error("Illegal internal state");
}
return Promise.resolve({ done: true });
}
return new Promise((resolve, reject) => {
this._settlers.enqueue({ resolve, reject });
});
}
};
// src/papaparse/papaparse.ts
var BYTE_ORDER_MARK = "\uFEFF";
function CsvToJson(_input, _config = {}, Streamer = StringStreamer) {
_config = _config || {};
var dynamicTyping = _config.dynamicTyping || false;
if (isFunction(dynamicTyping)) {
_config.dynamicTypingFunction = dynamicTyping;
dynamicTyping = {};
}
_config.dynamicTyping = dynamicTyping;
_config.transform = isFunction(_config.transform) ? _config.transform : false;
var streamer = new Streamer(_config);
return streamer.stream(_input);
}
function JsonToCsv(_input, _config) {
var _quotes = false;
var _writeHeader = true;
var _delimiter = ",";
var _newline = "\r\n";
var _quoteChar = '"';
var _escapedQuote = _quoteChar + _quoteChar;
var _skipEmptyLines = false;
var _columns = null;
unpackConfig();
var quoteCharRegex = new RegExp(escapeRegExp(_quoteChar), "g");
if (typeof _input === "string")
_input = JSON.parse(_input);
if (Array.isArray(_input)) {
if (!_input.length || Array.isArray(_input[0]))
return serialize(null, _input, _skipEmptyLines);
else if (typeof _input[0] === "object")
return serialize(_columns || Object.keys(_input[0]), _input, _skipEmptyLines);
} else if (typeof _input === "object") {
if (typeof _input.data === "string")
_input.data = JSON.parse(_input.data);
if (Array.isArray(_input.data)) {
if (!_input.fields)
_input.fields = _input.meta && _input.meta.fields;
if (!_input.fields)
_input.fields = Array.isArray(_input.data[0]) ? _input.fields : Object.keys(_input.data[0]);
if (!Array.isArray(_input.data[0]) && typeof _input.data[0] !== "object")
_input.data = [_input.data];
}
return serialize(_input.fields || [], _input.data || [], _skipEmptyLines);
}
throw new Error("Unable to serialize unrecognized input");
function unpackConfig() {
if (typeof _config !== "object")
return;
if (typeof _config.delimiter === "string" && !Papa.BAD_DELIMITERS.filter(function(value) {
return _config.delimiter.indexOf(value) !== -1;
}).length) {
_delimiter = _config.delimiter;
}
if (typeof _config.quotes === "boolean" || Array.isArray(_config.quotes))
_quotes = _config.quotes;
if (typeof _config.skipEmptyLines === "boolean" || typeof _config.skipEmptyLines === "string")
_skipEmptyLines = _config.skipEmptyLines;
if (typeof _config.newline === "string")
_newline = _config.newline;
if (typeof _config.quoteChar === "string")
_quoteChar = _config.quoteChar;
if (typeof _config.header === "boolean")
_writeHeader = _config.header;
if (Array.isArray(_config.columns)) {
if (_config.columns.length === 0)
throw new Error("Option columns is empty");
_columns = _config.columns;
}
if (_config.escapeChar !== void 0) {
_escapedQuote = _config.escapeChar + _quoteChar;
}
}
function serialize(fields, data, skipEmptyLines) {
var csv2 = "";
if (typeof fields === "string")
fields = JSON.parse(fields);
if (typeof data === "string")
data = JSON.parse(data);
var hasHeader = Array.isArray(fields) && fields.length > 0;
var dataKeyedByField = !Array.isArray(data[0]);
if (hasHeader && _writeHeader) {
for (var i = 0; i < fields.length; i++) {
if (i > 0)
csv2 += _delimiter;
csv2 += safe(fields[i], i);
}
if (data.length > 0)
csv2 += _newline;
}
for (var row = 0; row < data.length; row++) {
var maxCol = hasHeader ? fields.length : data[row].length;
var emptyLine = false;
var nullLine = hasHeader ? Object.keys(data[row]).length === 0 : data[row].length === 0;
if (skipEmptyLines && !hasHeader) {
emptyLine = skipEmptyLines === "greedy" ? data[row].join("").trim() === "" : data[row].length === 1 && data[row][0].length === 0;
}
if (skipEmptyLines === "greedy" && hasHeader) {
var line = [];
for (var c = 0; c < maxCol; c++) {
var cx = dataKeyedByField ? fields[c] : c;
line.push(data[row][cx]);
}
emptyLine = line.join("").trim() === "";
}
if (!emptyLine) {
for (var col = 0; col < maxCol; col++) {
if (col > 0 && !nullLine)
csv2 += _delimiter;
var colIdx = hasHeader && dataKeyedByField ? fields[col] : col;
csv2 += safe(data[row][colIdx], col);
}
if (row < data.length - 1 && (!skipEmptyLines || maxCol > 0 && !nullLine)) {
csv2 += _newline;
}
}
}
return csv2;
}
function safe(str, col) {
if (typeof str === "undefined" || str === null)
return "";
if (str.constructor === Date)
return JSON.stringify(str).slice(1, 25);
str = str.toString().replace(quoteCharRegex, _escapedQuote);
var needsQuotes = typeof _quotes === "boolean" && _quotes || Array.isArray(_quotes) && _quotes[col] || hasAny(str, Papa.BAD_DELIMITERS) || str.indexOf(_delimiter) > -1 || str.charAt(0) === " " || str.charAt(str.length - 1) === " ";
return needsQuotes ? _quoteChar + str + _quoteChar : str;
}
function hasAny(str, substrings) {
for (var i = 0; i < substrings.length; i++)
if (str.indexOf(substrings[i]) > -1)
return true;
return false;
}
}
var ChunkStreamer = class {
_handle;
_config;
_finished = false;
_completed = false;
_input = null;
_baseIndex = 0;
_partialLine = "";
_rowCount = 0;
_start = 0;
isFirstChunk = true;
_completeResults = {
data: [],
errors: [],
meta: {}
};
constructor(config) {
var configCopy = { ...config };
configCopy.chunkSize = parseInt(configCopy.chunkSize);
if (!config.step && !config.chunk) {
configCopy.chunkSize = null;
}
this._handle = new ParserHandle(configCopy);
this._handle.streamer = this;
this._config = configCopy;
}
parseChunk(chunk, isFakeChunk) {
if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) {
var modifiedChunk = this._config.beforeFirstChunk(chunk);
if (modifiedChunk !== void 0)
chunk = modifiedChunk;
}
this.isFirstChunk = false;
var aggregate = this._partialLine + chunk;
this._partialLine = "";
var results = this._handle.parse(aggregate, this._baseIndex, !this._finished);
if (this._handle.paused() || this._handle.aborted())
return;
var lastIndex = results.meta.cursor;
if (!this._finished) {
this._partialLine = aggregate.substring(lastIndex - this._baseIndex);
this._baseIndex = lastIndex;
}
if (results && results.data)
this._rowCount += results.data.length;
var finishedIncludingPreview = this._finished || this._config.preview && this._rowCount >= this._config.preview;
if (isFunction(this._config.chunk) && !isFakeChunk) {
this._config.chunk(results, this._handle);
if (this._handle.paused() || this._handle.aborted())
return;
results = void 0;
this._completeResults = void 0;
}
if (!this._config.step && !this._config.chunk) {
this._completeResults.data = this._completeResults.data.concat(results.data);
this._completeResults.errors = this._completeResults.errors.concat(results.errors);
this._completeResults.meta = results.meta;
}
if (!this._completed && finishedIncludingPreview && isFunction(this._config.complete) && (!results || !results.meta.aborted)) {
this._config.complete(this._completeResults, this._input);
this._completed = true;
}
return results;
}
_sendError(error) {
if (isFunction(this._config.error))
this._config.error(error);
}
};
var StringStreamer = class extends ChunkStreamer {
remaining;
constructor(config = {}) {
super(config);
}
stream(s) {
this.remaining = s;
return this._nextChunk();
}
_nextChunk() {
if (this._finished)
return;
var size = this._config.chunkSize;
var chunk = size ? this.remaining.substr(0, size) : this.remaining;
this.remaining = size ? this.remaining.substr(size) : "";
this._finished = !this.remaining;
return this.parseChunk(chunk);
}
};
var FLOAT = /^\s*-?(\d*\.?\d+|\d+\.?\d*)(e[-+]?\d+)?\s*$/i;
var ISO_DATE = /(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d\.\d+([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))|(\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d([+-][0-2]\d:[0-5]\d|Z))/;
var ParserHandle = class {
_config;
/** Number of times step was called (number of rows parsed) */
_stepCounter = 0;
/** Number of rows that have been parsed so far */
_rowCounter = 0;
/** The input being parsed */
_input;
/** The core parser being used */
_parser;
/** Whether we are paused or not */
_paused = false;
/** Whether the parser has aborted or not */
_aborted = false;
/** Temporary state between delimiter detection and processing results */
_delimiterError = false;
/** Fields are from the header row of the input, if there is one */
_fields = [];
/** The last results returned from the parser */
_results = {
data: [],
errors: [],
meta: {}
};
constructor(_config) {
if (isFunction(_config.step)) {
var userStep = _config.step;
_config.step = (results) => {
this._results = results;
if (this.needsHeaderRow()) {
this.processResults();
} else {
this.processResults();
if (!this._results.data || this._results.data.length === 0)
return;
this._stepCounter += results.data.length;
if (_config.preview && this._stepCounter > _config.preview) {
this._parser.abort();
} else {
userStep(this._results, this);
}
}
};
}
this._config = _config;
}
/**
* Parses input. Most users won't need, and shouldn't mess with, the baseIndex
* and ignoreLastRow parameters. They are used by streamers (wrapper functions)
* when an input comes in multiple chunks, like from a file.
*/
parse(input, baseIndex, ignoreLastRow) {
var quoteChar = this._config.quoteChar || '"';
if (!this._config.newline)
this._config.newline = guessLineEndings(input, quoteChar);
this._delimiterError = false;
if (!this._config.delimiter) {
var delimGuess = this.guessDelimiter(
input,
this._config.newline,
this._config.skipEmptyLines,
this._config.comments,
this._config.delimitersToGuess
);
if (delimGuess.successful) {
this._config.delimiter = delimGuess.bestDelimiter;
} else {
this._delimiterError = true;
this._config.delimiter = Papa.DefaultDelimiter;
}
this._results.meta.delimiter = this._config.delimiter;
} else if (isFunction(this._config.delimiter)) {
this._config.delimiter = this._config.delimiter(input);
this._results.meta.delimiter = this._config.delimiter;
}
var parserConfig = copy(this._config);
if (this._config.preview && this._config.header)
parserConfig.preview++;
this._input = input;
this._parser = new Parser(parserConfig);
this._results = this._parser.parse(this._input, baseIndex, ignoreLastRow);
this.processResults();
return this._paused ? { meta: { paused: true } } : this._results || { meta: { paused: false } };
}
paused() {
return this._paused;
}
pause() {
this._paused = true;
this._parser.abort();
this._input = this._input.substr(this._parser.getCharIndex());
}
resume() {
this._paused = false;
this.streamer.parseChunk(this._input, true);
}
aborted() {
return this._aborted;
}
abort() {
this._aborted = true;
this._parser.abort();
this._results.meta.aborted = true;
if (isFunction(this._config.complete)) {
this._config.complete(this._results);
}
this._input = "";
}
testEmptyLine(s) {
return this._config.skipEmptyLines === "greedy" ? s.join("").trim() === "" : s.length === 1 && s[0].length === 0;
}
processResults() {
if (this._results && this._delimiterError) {
this.addError(
"Delimiter",
"UndetectableDelimiter",
"Unable to auto-detect delimiting character; defaulted to '" + Papa.DefaultDelimiter + "'"
);
this._delimiterError = false;
}
if (this._config.skipEmptyLines) {
for (var i = 0; i < this._results.data.length; i++)
if (this.testEmptyLine(this._results.data[i]))
this._results.data.splice(i--, 1);
}
if (this.needsHeaderRow()) {
this.fillHeaderFields();
}
return this.applyHeaderAndDynamicTypingAndTransformation();
}
needsHeaderRow() {
return this._config.header && this._fields.length === 0;
}
fillHeaderFields() {
if (!this._results)
return;
const addHeder = (header) => {
if (isFunction(this._config.transformHeader))
header = this._config.transformHeader(header);
this._fields.push(header);
};
if (Array.isArray(this._results.data[0])) {
for (var i = 0; this.needsHeaderRow() && i < this._results.data.length; i++)
this._results.data[i].forEach(addHeder);
this._results.data.splice(0, 1);
} else {
this._results.data.forEach(addHeder);
}
}
shouldApplyDynamicTyping(field) {
if (this._config.dynamicTypingFunction && this._config.dynamicTyping[field] === void 0) {
this._config.dynamicTyping[field] = this._config.dynamicTypingFunction(field);
}
return (this._config.dynamicTyping[field] || this._config.dynamicTyping) === true;
}
parseDynamic(field, value) {
if (this.shouldApplyDynamicTyping(field)) {
if (value === "true" || value === "TRUE")
return true;
else if (value === "false" || value === "FALSE")
return false;
else if (FLOAT.test(value))
return parseFloat(value);
else if (ISO_DATE.test(value))
return new Date(value);
else
return value === "" ? null : value;
}
return value;
}
applyHeaderAndDynamicTypingAndTransformation() {
if (!this._results || !this._results.data || !this._config.header && !this._config.dynamicTyping && !this._config.transform) {
return this._results;
}
var incrementBy = 1;
if (!this._results.data[0] || Array.isArray(this._results.data[0])) {
this._results.data = this._results.data.map(this.processRow.bind(this));
incrementBy = this._results.data.length;
} else {
this._results.data = this.processRow(this._results.data, 0);
}
if (this._config.header && this._results.meta)
this._results.meta.fields = this._fields;
this._rowCounter += incrementBy;
return this._results;
}
processRow(rowSource, i) {
var row = this._config.header ? {} : [];
var j;
for (j = 0; j < rowSource.length; j++) {
var field = j;
var value = rowSource[j];
if (this._config.header)
field = j >= this._fields.length ? "__parsed_extra" : this._fields[j];
if (this._config.transform)
value = this._config.transform(value, field);
value = this.parseDynamic(field, value);
if (field === "__parsed_extra") {
row[field] = row[field] || [];
row[field].push(value);
} else
row[field] = value;
}
if (this._config.header) {
if (j > this._fields.length)
this.addError(
"FieldMismatch",
"TooManyFields",
"Too many fields: expected " + this._fields.length + " fields but parsed " + j,
this._rowCounter + i
);
else if (j < this._fields.length)
this.addError(
"FieldMismatch",
"TooFewFields",
"Too few fields: expected " + this._fields.length + " fields but parsed " + j,
this._rowCounter + i
);
}
return row;
}
guessDelimiter(input, newline, skipEmptyLines, comments, delimitersToGuess) {
var bestDelim, bestDelta, fieldCountPrevRow;
delimitersToGuess = delimitersToGuess || [",", " ", "|", ";", Papa.RECORD_SEP, Papa.UNIT_SEP];
for (var i = 0; i < delimitersToGuess.length; i++) {
var delim = delimitersToGuess[i];
var delta = 0, avgFieldCount = 0, emptyLinesCount = 0;
fieldCountPrevRow = void 0;
var preview = new Parser({
comments,
delimiter: delim,
newline,
preview: 10
}).parse(input);
for (var j = 0; j < preview.data.length; j++) {
if (skipEmptyLines && this.testEmptyLine(preview.data[j])) {
emptyLinesCount++;
continue;
}
var fieldCount = preview.data[j].length;
avgFieldCount += fieldCount;
if (typeof fieldCountPrevRow === "undefined") {
fieldCountPrevRow = 0;
continue;
} else if (fieldCount > 1) {
delta += Math.abs(fieldCount - fieldCountPrevRow);
fieldCountPrevRow = fieldCount;
}
}
if (preview.data.length > 0)
avgFieldCount /= preview.data.length - emptyLinesCount;
if ((typeof bestDelta === "undefined" || delta > bestDelta) && avgFieldCount > 1.99) {
bestDelta = delta;
bestDelim = delim;
}
}
this._config.delimiter = bestDelim;
return {
successful: !!bestDelim,
bestDelimiter: bestDelim
};
}
addError(type, code, msg, row) {
this._results.errors.push({
type,
code,
message: msg,
row
});
}
};
function guessLineEndings(input, quoteChar) {
input = input.substr(0, 1024 * 1024);
var re = new RegExp(escapeRegExp(quoteChar) + "([^]*?)" + escapeRegExp(quoteChar), "gm");
input = input.replace(re, "");
var r = input.split("\r");
var n = input.split("\n");
var nAppearsFirst = n.length > 1 && n[0].length < r[0].length;
if (r.length === 1 || nAppearsFirst)
return "\n";
var numWithN = 0;
for (var i = 0; i < r.length; i++) {
if (r[i][0] === "\n")
numWithN++;
}
return numWithN >= r.length / 2 ? "\r\n" : "\r";
}
function escapeRegExp(string) {
return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
function Parser(config) {
config = config || {};
var delim = config.delimiter;
var newline = config.newline;
var comments = config.comments;
var step = config.step;
var preview = config.preview;
var fastMode = config.fastMode;
var quoteChar;
if (config.quoteChar === void 0) {
quoteChar = '"';
} else {
quoteChar = config.quoteChar;
}
var escapeChar = quoteChar;
if (config.escapeChar !== void 0) {
escapeChar = config.escapeChar;
}
if (typeof delim !== "string" || Papa.BAD_DELIMITERS.indexOf(delim) > -1)
delim = ",";
if (comments === delim)
throw new Error("Comment character same as delimiter");
else if (comments === true)
comments = "#";
else if (typeof comments !== "string" || Papa.BAD_DELIMITERS.indexOf(comments) > -1)
comments = false;
if (newline !== "\n" && newline !== "\r" && newline !== "\r\n")
newline = "\n";
var cursor = 0;
var aborted = false;
this.parse = function(input, baseIndex, ignoreLastRow) {
if (typeof input !== "string")
throw new Error("Input must be a string");
var inputLen = input.length, delimLen = delim.length, newlineLen = newline.length, commentsLen = comments.length;
var stepIsFunction = isFunction(step);
cursor = 0;
var data = [], errors = [], row = [], lastCursor = 0;
if (!input)
return returnable();
if (fastMode || fastMode !== false && input.indexOf(quoteChar) === -1) {
var rows = input.split(newline);
for (var i = 0; i < rows.length; i++) {
const row2 = rows[i];
cursor += row2.length;
if (i !== rows.length - 1)
cursor += newline.length;
else if (ignoreLastRow)
return returnable();
if (comments && row2.substr(0, commentsLen) === comments)
continue;
if (stepIsFunction) {
data = [];
pushRow(row2.split(delim));
doStep();
if (aborted)
return returnable();
} else
pushRow(row2.split(delim));
if (preview && i >= preview) {
data = data.slice(0, preview);
return returnable(true);
}
}
return returnable();
}
var nextDelim = input.indexOf(delim, cursor);
var nextNewline = input.indexOf(newline, cursor);
var quoteCharRegex = new RegExp(escapeRegExp(escapeChar) + escapeRegExp(quoteChar), "g");
var quoteSearch;
for (; ; ) {
if (input[cursor] === quoteChar) {
quoteSearch = cursor;
cursor++;
for (; ; ) {
quoteSearch = input.indexOf(quoteChar, quoteSearch + 1);
if (quoteSearch === -1) {
if (!ignoreLastRow) {
errors.push({
type: "Quotes",
code: "MissingQuotes",
message: "Quoted field unterminated",
row: data.length,
// row has yet to be inserted
index: cursor
});
}
return finish();
}
if (quoteSearch === inputLen - 1) {
var value = input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar);
return finish(value);
}
if (quoteChar === escapeChar && input[quoteSearch + 1] === escapeChar) {
quoteSearch++;
continue;
}
if (quoteChar !== escapeChar && quoteSearch !== 0 && input[quoteSearch - 1] === escapeChar) {
continue;
}
var checkUpTo = nextNewline === -1 ? nextDelim : Math.min(nextDelim, nextNewline);
var spacesBetweenQuoteAndDelimiter = extraSpaces(checkUpTo);
if (input[quoteSearch + 1 + spacesBetweenQuoteAndDelimiter] === delim) {
row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar));
cursor = quoteSearch + 1 + spacesBetweenQuoteAndDelimiter + delimLen;
nextDelim = input.indexOf(delim, cursor);
nextNewline = input.indexOf(newline, cursor);
if (stepIsFunction) {
doStep();
if (aborted)
return returnable();
}
if (preview && data.length >= preview)
return returnable(true);
break;
}
var spacesBetweenQuoteAndNewLine = extraSpaces(nextNewline);
if (input.substr(quoteSearch + 1 + spacesBetweenQuoteAndNewLine, newlineLen) === newline) {
row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar));
saveRow(quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen);
nextDelim = input.indexOf(delim, cursor);
if (stepIsFunction) {
doStep();
if (aborted)
return returnable();
}
if (preview && data.length >= preview)
return returnable(true);
break;
}
errors.push({
type: "Quotes",
code: "InvalidQuotes",
message: "Trailing quote on quoted field is malformed",
row: data.length,
// row has yet to be inserted
index: cursor
});
quoteSearch++;
continue;
}
if (stepIsFunction) {
doStep();
if (aborted)
return returnable();
}
if (preview && data.length >= preview)
return returnable(true);
continue;
}
if (comments && row.length === 0 && input.substr(cursor, commentsLen) === comments) {
if (nextNewline === -1)
return returnable();
cursor = nextNewline + newlineLen;
nextNewline = input.indexOf(newline, cursor);
nextDelim = input.indexOf(delim, cursor);
continue;
}
if (nextDelim !== -1 && (nextDelim < nextNewline || nextNewline === -1)) {
row.push(input.substring(cursor, nextDelim));
cursor = nextDelim + delimLen;
nextDelim = input.indexOf(delim, cursor);
continue;
}
if (nextNewline !== -1) {
row.push(input.substring(cursor, nextNewline));
saveRow(nextNewline + newlineLen);
if (stepIsFunction) {
doStep();
if (aborted)
return returnable();
}
if (preview && data.length >= preview)
return returnable(true);
continue;
}
break;
}
return finish();
function pushRow(row2) {
data.push(row2);
lastCursor = cursor;
}
function extraSpaces(index) {
var spaceLength = 0;
if (index !== -1) {
var textBetweenClosingQuoteAndIndex = input.substring(quoteSearch + 1, index);
if (textBetweenClosingQuoteAndIndex && textBetweenClosingQuoteAndIndex.trim() === "") {
spaceLength = textBetweenClosingQuoteAndIndex.length;
}
}
return spaceLength;
}
function finish(value2) {
if (ignoreLastRow)
return returnable();
if (typeof value2 === "undefined")
value2 = input.substr(cursor);
row.push(value2);
cursor = inputLen;
pushRow(row);
if (stepIsFunction)
doStep();
return returnable();
}
function saveRow(newCursor) {
cursor = newCursor;
pushRow(row);
row = [];
nextNewline = input.indexOf(newline, cursor);
}
function returnable(stopped, step2) {
var isStep = step2 || false;
return {
data: isStep ? data[0] : data,
errors,
meta: {
delimiter: delim,
linebreak: newline,
aborted,
truncated: !!stopped,
cursor: lastCursor + (baseIndex || 0)
}
};
}
function doStep() {
step(returnable(void 0, true));
data = [];
errors = [];
}
};
this.abort = function() {
aborted = true;
};
this.getCharIndex = function() {
return cursor;
};
}
function copy(obj) {
if (typeof obj !== "object" || obj === null)
return obj;
var cpy = Array.isArray(obj) ? [] : {};
for (var key in obj)
cpy[key] = copy(obj[key]);
return cpy;
}
function isFunction(func) {
return typeof func === "function";
}
var Papa = {
parse: CsvToJson,
unparse: JsonToCsv,
RECORD_SEP: String.fromCharCode(30),
UNIT_SEP: String.fromCharCode(31),
BYTE_ORDER_MARK,
BAD_DELIMITERS: ["\r", "\n", '"', BYTE_ORDER_MARK],
WORKERS_SUPPORTED: false,
// !IS_WORKER && !!globalThis.Worker
NODE_STREAM_INPUT: 1,
// Configurable chunk sizes for local and remote files, respectively
LocalChunkSize: 1024 * 1024 * 10,
// 10 M,
RemoteChunkSize: 1024 * 1024 * 5,
// 5 M,
DefaultDelimiter: ",",
// Used if not specified and detection fail,
// Exposed for testing and development only
Parser,
ParserHandle,
// BEGIN FORK
ChunkStreamer
};
var papaparse_default = Papa;
// src/papaparse/async-iterator-streamer.ts
var { ChunkStreamer: ChunkStreamer2 } = papaparse_default;
var AsyncIteratorStreamer = class extends ChunkStreamer2 {
textDecoder = new TextDecoder(this._config.encoding);
constructor(config = {}) {
super(config);
}
// Implement ChunkStreamer base class methods
// this.pause = function() {
// ChunkStreamer.prototype.pause.apply(this, arguments);
// };
// this.resume = function() {
// ChunkStreamer.prototype.resume.apply(this, arguments);
// this._input.resume();
// };
async stream(asyncIterator) {
this._input = asyncIterator;
try {
for await (const chunk of asyncIterator) {
this.parseChunk(this.getStringChunk(chunk));
}
this._finished = true;
this.parseChunk("");
} catch (error) {
this._sendError(error);
}
}
_nextChunk() {
}
// HELPER METHODS
getStringChunk(chunk) {
return typeof chunk === "string" ? chunk : this.textDecoder.decode(chunk, { stream: true });
}
};
// src/csv-loader.ts
var VERSION = typeof __VERSION__ !== "undefined" ? __VERSION__ : "latest";
var DEFAULT_CSV_SHAPE = "object-row-table";
var CSVLoader = {
dataType: null,
batchType: null,
id: "csv",
module: "csv",
name: "CSV",
version: VERSION,
extensions: ["csv", "tsv", "dsv"],
mimeTypes: ["text/csv", "text/tab-separated-values", "text/dsv"],
category: "table",
parse: async (arrayBuffer, options) => parseCSV(new TextDecoder().decode(arrayBuffer), options),
parseText: (text, options) => parseCSV(text, options),
parseInBatches: parseCSVInBatches,
// @ts-ignore
// testText: null,
options: {
csv: {
shape: DEFAULT_CSV_SHAPE,
// 'object-row-table'
optimizeMemoryUsage: false,
// CSV options
header: "auto",
columnPrefix: "column",
// delimiter: auto
// newline: auto
quoteChar: '"',
escapeChar: '"',
dynamicTyping: true,
comments: false,