@loaders.gl/csv
Version:
Framework-independent loader for CSV and DSV table formats
4 lines • 79.9 kB
Source Map (JSON)
{
"version": 3,
"sources": ["../src/index.ts", "../src/csv-loader.ts", "../src/papaparse/papa-constants.ts", "../src/papaparse/papa-parser.ts", "../src/papaparse/papa-writer.ts", "../src/papaparse/papaparse.ts", "../src/papaparse/async-iterator-streamer.ts", "../src/csv-format.ts", "../src/lib/encoders/encode-csv.ts", "../src/csv-writer.ts", "../src/csv-arrow-loader.ts"],
"sourcesContent": ["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nexport type {CSVLoaderOptions} from './csv-loader';\nexport {CSVLoader} from './csv-loader';\n\nexport type {CSVWriterOptions} from './csv-writer';\nexport {CSVWriter} from './csv-writer';\n\nexport type {CSVArrowLoaderOptions} from './csv-arrow-loader';\nexport {CSVArrowLoader} from './csv-arrow-loader';\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';\nimport type {Schema, ArrayRowTable, ObjectRowTable, TableBatch} from '@loaders.gl/schema';\n\nimport {log, toArrayBufferIterator} from '@loaders.gl/loader-utils';\nimport {\n AsyncQueue,\n deduceTableSchema,\n TableBatchBuilder,\n convertToArrayRow,\n convertToObjectRow\n} from '@loaders.gl/schema-utils';\nimport Papa from './papaparse/papaparse';\nimport AsyncIteratorStreamer from './papaparse/async-iterator-streamer';\nimport {CSVFormat} from './csv-format';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nconst DEFAULT_CSV_SHAPE = 'object-row-table';\n\nexport type CSVLoaderOptions = LoaderOptions & {\n csv?: {\n // loaders.gl options\n shape?: 'array-row-table' | 'object-row-table';\n /** optimizes memory usage but increases parsing time. */\n optimizeMemoryUsage?: boolean;\n columnPrefix?: string;\n header?: 'auto';\n\n // CSV options (papaparse)\n // delimiter: auto\n // newline: auto\n quoteChar?: string;\n escapeChar?: string;\n // Convert numbers and boolean values in rows from strings\n dynamicTyping?: boolean;\n comments?: boolean;\n skipEmptyLines?: boolean | 'greedy';\n // transform: null?\n delimitersToGuess?: string[];\n // fastMode: auto\n };\n};\n\nexport const CSVLoader = {\n ...CSVFormat,\n\n dataType: null as unknown as ObjectRowTable | ArrayRowTable,\n batchType: null as unknown as TableBatch,\n version: VERSION,\n parse: async (arrayBuffer: ArrayBuffer, options?: CSVLoaderOptions) =>\n parseCSV(new TextDecoder().decode(arrayBuffer), options),\n parseText: (text: string, options?: CSVLoaderOptions) => parseCSV(text, options),\n parseInBatches: parseCSVInBatches,\n // @ts-ignore\n // testText: null,\n options: {\n csv: {\n shape: DEFAULT_CSV_SHAPE, // 'object-row-table'\n optimizeMemoryUsage: false,\n // CSV options\n header: 'auto',\n columnPrefix: 'column',\n // delimiter: auto\n // newline: auto\n quoteChar: '\"',\n escapeChar: '\"',\n dynamicTyping: true,\n comments: false,\n skipEmptyLines: true,\n // transform: null?\n delimitersToGuess: [',', '\\t', '|', ';']\n // fastMode: auto\n }\n }\n} as const satisfies LoaderWithParser<ObjectRowTable | ArrayRowTable, TableBatch, CSVLoaderOptions>;\n\nasync function parseCSV(\n csvText: string,\n options?: CSVLoaderOptions\n): Promise<ObjectRowTable | ArrayRowTable> {\n // Apps can call the parse method directly, so we apply default options here\n const csvOptions = {...CSVLoader.options.csv, ...options?.csv};\n\n const firstRow = readFirstRow(csvText);\n const header: boolean =\n csvOptions.header === 'auto' ? isHeaderRow(firstRow) : Boolean(csvOptions.header);\n\n const parseWithHeader = header;\n\n const papaparseConfig = {\n // dynamicTyping: true,\n ...csvOptions,\n header: parseWithHeader,\n download: false, // We handle loading, no need for papaparse to do it for us\n transformHeader: parseWithHeader ? duplicateColumnTransformer() : undefined,\n error: (e) => {\n throw new Error(e);\n }\n };\n\n const result = Papa.parse(csvText, papaparseConfig);\n const rows = result.data as any[];\n\n const headerRow = result.meta.fields || generateHeader(csvOptions.columnPrefix, firstRow.length);\n\n const shape = csvOptions.shape || DEFAULT_CSV_SHAPE;\n let table: ArrayRowTable | ObjectRowTable;\n switch (shape) {\n case 'object-row-table':\n table = {\n shape: 'object-row-table',\n data: rows.map((row) => (Array.isArray(row) ? convertToObjectRow(row, headerRow) : row))\n };\n break;\n case 'array-row-table':\n table = {\n shape: 'array-row-table',\n data: rows.map((row) => (Array.isArray(row) ? row : convertToArrayRow(row, headerRow)))\n };\n break;\n default:\n throw new Error(shape);\n }\n table.schema = deduceTableSchema(table!);\n return table;\n}\n\n// TODO - support batch size 0 = no batching/single batch?\nfunction parseCSVInBatches(\n asyncIterator:\n | AsyncIterable<ArrayBufferLike | ArrayBufferView>\n | Iterable<ArrayBufferLike | ArrayBufferView>,\n options?: CSVLoaderOptions\n): AsyncIterable<TableBatch> {\n // Papaparse does not support standard batch size handling\n // TODO - investigate papaparse chunks mode\n options = {...options};\n if (options?.core?.batchSize === 'auto') {\n options.core.batchSize = 4000;\n }\n\n // Apps can call the parse method directly, we so apply default options here\n const csvOptions = {...CSVLoader.options.csv, ...options?.csv};\n\n const asyncQueue = new AsyncQueue<TableBatch>();\n\n let isFirstRow: boolean = true;\n let headerRow: string[] | null = null;\n let tableBatchBuilder: TableBatchBuilder | null = null;\n let schema: Schema | null = null;\n\n const config = {\n // dynamicTyping: true, // Convert numbers and boolean values in rows from strings,\n ...csvOptions,\n header: false, // Unfortunately, header detection is not automatic and does not infer shapes\n download: false, // We handle loading, no need for papaparse to do it for us\n // chunkSize is set to 5MB explicitly (same as Papaparse default) due to a bug where the\n // streaming parser gets stuck if skipEmptyLines and a step callback are both supplied.\n // See https://github.com/mholt/PapaParse/issues/465\n chunkSize: 1024 * 1024 * 5,\n // skipEmptyLines is set to a boolean value if supplied. Greedy is set to true\n // skipEmptyLines is handled manually given two bugs where the streaming parser gets stuck if\n // both of the skipEmptyLines and step callback options are provided:\n // - true doesn't work unless chunkSize is set: https://github.com/mholt/PapaParse/issues/465\n // - greedy doesn't work: https://github.com/mholt/PapaParse/issues/825\n skipEmptyLines: false,\n\n // step is called on every row\n // eslint-disable-next-line complexity, max-statements\n step(results) {\n let row = results.data;\n\n if (csvOptions.skipEmptyLines) {\n // Manually reject lines that are empty\n const collapsedRow = row.flat().join('').trim();\n if (collapsedRow === '') {\n return;\n }\n }\n const bytesUsed = results.meta.cursor;\n\n // Check if we need to save a header row\n if (isFirstRow && !headerRow) {\n // Auto detects or can be forced with csvOptions.header\n const header = csvOptions.header === 'auto' ? isHeaderRow(row) : Boolean(csvOptions.header);\n if (header) {\n headerRow = row.map(duplicateColumnTransformer());\n return;\n }\n }\n\n // If first data row, we can deduce the schema\n if (isFirstRow) {\n isFirstRow = false;\n if (!headerRow) {\n headerRow = generateHeader(csvOptions.columnPrefix, row.length);\n }\n schema = deduceCSVSchema(row, headerRow);\n }\n\n if (csvOptions.optimizeMemoryUsage) {\n // A workaround to allocate new strings and don't retain pointers to original strings.\n // https://bugs.chromium.org/p/v8/issues/detail?id=2869\n row = JSON.parse(JSON.stringify(row));\n }\n\n const shape = (options as any)?.shape || csvOptions.shape || DEFAULT_CSV_SHAPE;\n\n // Add the row\n tableBatchBuilder =\n tableBatchBuilder ||\n new TableBatchBuilder(\n // @ts-expect-error TODO this is not a proper schema\n schema,\n {\n shape,\n ...(options?.core || {})\n }\n );\n\n try {\n tableBatchBuilder.addRow(row);\n // If a batch has been completed, emit it\n const batch = tableBatchBuilder && tableBatchBuilder.getFullBatch({bytesUsed});\n if (batch) {\n asyncQueue.enqueue(batch);\n }\n } catch (error) {\n asyncQueue.enqueue(error as Error);\n }\n },\n\n // complete is called when all rows have been read\n complete(results) {\n try {\n const bytesUsed = results.meta.cursor;\n // Ensure any final (partial) batch gets emitted\n const batch = tableBatchBuilder && tableBatchBuilder.getFinalBatch({bytesUsed});\n if (batch) {\n asyncQueue.enqueue(batch);\n }\n } catch (error) {\n asyncQueue.enqueue(error as Error);\n }\n\n asyncQueue.close();\n }\n };\n\n Papa.parse(toArrayBufferIterator(asyncIterator), config, AsyncIteratorStreamer);\n\n // TODO - Does it matter if we return asyncIterable or asyncIterator\n // return asyncQueue[Symbol.asyncIterator]();\n return asyncQueue;\n}\n\n/**\n * Checks if a certain row is a header row\n * @param row the row to check\n * @returns true if the row looks like a header\n */\nfunction isHeaderRow(row: string[]): boolean {\n return row && row.every((value) => typeof value === 'string');\n}\n\n/**\n * Reads, parses, and returns the first row of a CSV text\n * @param csvText the csv text to parse\n * @returns the first row\n */\nfunction readFirstRow(csvText: string): any[] {\n const result = Papa.parse(csvText, {\n dynamicTyping: true,\n preview: 1\n });\n return result.data[0];\n}\n\n/**\n * Creates a transformer that renames duplicate columns. This is needed as Papaparse doesn't handle\n * duplicate header columns and would use the latest occurrence by default.\n * See the header option in https://www.papaparse.com/docs#config\n * @returns a transform function that returns sanitized names for duplicate fields\n */\nfunction duplicateColumnTransformer(): (column: string) => string {\n const observedColumns = new Set<string>();\n return (col) => {\n let colName = col;\n let counter = 1;\n while (observedColumns.has(colName)) {\n colName = `${col}.${counter}`;\n counter++;\n }\n observedColumns.add(colName);\n return colName;\n };\n}\n\n/**\n * Generates the header of a CSV given a prefix and a column count\n * @param columnPrefix the columnPrefix to use\n * @param count the count of column names to generate\n * @returns an array of column names\n */\nfunction generateHeader(columnPrefix: string, count: number = 0): string[] {\n const headers: string[] = [];\n for (let i = 0; i < count; i++) {\n headers.push(`${columnPrefix}${i + 1}`);\n }\n return headers;\n}\n\nfunction deduceCSVSchema(row, headerRow): Schema {\n const fields: Schema['fields'] = [];\n for (let i = 0; i < row.length; i++) {\n const columnName = (headerRow && headerRow[i]) || i;\n const value = row[i];\n switch (typeof value) {\n case 'number':\n fields.push({name: String(columnName), type: 'float64', nullable: true});\n break;\n case 'boolean':\n fields.push({name: String(columnName), type: 'bool', nullable: true});\n break;\n case 'string':\n fields.push({name: String(columnName), type: 'utf8', nullable: true});\n break;\n default:\n log.warn(`CSV: Unknown column type: ${typeof value}`)();\n fields.push({name: String(columnName), type: 'utf8', nullable: true});\n }\n }\n return {\n fields,\n metadata: {\n 'loaders.gl#format': 'csv',\n 'loaders.gl#loader': 'CSVLoader'\n }\n };\n}\n\n// TODO - remove\n// type ObjectField = {name: string; index: number; type: any};\n// type ObjectSchema = {[key: string]: ObjectField} | ObjectField[];\n\n// function deduceObjectSchema(row, headerRow): ObjectSchema {\n// const schema: ObjectSchema = headerRow ? {} : [];\n// for (let i = 0; i < row.length; i++) {\n// const columnName = (headerRow && headerRow[i]) || i;\n// const value = row[i];\n// switch (typeof value) {\n// case 'number':\n// case 'boolean':\n// // TODO - booleans could be handled differently...\n// schema[columnName] = {name: String(columnName), index: i, type: Float32Array};\n// break;\n// case 'string':\n// default:\n// schema[columnName] = {name: String(columnName), index: i, type: Array};\n// // We currently only handle numeric rows\n// // TODO we could offer a function to map strings to numbers?\n// }\n// }\n// return schema;\n// }\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n// Copyright (c) 2015 Matthew Holt\n\n// This is a fork of papaparse v5.0.0-beta.0 under MIT license\n// https://github.com/mholt/PapaParse\n\nconst BYTE_ORDER_MARK = '\\ufeff';\n\nexport const Papa = {\n RECORD_SEP: String.fromCharCode(30),\n UNIT_SEP: String.fromCharCode(31),\n BYTE_ORDER_MARK,\n BAD_DELIMITERS: ['\\r', '\\n', '\"', BYTE_ORDER_MARK],\n WORKERS_SUPPORTED: false, // !IS_WORKER && !!globalThis.Worker\n NODE_STREAM_INPUT: 1,\n\n // Configurable chunk sizes for local and remote files, respectively\n LocalChunkSize: 1024 * 1024 * 10, // 10 M,\n RemoteChunkSize: 1024 * 1024 * 5, // 5 M,\n DefaultDelimiter: ',' // Used if not specified and detection fail,\n};\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n// Copyright (c) 2015 Matthew Holt\n\n// This is a fork of papaparse v5.0.0-beta.0 under MIT license\n// https://github.com/mholt/PapaParse\n\n/* eslint-disable no-continue, max-depth */\n\nimport {Papa} from './papa-constants';\n\nexport type CSVParserConfig = {\n chunk?: boolean;\n chunkSize?: number | null;\n preview?: number;\n newline?: string;\n comments?: boolean | string;\n skipEmptyLines?: boolean | 'greedy';\n delimitersToGuess?: string[];\n quotes?: string[] | boolean;\n quoteChar?: string;\n escapeChar?: string;\n delimiter?: string | Function;\n // Convert numbers and boolean values in rows from strings\n fastMode?: boolean;\n\n dynamicTyping?: boolean | {};\n dynamicTypingFunction?: Function;\n step?: Function;\n transform?: Function;\n complete?: Function;\n};\n\n// const defaultConfig: Required<CSVParserConfig> = {\n// dynamicTyping: false,\n// dynamicTypingFunction: undefined!,\n// transform: false\n// };\n\nexport function CsvToJson(_input, _config: CSVParserConfig = {}, Streamer: any = StringStreamer) {\n const streamer = new Streamer(_config);\n\n return streamer.stream(_input);\n}\n\n/** ChunkStreamer is the base prototype for various streamer implementations. */\nexport class ChunkStreamer {\n _handle;\n _config;\n\n _finished = false;\n _completed = false;\n _input = null;\n _baseIndex = 0;\n _partialLine = '';\n _rowCount = 0;\n _start = 0;\n isFirstChunk = true;\n _completeResults = {\n data: [],\n errors: [],\n meta: {}\n };\n\n constructor(config: CSVParserConfig) {\n // Deep-copy the config so we can edit it\n const configCopy = {...config};\n if (configCopy.dynamicTypingFunction) {\n configCopy.dynamicTyping = {};\n }\n // @ts-expect-error\n configCopy.chunkSize = parseInt(configCopy.chunkSize); // parseInt VERY important so we don't concatenate strings!\n if (!config.step && !config.chunk) {\n configCopy.chunkSize = null; // disable Range header if not streaming; bad values break IIS - see issue #196\n }\n this._handle = new ParserHandle(configCopy);\n this._handle.streamer = this;\n this._config = configCopy; // persist the copy to the caller\n }\n\n // eslint-disable-next-line complexity, max-statements\n parseChunk(chunk, isFakeChunk?: boolean) {\n // First chunk pre-processing\n if (this.isFirstChunk && isFunction(this._config.beforeFirstChunk)) {\n const modifiedChunk = this._config.beforeFirstChunk(chunk);\n if (modifiedChunk !== undefined) chunk = modifiedChunk;\n }\n this.isFirstChunk = false;\n\n // Rejoin the line we likely just split in two by chunking the file\n const aggregate = this._partialLine + chunk;\n this._partialLine = '';\n\n let results = this._handle.parse(aggregate, this._baseIndex, !this._finished);\n\n if (this._handle.paused() || this._handle.aborted()) return;\n\n const lastIndex = results.meta.cursor;\n\n if (!this._finished) {\n this._partialLine = aggregate.substring(lastIndex - this._baseIndex);\n this._baseIndex = lastIndex;\n }\n\n if (results && results.data) this._rowCount += results.data.length;\n\n const finishedIncludingPreview =\n this._finished || (this._config.preview && this._rowCount >= this._config.preview);\n\n if (isFunction(this._config.chunk) && !isFakeChunk) {\n this._config.chunk(results, this._handle);\n if (this._handle.paused() || this._handle.aborted()) return;\n results = undefined;\n // @ts-expect-error\n this._completeResults = undefined;\n }\n\n if (!this._config.step && !this._config.chunk) {\n this._completeResults.data = this._completeResults.data.concat(results.data);\n this._completeResults.errors = this._completeResults.errors.concat(results.errors);\n this._completeResults.meta = results.meta;\n }\n\n if (\n !this._completed &&\n finishedIncludingPreview &&\n isFunction(this._config.complete) &&\n (!results || !results.meta.aborted)\n ) {\n this._config.complete(this._completeResults, this._input);\n this._completed = true;\n }\n\n // if (!finishedIncludingPreview && (!results || !results.meta.paused)) this._nextChunk();\n\n // eslint-disable-next-line consistent-return\n return results;\n }\n\n _sendError(error) {\n if (isFunction(this._config.error)) this._config.error(error);\n }\n}\n\nclass StringStreamer extends ChunkStreamer {\n remaining;\n\n constructor(config = {}) {\n super(config);\n }\n\n stream(s) {\n this.remaining = s;\n return this._nextChunk();\n }\n\n _nextChunk() {\n if (this._finished) return;\n const size = this._config.chunkSize;\n const chunk = size ? this.remaining.substr(0, size) : this.remaining;\n this.remaining = size ? this.remaining.substr(size) : '';\n this._finished = !this.remaining;\n // eslint-disable-next-line consistent-return\n return this.parseChunk(chunk);\n }\n}\n\nconst FLOAT = /^\\s*-?(\\d*\\.?\\d+|\\d+\\.?\\d*)(e[-+]?\\d+)?\\s*$/i;\nconst ISO_DATE =\n /(\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\d\\.\\d+([+-][0-2]\\d:[0-5]\\d|Z))|(\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d:[0-5]\\d([+-][0-2]\\d:[0-5]\\d|Z))|(\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:[0-5]\\d([+-][0-2]\\d:[0-5]\\d|Z))/;\n\n// Use one ParserHandle per entire CSV file or string\nexport class ParserHandle {\n _config;\n\n /** Number of times step was called (number of rows parsed) */\n _stepCounter = 0;\n /** Number of rows that have been parsed so far */\n _rowCounter = 0;\n /** The input being parsed */\n _input;\n /** The core parser being used */\n _parser;\n /** Whether we are paused or not */\n _paused = false;\n /** Whether the parser has aborted or not */\n _aborted = false;\n /** Temporary state between delimiter detection and processing results */\n _delimiterError: boolean = false;\n /** Fields are from the header row of the input, if there is one */\n _fields: string[] = [];\n /** The last results returned from the parser */\n _results: {\n data: any[][] | Record<string, any>[];\n errors: any[];\n meta: Record<string, any>;\n } = {\n data: [],\n errors: [],\n meta: {}\n };\n\n constructor(_config: CSVParserConfig) {\n // One goal is to minimize the use of regular expressions...\n\n if (isFunction(_config.step)) {\n const userStep = _config.step;\n _config.step = (results) => {\n this._results = results;\n\n if (this.needsHeaderRow()) {\n this.processResults();\n }\n // only call user's step function after header row\n else {\n this.processResults();\n\n // It's possbile that this line was empty and there's no row here after all\n if (!this._results.data || this._results.data.length === 0) return;\n\n this._stepCounter += results.data.length;\n if (_config.preview && this._stepCounter > _config.preview) {\n this._parser.abort();\n } else {\n userStep(this._results, this);\n }\n }\n };\n }\n this._config = _config;\n }\n\n /**\n * Parses input. Most users won't need, and shouldn't mess with, the baseIndex\n * and ignoreLastRow parameters. They are used by streamers (wrapper functions)\n * when an input comes in multiple chunks, like from a file.\n */\n parse(input, baseIndex, ignoreLastRow) {\n const quoteChar = this._config.quoteChar || '\"';\n if (!this._config.newline) this._config.newline = guessLineEndings(input, quoteChar);\n\n this._delimiterError = false;\n if (!this._config.delimiter) {\n const delimGuess = this.guessDelimiter(\n input,\n this._config.newline,\n this._config.skipEmptyLines,\n this._config.comments,\n this._config.delimitersToGuess\n );\n if (delimGuess.successful) {\n this._config.delimiter = delimGuess.bestDelimiter;\n } else {\n this._delimiterError = true; // add error after parsing (otherwise it would be overwritten)\n this._config.delimiter = Papa.DefaultDelimiter;\n }\n this._results.meta.delimiter = this._config.delimiter;\n } else if (isFunction(this._config.delimiter)) {\n this._config.delimiter = this._config.delimiter(input);\n this._results.meta.delimiter = this._config.delimiter;\n }\n\n const parserConfig = copy(this._config);\n if (this._config.preview && this._config.header) parserConfig.preview++; // to compensate for header row\n\n this._input = input;\n this._parser = new Parser(parserConfig);\n this._results = this._parser.parse(this._input, baseIndex, ignoreLastRow);\n this.processResults();\n return this._paused ? {meta: {paused: true}} : this._results || {meta: {paused: false}};\n }\n\n paused() {\n return this._paused;\n }\n\n pause() {\n this._paused = true;\n this._parser.abort();\n this._input = this._input.substr(this._parser.getCharIndex());\n }\n\n resume() {\n this._paused = false;\n // @ts-expect-error\n this.streamer.parseChunk(this._input, true);\n }\n\n aborted() {\n return this._aborted;\n }\n\n abort() {\n this._aborted = true;\n this._parser.abort();\n this._results.meta.aborted = true;\n if (isFunction(this._config.complete)) {\n this._config.complete(this._results);\n }\n this._input = '';\n }\n\n testEmptyLine(s) {\n return this._config.skipEmptyLines === 'greedy'\n ? s.join('').trim() === ''\n : s.length === 1 && s[0].length === 0;\n }\n\n processResults() {\n if (this._results && this._delimiterError) {\n this.addError(\n 'Delimiter',\n 'UndetectableDelimiter',\n `Unable to auto-detect delimiting character; defaulted to '${Papa.DefaultDelimiter}'`\n );\n this._delimiterError = false;\n }\n\n if (this._config.skipEmptyLines) {\n for (let i = 0; i < this._results.data.length; i++)\n if (this.testEmptyLine(this._results.data[i])) this._results.data.splice(i--, 1);\n }\n\n if (this.needsHeaderRow()) {\n this.fillHeaderFields();\n }\n\n return this.applyHeaderAndDynamicTypingAndTransformation();\n }\n\n needsHeaderRow() {\n return this._config.header && this._fields.length === 0;\n }\n\n fillHeaderFields() {\n if (!this._results) return;\n\n const addHeder = (header) => {\n if (isFunction(this._config.transformHeader)) header = this._config.transformHeader(header);\n this._fields.push(header);\n };\n\n if (Array.isArray(this._results.data[0])) {\n for (let i = 0; this.needsHeaderRow() && i < this._results.data.length; i++)\n this._results.data[i].forEach(addHeder);\n\n this._results.data.splice(0, 1);\n }\n // if _results.data[0] is not an array, we are in a step where _results.data is the row.\n else {\n this._results.data.forEach(addHeder);\n }\n }\n\n shouldApplyDynamicTyping(field) {\n // Cache function values to avoid calling it for each row\n if (this._config.dynamicTypingFunction && this._config.dynamicTyping?.[field] === undefined) {\n this._config.dynamicTyping[field] = this._config.dynamicTypingFunction(field);\n }\n return (this._config.dynamicTyping?.[field] || this._config.dynamicTyping) === true;\n }\n\n parseDynamic(field, value) {\n if (this.shouldApplyDynamicTyping(field)) {\n if (value === 'true' || value === 'TRUE') return true;\n else if (value === 'false' || value === 'FALSE') return false;\n else if (FLOAT.test(value)) return parseFloat(value);\n else if (ISO_DATE.test(value)) return new Date(value);\n return value === '' ? null : value;\n }\n return value;\n }\n\n applyHeaderAndDynamicTypingAndTransformation() {\n if (\n !this._results ||\n !this._results.data ||\n (!this._config.header && !this._config.dynamicTyping && !this._config.transform)\n ) {\n return this._results;\n }\n\n let incrementBy = 1;\n if (!this._results.data[0] || Array.isArray(this._results.data[0])) {\n this._results.data = this._results.data.map(this.processRow.bind(this));\n incrementBy = this._results.data.length;\n } else {\n // @ts-expect-error\n this._results.data = this.processRow(this._results.data, 0);\n }\n\n if (this._config.header && this._results.meta) this._results.meta.fields = this._fields;\n\n this._rowCounter += incrementBy;\n return this._results;\n }\n\n processRow(rowSource, i): any[] | Record<string, any> {\n const row = this._config.header ? {} : [];\n\n let j;\n for (j = 0; j < rowSource.length; j++) {\n let field = j;\n let value = rowSource[j];\n\n if (this._config.header)\n field = j >= this._fields.length ? '__parsed_extra' : this._fields[j];\n\n if (this._config.transform) value = this._config.transform(value, field);\n\n value = this.parseDynamic(field, value);\n\n if (field === '__parsed_extra') {\n row[field] = row[field] || [];\n row[field].push(value);\n } else row[field] = value;\n }\n\n if (this._config.header) {\n if (j > this._fields.length)\n this.addError(\n 'FieldMismatch',\n 'TooManyFields',\n `Too many fields: expected ${this._fields.length} fields but parsed ${j}`,\n this._rowCounter + i\n );\n else if (j < this._fields.length)\n this.addError(\n 'FieldMismatch',\n 'TooFewFields',\n `Too few fields: expected ${this._fields.length} fields but parsed ${j}`,\n this._rowCounter + i\n );\n }\n\n return row;\n }\n\n // eslint-disable-next-line complexity, max-statements\n guessDelimiter(input, newline, skipEmptyLines, comments, delimitersToGuess) {\n let bestDelim;\n let bestDelta;\n let fieldCountPrevRow;\n\n delimitersToGuess = delimitersToGuess || [',', '\\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP];\n\n for (let i = 0; i < delimitersToGuess.length; i++) {\n const delim = delimitersToGuess[i];\n let avgFieldCount = 0;\n let delta = 0;\n let emptyLinesCount = 0;\n fieldCountPrevRow = undefined;\n\n const preview = new Parser({\n comments,\n delimiter: delim,\n newline,\n preview: 10\n }).parse(input);\n\n for (let j = 0; j < preview.data.length; j++) {\n if (skipEmptyLines && this.testEmptyLine(preview.data[j])) {\n emptyLinesCount++;\n continue;\n }\n const fieldCount = preview.data[j].length;\n avgFieldCount += fieldCount;\n\n if (typeof fieldCountPrevRow === 'undefined') {\n fieldCountPrevRow = 0;\n continue;\n } else if (fieldCount > 1) {\n delta += Math.abs(fieldCount - fieldCountPrevRow);\n fieldCountPrevRow = fieldCount;\n }\n }\n\n if (preview.data.length > 0) avgFieldCount /= preview.data.length - emptyLinesCount;\n\n if ((typeof bestDelta === 'undefined' || delta > bestDelta) && avgFieldCount > 1.99) {\n bestDelta = delta;\n bestDelim = delim;\n }\n }\n\n this._config.delimiter = bestDelim;\n\n return {\n successful: Boolean(bestDelim),\n bestDelimiter: bestDelim\n };\n }\n\n addError(type, code, msg, row?) {\n this._results.errors.push({\n type,\n code,\n message: msg,\n row\n });\n }\n}\n\nfunction guessLineEndings(input, quoteChar) {\n input = input.substr(0, 1024 * 1024); // max length 1 MB\n // Replace all the text inside quotes\n const re = new RegExp(`${escapeRegExp(quoteChar)}([^]*?)${escapeRegExp(quoteChar)}`, 'gm');\n input = input.replace(re, '');\n\n const r = input.split('\\r');\n\n const n = input.split('\\n');\n\n const nAppearsFirst = n.length > 1 && n[0].length < r[0].length;\n\n if (r.length === 1 || nAppearsFirst) return '\\n';\n\n let numWithN = 0;\n for (let i = 0; i < r.length; i++) {\n if (r[i][0] === '\\n') numWithN++;\n }\n\n return numWithN >= r.length / 2 ? '\\r\\n' : '\\r';\n}\n\n/** https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions */\nfunction escapeRegExp(string) {\n return string.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&'); // $& means the whole matched string\n}\n\n/** The core parser implements speedy and correct CSV parsing */\n// eslint-disable-next-line complexity, max-statements\nexport function Parser(config: CSVParserConfig = {}) {\n // Unpack the config object\n let delim = config.delimiter;\n let newline = config.newline;\n let comments = config.comments;\n const step = config.step;\n const preview = config.preview;\n const fastMode = config.fastMode;\n let quoteChar;\n /** Allows for no quoteChar by setting quoteChar to undefined in config */\n if (config.quoteChar === undefined) {\n quoteChar = '\"';\n } else {\n quoteChar = config.quoteChar;\n }\n let escapeChar = quoteChar;\n if (config.escapeChar !== undefined) {\n escapeChar = config.escapeChar;\n }\n\n // Delimiter must be valid\n if (typeof delim !== 'string' || Papa.BAD_DELIMITERS.indexOf(delim) > -1) delim = ',';\n\n // Comment character must be valid\n if (comments === delim) {\n throw new Error('Comment character same as delimiter');\n } else if (comments === true) {\n comments = '#';\n } else if (typeof comments !== 'string' || Papa.BAD_DELIMITERS.indexOf(comments) > -1) {\n comments = false;\n }\n\n // Newline must be valid: \\r, \\n, or \\r\\n\n if (newline !== '\\n' && newline !== '\\r' && newline !== '\\r\\n') newline = '\\n';\n\n // We're gonna need these at the Parser scope\n let cursor = 0;\n let aborted = false;\n\n // @ts-expect-error\n // eslint-disable-next-line complexity, max-statements\n this.parse = function (input, baseIndex, ignoreLastRow) {\n // For some reason, in Chrome, this speeds things up (!?)\n if (typeof input !== 'string') throw new Error('Input must be a string');\n\n // We don't need to compute some of these every time parse() is called,\n // but having them in a more local scope seems to perform better\n const inputLen = input.length;\n const delimLen = delim.length;\n const newlineLen = newline.length;\n // @ts-expect-error\n const commentsLen = comments.length;\n const stepIsFunction = isFunction(step);\n\n // Establish starting state\n cursor = 0;\n let data: any[][] | Record<string, any> = [];\n let errors: any[] = [];\n let row: any[] | Record<string, any> = [];\n let lastCursor: number = 0;\n\n if (!input) return returnable();\n\n if (fastMode || (fastMode !== false && input.indexOf(quoteChar) === -1)) {\n const rows = input.split(newline);\n for (let i = 0; i < rows.length; i++) {\n const row = rows[i];\n cursor += row.length;\n if (i !== rows.length - 1) cursor += newline.length;\n else if (ignoreLastRow) return returnable();\n if (comments && row.substr(0, commentsLen) === comments) continue;\n if (stepIsFunction) {\n data = [];\n pushRow(row.split(delim));\n doStep();\n if (aborted) return returnable();\n } else pushRow(row.split(delim));\n if (preview && i >= preview) {\n data = data.slice(0, preview);\n return returnable(true);\n }\n }\n return returnable();\n }\n\n let nextDelim = input.indexOf(delim, cursor);\n let nextNewline = input.indexOf(newline, cursor);\n const quoteCharRegex = new RegExp(escapeRegExp(escapeChar) + escapeRegExp(quoteChar), 'g');\n let quoteSearch;\n\n // Parser loop\n for (;;) {\n // Field has opening quote\n if (input[cursor] === quoteChar) {\n // Start our search for the closing quote where the cursor is\n quoteSearch = cursor;\n\n // Skip the opening quote\n cursor++;\n\n for (;;) {\n // Find closing quote\n quoteSearch = input.indexOf(quoteChar, quoteSearch + 1);\n\n // No other quotes are found - no other delimiters\n if (quoteSearch === -1) {\n if (!ignoreLastRow) {\n // No closing quote... what a pity\n errors.push({\n type: 'Quotes',\n code: 'MissingQuotes',\n message: 'Quoted field unterminated',\n row: data.length, // row has yet to be inserted\n index: cursor\n });\n }\n return finish();\n }\n\n // Closing quote at EOF\n if (quoteSearch === inputLen - 1) {\n const value = input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar);\n return finish(value);\n }\n\n // If this quote is escaped, it's part of the data; skip it\n // If the quote character is the escape character, then check if the next character is the escape character\n if (quoteChar === escapeChar && input[quoteSearch + 1] === escapeChar) {\n quoteSearch++;\n continue;\n }\n\n // If the quote character is not the escape character, then check if the previous character was the escape character\n if (\n quoteChar !== escapeChar &&\n quoteSearch !== 0 &&\n input[quoteSearch - 1] === escapeChar\n ) {\n continue;\n }\n\n // Check up to nextDelim or nextNewline, whichever is closest\n const checkUpTo = nextNewline === -1 ? nextDelim : Math.min(nextDelim, nextNewline);\n const spacesBetweenQuoteAndDelimiter = extraSpaces(checkUpTo);\n\n // Closing quote followed by delimiter or 'unnecessary spaces + delimiter'\n if (input[quoteSearch + 1 + spacesBetweenQuoteAndDelimiter] === delim) {\n row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar));\n cursor = quoteSearch + 1 + spacesBetweenQuoteAndDelimiter + delimLen;\n nextDelim = input.indexOf(delim, cursor);\n nextNewline = input.indexOf(newline, cursor);\n\n if (stepIsFunction) {\n doStep();\n if (aborted) return returnable();\n }\n\n if (preview && data.length >= preview) return returnable(true);\n\n break;\n }\n\n const spacesBetweenQuoteAndNewLine = extraSpaces(nextNewline);\n\n // Closing quote followed by newline or 'unnecessary spaces + newLine'\n if (\n input.substr(quoteSearch + 1 + spacesBetweenQuoteAndNewLine, newlineLen) === newline\n ) {\n row.push(input.substring(cursor, quoteSearch).replace(quoteCharRegex, quoteChar));\n saveRow(quoteSearch + 1 + spacesBetweenQuoteAndNewLine + newlineLen);\n nextDelim = input.indexOf(delim, cursor); // because we may have skipped the nextDelim in the quoted field\n\n if (stepIsFunction) {\n doStep();\n if (aborted) return returnable();\n }\n\n if (preview && data.length >= preview) return returnable(true);\n\n break;\n }\n\n // Checks for valid closing quotes are complete (escaped quotes or quote followed by EOF/delimiter/newline) -- assume these quotes are part of an invalid text string\n errors.push({\n type: 'Quotes',\n code: 'InvalidQuotes',\n message: 'Trailing quote on quoted field is malformed',\n row: data.length, // row has yet to be inserted\n index: cursor\n });\n\n quoteSearch++;\n continue;\n }\n\n if (stepIsFunction) {\n doStep();\n if (aborted) return returnable();\n }\n\n if (preview && data.length >= preview) return returnable(true);\n continue;\n }\n\n // Comment found at start of new line\n if (comments && row.length === 0 && input.substr(cursor, commentsLen) === comments) {\n if (nextNewline === -1)\n // Comment ends at EOF\n return returnable();\n cursor = nextNewline + newlineLen;\n nextNewline = input.indexOf(newline, cursor);\n nextDelim = input.indexOf(delim, cursor);\n continue;\n }\n\n // Next delimiter comes before next newline, so we've reached end of field\n if (nextDelim !== -1 && (nextDelim < nextNewline || nextNewline === -1)) {\n row.push(input.substring(cursor, nextDelim));\n cursor = nextDelim + delimLen;\n nextDelim = input.indexOf(delim, cursor);\n continue;\n }\n\n // End of row\n if (nextNewline !== -1) {\n row.push(input.substring(cursor, nextNewline));\n saveRow(nextNewline + newlineLen);\n\n if (stepIsFunction) {\n doStep();\n if (aborted) return returnable();\n }\n\n if (preview && data.length >= preview) return returnable(true);\n\n continue;\n }\n\n break;\n }\n\n return finish();\n\n function pushRow(row) {\n data.push(row);\n lastCursor = cursor;\n }\n\n /**\n * checks if there are extra spaces after closing quote and given index without any text\n * if Yes, returns the number of spaces\n */\n function extraSpaces(index) {\n let spaceLength = 0;\n if (index !== -1) {\n const textBetweenClosingQuoteAndIndex = input.substring(quoteSearch + 1, index);\n if (textBetweenClosingQuoteAndIndex && textBetweenClosingQuoteAndIndex.trim() === '') {\n spaceLength = textBetweenClosingQuoteAndIndex.length;\n }\n }\n return spaceLength;\n }\n\n /**\n * Appends the remaining input from cursor to the end into\n * row, saves the row, calls step, and returns the results.\n */\n function finish(value?: any) {\n if (ignoreLastRow) return returnable();\n if (typeof value === 'undefined') value = input.substr(cursor);\n row.push(value);\n cursor = inputLen; // important in case parsing is paused\n pushRow(row);\n if (stepIsFunction) doStep();\n return returnable();\n }\n\n /**\n * Appends the current row to the results. It sets the cursor\n * to newCursor and finds the nextNewline. The caller should\n * take care to execute user's step function and check for\n * preview and end parsing if necessary.\n */\n function saveRow(newCursor) {\n cursor = newCursor;\n pushRow(row);\n row = [];\n nextNewline = input.indexOf(newline, cursor);\n }\n\n /** Returns an object with the results, errors, and meta. */\n function returnable(stopped?: boolean, step?) {\n const isStep = step || false;\n return {\n data: isStep ? data[0] : data,\n errors,\n meta: {\n delimiter: delim,\n linebreak: newline,\n aborted,\n truncated: Boolean(stopped),\n cursor: lastCursor + (baseIndex || 0)\n }\n };\n }\n\n /** Executes the user's step function and resets data & errors. */\n function doStep() {\n // @ts-expect-error\n step(returnable(undefined, true));\n data = [];\n errors = [];\n }\n };\n\n /** Sets the abort flag */\n // @ts-expect-error\n this.abort = function () {\n aborted = true;\n };\n\n /** Gets the cursor position */\n // @ts-expect-error\n this.getCharIndex = function () {\n return cursor;\n };\n}\n\n/** Makes a deep copy of an array or object (mostly) */\nfunction copy(obj) {\n if (typeof obj !== 'object' || obj === null) return obj;\n const cpy = Array.isArray(obj) ? [] : {};\n for (const key in obj) cpy[key] = copy(obj[key]);\n return cpy;\n}\n\nfunction isFunction(func: unknown): func is Function {\n return typeof func === 'function';\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n// Copyright (c) 2015 Matthew Holt\n\n// This is a fork of papaparse v5.0.0-beta.0 under MIT license\n// https://github.com/mholt/PapaParse\n\nimport {Papa} from './papa-constants';\n\nexport type CSVWriterConfig = {\n chunk?: boolean;\n chunkSize?: number | null;\n preview?: number;\n newline?: string;\n comments?: boolean;\n skipEmptyLines?: boolean | 'greedy';\n delimitersToGuess?: string[];\n quotes?: string[] | boolean;\n quoteChar?: string;\n escapeChar?: string;\n delimiter?: string;\n // Convert numbers and boolean values in rows from strings\n fastMode?: boolean;\n\n dynamicTyping?: boolean | {};\n dynamicTypingFunction?: Function;\n step?: Function;\n transform?: Function;\n\n header?: any;\n columns?: any;\n};\n\n// eslint-disable-next-line complexity, max-statements\nexport function JsonToCsv(_input, _config: CSVWriterConfig = {}) {\n // Default configuration\n\n /** whether to surround every datum with quotes */\n let _quotes: string[] | boolean = false;\n\n /** whether to write headers */\n let _writeHeader = true;\n\n /** delimiting character(s) */\n let _delimiter = ',';\n\n /** newline character(s) */\n let _newline = '\\r\\n';\n\n /** quote character */\n let _quoteChar = '\"';\n\n /** escaped quote character, either \"\" or <config.escapeChar>\" */\n let _escapedQuote = _quoteChar + _quoteChar;\n\n /** whether to skip empty lines */\n let _skipEmptyLines: 'greedy' | boolean = false;\n\n /** the columns (keys) we expect when we unparse objects */\n let _columns: any = null;\n\n unpackConfig();\n\n const quoteCharRegex = new RegExp(escapeRegExp(_quoteChar), 'g');\n\n if (typeof _input === 'string') _input = JSON.parse(_input);\n\n if (Array.isArray(_input)) {\n if (!_input.length || Array.isArray(_input[0])) {\n return serialize(null, _input, _skipEmptyLines);\n } else if (typeof _input[0] === 'object') {\n return serialize(_columns || Object.keys(_input[0]), _input, _skipEmptyLines);\n }\n } else if (typeof _input === 'object') {\n if (typeof _input.data === 'string') {\n _input.data = JSON.parse(_input.data);\n }\n\n if (Array.isArray(_input.data)) {\n if (!_input.fields) {\n _input.fields = _input.meta && _input.meta.fields;\n }\n\n if (!_input.fields) {\n _input.fields = Array.isArray(_input.data[0]) ? _input.fields : Object.keys(_input.data[0]);\n }\n\n if (!Array.isArray(_input.data[0]) && typeof _input.data[0] !== 'object') {\n _input.data = [_input.data]; // handles input like [1,2,3] or ['asdf']\n }\n }\n\n return serialize(_input.fields || [], _input.data || [], _skipEmptyLines);\n }\n\n // Default (any valid paths should return before this)\n throw new Error('Unable to serialize unrecognized input');\n\n // eslint-disable-next-line complexity\n function unpackConfig() {\n if (typeof _config !== 'object') return;\n\n if (\n typeof _config.delimiter === 'string' &&\n !Papa.BAD_DELIMITERS.filter(function (value) {\n return _config.delimiter?.indexOf(value) !== -1;\n }).length\n ) {\n _delimiter = _config.delimiter;\n }\n\n if (typeof _config.quotes === 'boolean' || Array.isArray(_config.quotes))\n _quotes = _config.quotes;\n\n if (typeof _config.skipEmptyLines === 'boolean' || _config.skipEmptyLines === 'greedy')\n _skipEmptyLines = _config.skipEmptyLines;\n\n if (typeof _config.newline === 'string') _newline = _config.newline;\n\n if (typeof _config.quoteChar === 'string') _quoteChar = _config.quoteChar;\n\n if (typeof _config.header === 'boolean') _writeHeader = _config.header;\n\n if (Array.isArray(_config.columns)) {\n if (_config.columns.length === 0) throw new Error('Option columns is empty');\n\n _columns = _config.columns;\n }\n\n if (_config.escapeChar !== undefined) {\n _escapedQuote = _config.escapeChar + _quoteChar;\n }\n }\n\n /** The double for loop that iterates the data and writes out a CSV string including header row */\n // eslint-disable-next-line complexity, max-statements\n function serialize(fields, data, skipEmptyLines) {\n let csv = '';\n\n if (typeof fields === 'string') fields = JSON.parse(fields);\n if (typeof data === 'string') data = JSON.parse(data);\n\n const hasHeader = Array.isArray(fields) && fields.length > 0;\n const dataKeyedByField = !Array.isArray(data[0]);\n\n // If there a header row, write it first\n if (hasHeader && _writeHeader) {\n for (let i = 0; i < fields.length; i++) {\n if (i > 0) csv += _delimiter;\n csv += safe(fields[i], i);\n }\n if (data.length > 0) csv += _newline;\n }\n\n // Then write out the data\n for (let row = 0; row < data.length; row++) {\n const maxCol = hasHeader ? fields.length : data[row].length;\n\n let emptyLine = false;\n const nullLine = hasHeader ? Object.keys(data[row]).length === 0 : data[row].length === 0;\n if (skipEmptyLines && !hasHeader) {\n emptyLine =\n skipEmptyLines === 'greedy'\n ? data[row].join('').trim() === ''\n : data[row].length === 1 && data[row][0].length === 0;\n }\n if (skipEmptyLines === 'greedy' && hasHeader) {\n const line: string[] = [];\n for (let c = 0; c < maxCol; c++) {\n const cx = dataKeyedByField ? fields[c] : c;\n line.push(data[row][cx]);\n }\n emptyLine = line.join('').trim() === '';\n }\n if (!emptyLine) {\n for (let col = 0; col < maxCol; col++) {\n if (col > 0 && !nullLine) csv += _delimiter;\n const colIdx = hasHeader && dataKeyedByField ? fields[col] : col;\n csv += safe(data[row][colIdx], col);\n }\n if (row < data.length - 1 && (!skipEmptyLines || (maxCol > 0 && !nullLine))) {\n csv += _newline;\n }\n }\n }\n return csv;\n }\n\n /** Encloses a value around quotes if needed (makes a value safe for CSV insertion) */\n // eslint-disable-next-line complexity\n function safe(str, col) {\n if (typeof str === 'undefined' || str === null) return '';\n\n if (str.constructor === Date) return JSON.stringify(str).slice(1, 25);\n\n str = str.toString().replace(quoteCharRegex, _escapedQuote);\n\n const needsQuotes =\n (typeof _quotes === 'boolean' && _quotes) ||\n (Array.isArray(_quotes) && _quotes[col]) ||\n hasAny(str, Papa.BAD_DELIMITERS) ||\n str.indexOf(_delimiter) > -1 ||\n str.charAt(0) === ' ' ||\n str.charAt(str.length - 1) === ' ';\n\n return needsQuotes ? _quoteChar + str + _quoteChar : str;\n }\n\n function hasAny(str, substrings) {\n for (let i = 0; i < substrings.length; i++) if (str.indexOf(substrings[i]) > -1) return true;\n return false;\n }\n}\n\n/** https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions */\nfunction escapeRegExp(string) {\n return string.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&'); // $& means the whole matched string\n}\n", "// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n// Copyright (c) 2015 Matthew Holt\n\n// This is a fork of papaparse v5.0.0-beta.0 under MIT license\n// https://github.com/mholt/PapaParse\n\n// FORK SUMMARY:\n// - Adopt ES6 exports\n// - Implement new AsyncIteratorStreamer\n// - Remove non Async Iterator streamers (can all be handled by new streamer)\n// - Remove unused Worker support (loaders.gl worker system used instead)\n// - Remove unused jQuery plugin support\n\nimport {CsvToJson, Parser, ParserHandle,