@loaders.gl/shapefile
Version:
Loader for the Shapefile Format
635 lines (626 loc) • 19.8 kB
JavaScript
"use strict";
(() => {
// ../worker-utils/src/lib/node/worker_threads-browser.ts
var parentPort = null;
// ../worker-utils/src/lib/worker-utils/get-transfer-list.ts
function getTransferList(object, recursive = true, transfers) {
const transfersSet = transfers || /* @__PURE__ */ new Set();
if (!object) {
} else if (isTransferable(object)) {
transfersSet.add(object);
} else if (isTransferable(object.buffer)) {
transfersSet.add(object.buffer);
} else if (ArrayBuffer.isView(object)) {
} else if (recursive && typeof object === "object") {
for (const key in object) {
getTransferList(object[key], recursive, transfersSet);
}
}
return transfers === void 0 ? Array.from(transfersSet) : [];
}
function isTransferable(object) {
if (!object) {
return false;
}
if (object instanceof ArrayBuffer) {
return true;
}
if (typeof MessagePort !== "undefined" && object instanceof MessagePort) {
return true;
}
if (typeof ImageBitmap !== "undefined" && object instanceof ImageBitmap) {
return true;
}
if (typeof OffscreenCanvas !== "undefined" && object instanceof OffscreenCanvas) {
return true;
}
return false;
}
// ../worker-utils/src/lib/worker-farm/worker-body.ts
async function getParentPort() {
return parentPort;
}
var onMessageWrapperMap = /* @__PURE__ */ new Map();
var WorkerBody = class {
/** Check that we are actually in a worker thread */
static async inWorkerThread() {
return typeof self !== "undefined" || Boolean(await getParentPort());
}
/*
* (type: WorkerMessageType, payload: WorkerMessagePayload) => any
*/
static set onmessage(onMessage) {
async function handleMessage(message) {
const parentPort2 = await getParentPort();
const { type, payload } = parentPort2 ? message : message.data;
onMessage(type, payload);
}
getParentPort().then((parentPort2) => {
if (parentPort2) {
parentPort2.on("message", (message) => {
handleMessage(message);
});
parentPort2.on("exit", () => console.debug("Node worker closing"));
} else {
globalThis.onmessage = handleMessage;
}
});
}
static async addEventListener(onMessage) {
let onMessageWrapper = onMessageWrapperMap.get(onMessage);
if (!onMessageWrapper) {
onMessageWrapper = async (message) => {
if (!isKnownMessage(message)) {
return;
}
const parentPort3 = await getParentPort();
const { type, payload } = parentPort3 ? message : message.data;
onMessage(type, payload);
};
}
const parentPort2 = await getParentPort();
if (parentPort2) {
console.error("not implemented");
} else {
globalThis.addEventListener("message", onMessageWrapper);
}
}
static async removeEventListener(onMessage) {
const onMessageWrapper = onMessageWrapperMap.get(onMessage);
onMessageWrapperMap.delete(onMessage);
const parentPort2 = await getParentPort();
if (parentPort2) {
console.error("not implemented");
} else {
globalThis.removeEventListener("message", onMessageWrapper);
}
}
/**
* Send a message from a worker to creating thread (main thread)
* @param type
* @param payload
*/
static async postMessage(type, payload) {
const data = { source: "loaders.gl", type, payload };
const transferList = getTransferList(payload);
const parentPort2 = await getParentPort();
if (parentPort2) {
parentPort2.postMessage(data, transferList);
} else {
globalThis.postMessage(data, transferList);
}
}
};
function isKnownMessage(message) {
const { type, data } = message;
return type === "message" && data && typeof data.source === "string" && data.source.startsWith("loaders.gl");
}
// ../loader-utils/src/lib/worker-loader-utils/create-loader-worker.ts
var requestId = 0;
async function createLoaderWorker(loader) {
if (!await WorkerBody.inWorkerThread()) {
return;
}
WorkerBody.onmessage = async (type, payload) => {
switch (type) {
case "process":
try {
const { input, options = {}, context = {} } = payload;
const result = await parseData({
loader,
arrayBuffer: input,
options,
// @ts-expect-error fetch missing
context: {
...context,
_parse: parseOnMainThread
}
});
WorkerBody.postMessage("done", { result });
} catch (error) {
const message = error instanceof Error ? error.message : "";
WorkerBody.postMessage("error", { error: message });
}
break;
default:
}
};
}
function parseOnMainThread(arrayBuffer, loader, options, context) {
return new Promise((resolve, reject) => {
const id = requestId++;
const onMessage = (type, payload2) => {
if (payload2.id !== id) {
return;
}
switch (type) {
case "done":
WorkerBody.removeEventListener(onMessage);
resolve(payload2.result);
break;
case "error":
WorkerBody.removeEventListener(onMessage);
reject(payload2.error);
break;
default:
}
};
WorkerBody.addEventListener(onMessage);
const payload = { id, input: arrayBuffer, options };
WorkerBody.postMessage("process", payload);
});
}
async function parseData({
loader,
arrayBuffer,
options,
context
}) {
let data;
let parser;
if (loader.parseSync || loader.parse) {
data = arrayBuffer;
parser = loader.parseSync || loader.parse;
} else if (loader.parseTextSync) {
const textDecoder = new TextDecoder();
data = textDecoder.decode(arrayBuffer);
parser = loader.parseTextSync;
} else {
throw new Error(`Could not load data with ${loader.name} loader`);
}
options = {
...options,
modules: loader && loader.options && loader.options.modules || {},
core: {
...options.core,
worker: false
}
};
return await parser(data, { ...options }, context, loader);
}
// ../loader-utils/src/lib/iterators/async-iteration.ts
async function* toArrayBufferIterator(asyncIterator) {
for await (const chunk of asyncIterator) {
yield copyToArrayBuffer(chunk);
}
}
function copyToArrayBuffer(chunk) {
if (chunk instanceof ArrayBuffer) {
return chunk;
}
if (ArrayBuffer.isView(chunk)) {
const { buffer, byteOffset, byteLength } = chunk;
return copyFromBuffer(buffer, byteOffset, byteLength);
}
return copyFromBuffer(chunk);
}
function copyFromBuffer(buffer, byteOffset = 0, byteLength = buffer.byteLength - byteOffset) {
const view = new Uint8Array(buffer, byteOffset, byteLength);
const copy = new Uint8Array(view.length);
copy.set(view);
return copy.buffer;
}
// src/lib/streaming/binary-chunk-reader.ts
var BinaryChunkReader = class {
offset;
arrayBuffers;
ended;
maxRewindBytes;
constructor(options) {
const { maxRewindBytes = 0 } = options || {};
this.offset = 0;
this.arrayBuffers = [];
this.ended = false;
this.maxRewindBytes = maxRewindBytes;
}
/**
* @param arrayBuffer
*/
write(arrayBuffer) {
this.arrayBuffers.push(arrayBuffer);
}
end() {
this.arrayBuffers = [];
this.ended = true;
}
/**
* Has enough bytes available in array buffers
*
* @param bytes Number of bytes
* @return boolean
*/
hasAvailableBytes(bytes) {
let bytesAvailable = -this.offset;
for (const arrayBuffer of this.arrayBuffers) {
bytesAvailable += arrayBuffer.byteLength;
if (bytesAvailable >= bytes) {
return true;
}
}
return false;
}
/**
* Find offsets of byte ranges within this.arrayBuffers
*
* @param bytes Byte length to read
* @return Arrays with byte ranges pointing to this.arrayBuffers, Output type is nested array, e.g. [ [0, [1, 2]], ...]
*/
findBufferOffsets(bytes) {
let offset = -this.offset;
const selectedBuffers = [];
for (let i = 0; i < this.arrayBuffers.length; i++) {
const buf = this.arrayBuffers[i];
if (offset + buf.byteLength <= 0) {
offset += buf.byteLength;
continue;
}
const start = offset <= 0 ? Math.abs(offset) : 0;
let end;
if (start + bytes <= buf.byteLength) {
end = start + bytes;
selectedBuffers.push([i, [start, end]]);
return selectedBuffers;
}
end = buf.byteLength;
selectedBuffers.push([i, [start, end]]);
bytes -= buf.byteLength - start;
offset += buf.byteLength;
}
return null;
}
/**
* Get the required number of bytes from the iterator
*
* @param bytes Number of bytes
* @return DataView with data
*/
getDataView(bytes) {
const bufferOffsets = this.findBufferOffsets(bytes);
if (!bufferOffsets && this.ended) {
throw new Error("binary data exhausted");
}
if (!bufferOffsets) {
return null;
}
if (bufferOffsets.length === 1) {
const [bufferIndex, [start, end]] = bufferOffsets[0];
const arrayBuffer = this.arrayBuffers[bufferIndex];
const view2 = new DataView(arrayBuffer, start, end - start);
this.offset += bytes;
this.disposeBuffers();
return view2;
}
const view = new DataView(this._combineArrayBuffers(bufferOffsets));
this.offset += bytes;
this.disposeBuffers();
return view;
}
/**
* Dispose of old array buffers
*/
disposeBuffers() {
while (this.arrayBuffers.length > 0 && this.offset - this.maxRewindBytes >= this.arrayBuffers[0].byteLength) {
this.offset -= this.arrayBuffers[0].byteLength;
this.arrayBuffers.shift();
}
}
/**
* Copy multiple ArrayBuffers into one contiguous ArrayBuffer
*
* In contrast to concatenateArrayBuffers, this only copies the necessary
* portions of the source arrays, rather than first copying the entire arrays
* then taking a part of them.
*
* @param bufferOffsets List of internal array offsets
* @return New contiguous ArrayBuffer
*/
_combineArrayBuffers(bufferOffsets) {
let byteLength = 0;
for (const bufferOffset of bufferOffsets) {
const [start, end] = bufferOffset[1];
byteLength += end - start;
}
const result = new Uint8Array(byteLength);
let resultOffset = 0;
for (const bufferOffset of bufferOffsets) {
const [bufferIndex, [start, end]] = bufferOffset;
const sourceArray = new Uint8Array(this.arrayBuffers[bufferIndex]);
result.set(sourceArray.subarray(start, end), resultOffset);
resultOffset += end - start;
}
return result.buffer;
}
/**
* @param bytes
*/
skip(bytes) {
this.offset += bytes;
}
/**
* @param bytes
*/
rewind(bytes) {
this.offset -= bytes;
}
};
// src/lib/parsers/parse-dbf.ts
var LITTLE_ENDIAN = true;
var DBF_HEADER_SIZE = 32;
var DBFParser = class {
binaryReader = new BinaryChunkReader();
textDecoder;
state = 0 /* START */;
result = {
data: []
};
constructor(options) {
this.textDecoder = new TextDecoder(options.encoding);
}
/**
* @param arrayBuffer
*/
write(arrayBuffer) {
this.binaryReader.write(arrayBuffer);
this.state = parseState(this.state, this.result, this.binaryReader, this.textDecoder);
}
end() {
this.binaryReader.end();
this.state = parseState(this.state, this.result, this.binaryReader, this.textDecoder);
if (this.state !== 3 /* END */) {
this.state = 4 /* ERROR */;
this.result.error = "DBF incomplete file";
}
}
};
function parseDBF(arrayBuffer, options = {}) {
const { encoding = "latin1" } = options.dbf || {};
const dbfParser = new DBFParser({ encoding });
dbfParser.write(arrayBuffer);
dbfParser.end();
const { data, schema } = dbfParser.result;
const shape = options?.dbf?.shape;
switch (shape) {
case "object-row-table": {
const table = {
shape: "object-row-table",
schema,
data
};
return table;
}
case "table":
return { schema, rows: data };
case "rows":
default:
return data;
}
}
async function* parseDBFInBatches(asyncIterator, options = {}) {
const { encoding = "latin1" } = options.dbf || {};
const parser = new DBFParser({ encoding });
let headerReturned = false;
for await (const arrayBuffer of toArrayBufferIterator(asyncIterator)) {
parser.write(arrayBuffer);
if (!headerReturned && parser.result.dbfHeader) {
headerReturned = true;
yield parser.result.dbfHeader;
}
if (parser.result.data.length > 0) {
yield parser.result.data;
parser.result.data = [];
}
}
parser.end();
if (parser.result.data.length > 0) {
yield parser.result.data;
}
}
function parseState(state, result, binaryReader, textDecoder) {
while (true) {
try {
switch (state) {
case 4 /* ERROR */:
case 3 /* END */:
return state;
case 0 /* START */:
const dataView = binaryReader.getDataView(DBF_HEADER_SIZE);
if (!dataView) {
return state;
}
result.dbfHeader = parseDBFHeader(dataView);
result.progress = {
bytesUsed: 0,
rowsTotal: result.dbfHeader.nRecords,
rows: 0
};
state = 1 /* FIELD_DESCRIPTORS */;
break;
case 1 /* FIELD_DESCRIPTORS */:
const fieldDescriptorView = binaryReader.getDataView(
// @ts-ignore
result.dbfHeader.headerLength - DBF_HEADER_SIZE
);
if (!fieldDescriptorView) {
return state;
}
result.dbfFields = parseFieldDescriptors(fieldDescriptorView, textDecoder);
result.schema = {
fields: result.dbfFields.map((dbfField) => makeField(dbfField)),
metadata: {}
};
state = 2 /* FIELD_PROPERTIES */;
binaryReader.skip(1);
break;
case 2 /* FIELD_PROPERTIES */:
const { recordLength = 0, nRecords = 0 } = result?.dbfHeader || {};
while (result.data.length < nRecords) {
const recordView = binaryReader.getDataView(recordLength - 1);
if (!recordView) {
return state;
}
binaryReader.skip(1);
const row = parseRow(recordView, result.dbfFields, textDecoder);
result.data.push(row);
result.progress.rows = result.data.length;
}
state = 3 /* END */;
break;
default:
state = 4 /* ERROR */;
result.error = `illegal parser state ${state}`;
return state;
}
} catch (error) {
state = 4 /* ERROR */;
result.error = `DBF parsing failed: ${error.message}`;
return state;
}
}
}
function parseDBFHeader(headerView) {
return {
// Last updated date
year: headerView.getUint8(1) + 1900,
month: headerView.getUint8(2),
day: headerView.getUint8(3),
// Number of records in data file
nRecords: headerView.getUint32(4, LITTLE_ENDIAN),
// Length of header in bytes
headerLength: headerView.getUint16(8, LITTLE_ENDIAN),
// Length of each record
recordLength: headerView.getUint16(10, LITTLE_ENDIAN),
// Not sure if this is usually set
languageDriver: headerView.getUint8(29)
};
}
function parseFieldDescriptors(view, textDecoder) {
const nFields = (view.byteLength - 1) / 32;
const fields = [];
let offset = 0;
for (let i = 0; i < nFields; i++) {
const name = textDecoder.decode(new Uint8Array(view.buffer, view.byteOffset + offset, 11)).replace(/\u0000/g, "");
fields.push({
name,
dataType: String.fromCharCode(view.getUint8(offset + 11)),
fieldLength: view.getUint8(offset + 16),
decimal: view.getUint8(offset + 17)
});
offset += 32;
}
return fields;
}
function parseRow(view, fields, textDecoder) {
const out = {};
let offset = 0;
for (const field of fields) {
const text = textDecoder.decode(
new Uint8Array(view.buffer, view.byteOffset + offset, field.fieldLength)
);
out[field.name] = parseField(text, field.dataType);
offset += field.fieldLength;
}
return out;
}
function parseField(text, dataType) {
switch (dataType) {
case "B":
return parseNumber(text);
case "C":
return parseCharacter(text);
case "F":
return parseNumber(text);
case "N":
return parseNumber(text);
case "O":
return parseNumber(text);
case "D":
return parseDate(text);
case "L":
return parseBoolean(text);
default:
throw new Error("Unsupported data type");
}
}
function parseDate(str) {
return Date.UTC(str.slice(0, 4), parseInt(str.slice(4, 6), 10) - 1, str.slice(6, 8));
}
function parseBoolean(value) {
return /^[nf]$/i.test(value) ? false : /^[yt]$/i.test(value) ? true : null;
}
function parseNumber(text) {
const number = parseFloat(text);
return isNaN(number) ? null : number;
}
function parseCharacter(text) {
return text.trim() || null;
}
function makeField({ name, dataType, fieldLength, decimal }) {
switch (dataType) {
case "B":
return { name, type: "float64", nullable: true, metadata: {} };
case "C":
return { name, type: "utf8", nullable: true, metadata: {} };
case "F":
return { name, type: "float64", nullable: true, metadata: {} };
case "N":
return { name, type: "float64", nullable: true, metadata: {} };
case "O":
return { name, type: "float64", nullable: true, metadata: {} };
case "D":
return { name, type: "timestamp-millisecond", nullable: true, metadata: {} };
case "L":
return { name, type: "bool", nullable: true, metadata: {} };
default:
throw new Error("Unsupported data type");
}
}
// src/dbf-loader.ts
var VERSION = true ? "4.4.4" : "latest";
var DBFWorkerLoader = {
name: "DBF",
dataType: null,
batchType: null,
id: "dbf",
module: "shapefile",
version: VERSION,
worker: true,
category: "table",
extensions: ["dbf"],
mimeTypes: ["application/x-dbf"],
options: {
dbf: {
encoding: "latin1"
}
}
};
var DBFLoader = {
...DBFWorkerLoader,
parse: async (arrayBuffer, options) => parseDBF(arrayBuffer, options),
parseSync: parseDBF,
parseInBatches(arrayBufferIterator, options) {
return parseDBFInBatches(arrayBufferIterator, options);
}
};
// src/workers/dbf-worker.ts
createLoaderWorker(DBFLoader);
})();