@loaders.gl/shapefile
Version:
Loader for the Shapefile Format
1,414 lines (1,395 loc) • 44.8 kB
JavaScript
"use strict";
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
// dist/index.js
var dist_exports = {};
__export(dist_exports, {
DBFArrowLoader: () => DBFArrowLoader,
DBFArrowWorkerLoader: () => DBFArrowWorkerLoader,
DBFLoader: () => DBFLoader,
DBFWorkerLoader: () => DBFWorkerLoader,
SHPLoader: () => SHPLoader,
SHPWorkerLoader: () => SHPWorkerLoader,
ShapefileLoader: () => ShapefileLoader,
_BinaryChunkReader: () => BinaryChunkReader,
_BinaryReader: () => BinaryReader,
_zipBatchIterators: () => zipBatchIterators
});
module.exports = __toCommonJS(dist_exports);
// dist/lib/parsers/parse-shp.js
var import_loader_utils = require("@loaders.gl/loader-utils");
// dist/lib/streaming/binary-chunk-reader.js
var BinaryChunkReader = class {
offset;
arrayBuffers;
ended;
maxRewindBytes;
constructor(options) {
const { maxRewindBytes = 0 } = options || {};
this.offset = 0;
this.arrayBuffers = [];
this.ended = false;
this.maxRewindBytes = maxRewindBytes;
}
/**
* @param arrayBuffer
*/
write(arrayBuffer) {
this.arrayBuffers.push(arrayBuffer);
}
end() {
this.arrayBuffers = [];
this.ended = true;
}
/**
* Has enough bytes available in array buffers
*
* @param bytes Number of bytes
* @return boolean
*/
hasAvailableBytes(bytes) {
let bytesAvailable = -this.offset;
for (const arrayBuffer of this.arrayBuffers) {
bytesAvailable += arrayBuffer.byteLength;
if (bytesAvailable >= bytes) {
return true;
}
}
return false;
}
/**
* Find offsets of byte ranges within this.arrayBuffers
*
* @param bytes Byte length to read
* @return Arrays with byte ranges pointing to this.arrayBuffers, Output type is nested array, e.g. [ [0, [1, 2]], ...]
*/
findBufferOffsets(bytes) {
let offset = -this.offset;
const selectedBuffers = [];
for (let i = 0; i < this.arrayBuffers.length; i++) {
const buf = this.arrayBuffers[i];
if (offset + buf.byteLength <= 0) {
offset += buf.byteLength;
continue;
}
const start = offset <= 0 ? Math.abs(offset) : 0;
let end;
if (start + bytes <= buf.byteLength) {
end = start + bytes;
selectedBuffers.push([i, [start, end]]);
return selectedBuffers;
}
end = buf.byteLength;
selectedBuffers.push([i, [start, end]]);
bytes -= buf.byteLength - start;
offset += buf.byteLength;
}
return null;
}
/**
* Get the required number of bytes from the iterator
*
* @param bytes Number of bytes
* @return DataView with data
*/
getDataView(bytes) {
const bufferOffsets = this.findBufferOffsets(bytes);
if (!bufferOffsets && this.ended) {
throw new Error("binary data exhausted");
}
if (!bufferOffsets) {
return null;
}
if (bufferOffsets.length === 1) {
const [bufferIndex, [start, end]] = bufferOffsets[0];
const arrayBuffer = this.arrayBuffers[bufferIndex];
const view2 = new DataView(arrayBuffer, start, end - start);
this.offset += bytes;
this.disposeBuffers();
return view2;
}
const view = new DataView(this._combineArrayBuffers(bufferOffsets));
this.offset += bytes;
this.disposeBuffers();
return view;
}
/**
* Dispose of old array buffers
*/
disposeBuffers() {
while (this.arrayBuffers.length > 0 && this.offset - this.maxRewindBytes >= this.arrayBuffers[0].byteLength) {
this.offset -= this.arrayBuffers[0].byteLength;
this.arrayBuffers.shift();
}
}
/**
* Copy multiple ArrayBuffers into one contiguous ArrayBuffer
*
* In contrast to concatenateArrayBuffers, this only copies the necessary
* portions of the source arrays, rather than first copying the entire arrays
* then taking a part of them.
*
* @param bufferOffsets List of internal array offsets
* @return New contiguous ArrayBuffer
*/
_combineArrayBuffers(bufferOffsets) {
let byteLength = 0;
for (const bufferOffset of bufferOffsets) {
const [start, end] = bufferOffset[1];
byteLength += end - start;
}
const result = new Uint8Array(byteLength);
let resultOffset = 0;
for (const bufferOffset of bufferOffsets) {
const [bufferIndex, [start, end]] = bufferOffset;
const sourceArray = new Uint8Array(this.arrayBuffers[bufferIndex]);
result.set(sourceArray.subarray(start, end), resultOffset);
resultOffset += end - start;
}
return result.buffer;
}
/**
* @param bytes
*/
skip(bytes) {
this.offset += bytes;
}
/**
* @param bytes
*/
rewind(bytes) {
this.offset -= bytes;
}
};
// dist/lib/parsers/parse-shp-header.js
var LITTLE_ENDIAN = true;
var BIG_ENDIAN = false;
var SHP_MAGIC_NUMBER = 9994;
function parseSHPHeader(headerView) {
const header = {
magic: headerView.getInt32(0, BIG_ENDIAN),
// Length is stored as # of 2-byte words; multiply by 2 to get # of bytes
length: headerView.getInt32(24, BIG_ENDIAN) * 2,
version: headerView.getInt32(28, LITTLE_ENDIAN),
type: headerView.getInt32(32, LITTLE_ENDIAN),
bbox: {
minX: headerView.getFloat64(36, LITTLE_ENDIAN),
minY: headerView.getFloat64(44, LITTLE_ENDIAN),
minZ: headerView.getFloat64(68, LITTLE_ENDIAN),
minM: headerView.getFloat64(84, LITTLE_ENDIAN),
maxX: headerView.getFloat64(52, LITTLE_ENDIAN),
maxY: headerView.getFloat64(60, LITTLE_ENDIAN),
maxZ: headerView.getFloat64(76, LITTLE_ENDIAN),
maxM: headerView.getFloat64(92, LITTLE_ENDIAN)
}
};
if (header.magic !== SHP_MAGIC_NUMBER) {
console.error(`SHP file: bad magic number ${header.magic}`);
}
if (header.version !== 1e3) {
console.error(`SHP file: bad version ${header.version}`);
}
return header;
}
// dist/lib/parsers/parse-shp-geometry.js
var LITTLE_ENDIAN2 = true;
function parseRecord(view, options) {
const { _maxDimensions = 4 } = (options == null ? void 0 : options.shp) || {};
let offset = 0;
const type = view.getInt32(offset, LITTLE_ENDIAN2);
offset += Int32Array.BYTES_PER_ELEMENT;
switch (type) {
case 0:
return parseNull();
case 1:
return parsePoint(view, offset, Math.min(2, _maxDimensions));
case 3:
return parsePoly(view, offset, Math.min(2, _maxDimensions), "LineString");
case 5:
return parsePoly(view, offset, Math.min(2, _maxDimensions), "Polygon");
case 8:
return parseMultiPoint(view, offset, Math.min(2, _maxDimensions));
case 11:
return parsePoint(view, offset, Math.min(4, _maxDimensions));
case 13:
return parsePoly(view, offset, Math.min(4, _maxDimensions), "LineString");
case 15:
return parsePoly(view, offset, Math.min(4, _maxDimensions), "Polygon");
case 18:
return parseMultiPoint(view, offset, Math.min(4, _maxDimensions));
case 21:
return parsePoint(view, offset, Math.min(3, _maxDimensions));
case 23:
return parsePoly(view, offset, Math.min(3, _maxDimensions), "LineString");
case 25:
return parsePoly(view, offset, Math.min(3, _maxDimensions), "Polygon");
case 28:
return parseMultiPoint(view, offset, Math.min(3, _maxDimensions));
default:
throw new Error(`unsupported shape type: ${type}`);
}
}
function parseNull() {
return null;
}
function parsePoint(view, offset, dim) {
let positions;
[positions, offset] = parsePositions(view, offset, 1, dim);
return {
positions: { value: positions, size: dim },
type: "Point"
};
}
function parseMultiPoint(view, offset, dim) {
offset += 4 * Float64Array.BYTES_PER_ELEMENT;
const nPoints = view.getInt32(offset, LITTLE_ENDIAN2);
offset += Int32Array.BYTES_PER_ELEMENT;
let xyPositions = null;
let mPositions = null;
let zPositions = null;
[xyPositions, offset] = parsePositions(view, offset, nPoints, 2);
if (dim === 4) {
offset += 2 * Float64Array.BYTES_PER_ELEMENT;
[zPositions, offset] = parsePositions(view, offset, nPoints, 1);
}
if (dim >= 3) {
offset += 2 * Float64Array.BYTES_PER_ELEMENT;
[mPositions, offset] = parsePositions(view, offset, nPoints, 1);
}
const positions = concatPositions(xyPositions, mPositions, zPositions);
return {
positions: { value: positions, size: dim },
type: "Point"
};
}
function parsePoly(view, offset, dim, type) {
offset += 4 * Float64Array.BYTES_PER_ELEMENT;
const nParts = view.getInt32(offset, LITTLE_ENDIAN2);
offset += Int32Array.BYTES_PER_ELEMENT;
const nPoints = view.getInt32(offset, LITTLE_ENDIAN2);
offset += Int32Array.BYTES_PER_ELEMENT;
const bufferOffset = view.byteOffset + offset;
const bufferLength = nParts * Int32Array.BYTES_PER_ELEMENT;
const ringIndices = new Int32Array(nParts + 1);
ringIndices.set(new Int32Array(view.buffer.slice(bufferOffset, bufferOffset + bufferLength)));
ringIndices[nParts] = nPoints;
offset += nParts * Int32Array.BYTES_PER_ELEMENT;
let xyPositions = null;
let mPositions = null;
let zPositions = null;
[xyPositions, offset] = parsePositions(view, offset, nPoints, 2);
if (dim === 4) {
offset += 2 * Float64Array.BYTES_PER_ELEMENT;
[zPositions, offset] = parsePositions(view, offset, nPoints, 1);
}
if (dim >= 3) {
offset += 2 * Float64Array.BYTES_PER_ELEMENT;
[mPositions, offset] = parsePositions(view, offset, nPoints, 1);
}
const positions = concatPositions(xyPositions, mPositions, zPositions);
if (type === "LineString") {
return {
type,
positions: { value: positions, size: dim },
pathIndices: { value: ringIndices, size: 1 }
};
}
const polygonIndices = [];
for (let i = 1; i < ringIndices.length; i++) {
const startRingIndex = ringIndices[i - 1];
const endRingIndex = ringIndices[i];
const ring = xyPositions.subarray(startRingIndex * 2, endRingIndex * 2);
const sign = getWindingDirection(ring);
if (sign > 0) {
polygonIndices.push(startRingIndex);
}
}
polygonIndices.push(nPoints);
return {
type,
positions: { value: positions, size: dim },
primitivePolygonIndices: { value: ringIndices, size: 1 },
// TODO: Dynamically choose Uint32Array over Uint16Array only when
// necessary. I believe the implementation requires nPoints to be the
// largest value in the array, so you should be able to use Uint32Array only
// when nPoints > 65535.
polygonIndices: { value: new Uint32Array(polygonIndices), size: 1 }
};
}
function parsePositions(view, offset, nPoints, dim) {
const bufferOffset = view.byteOffset + offset;
const bufferLength = nPoints * dim * Float64Array.BYTES_PER_ELEMENT;
return [
new Float64Array(view.buffer.slice(bufferOffset, bufferOffset + bufferLength)),
offset + bufferLength
];
}
function concatPositions(xyPositions, mPositions, zPositions) {
if (!(mPositions || zPositions)) {
return xyPositions;
}
let arrayLength = xyPositions.length;
let nDim = 2;
if (zPositions && zPositions.length) {
arrayLength += zPositions.length;
nDim++;
}
if (mPositions && mPositions.length) {
arrayLength += mPositions.length;
nDim++;
}
const positions = new Float64Array(arrayLength);
for (let i = 0; i < xyPositions.length / 2; i++) {
positions[nDim * i] = xyPositions[i * 2];
positions[nDim * i + 1] = xyPositions[i * 2 + 1];
}
if (zPositions && zPositions.length) {
for (let i = 0; i < zPositions.length; i++) {
positions[nDim * i + 2] = zPositions[i];
}
}
if (mPositions && mPositions.length) {
for (let i = 0; i < mPositions.length; i++) {
positions[nDim * i + (nDim - 1)] = mPositions[i];
}
}
return positions;
}
function getWindingDirection(positions) {
return Math.sign(getSignedArea(positions));
}
function getSignedArea(positions) {
let area = 0;
const nCoords = positions.length / 2 - 1;
for (let i = 0; i < nCoords; i++) {
area += (positions[i * 2] + positions[(i + 1) * 2]) * (positions[i * 2 + 1] - positions[(i + 1) * 2 + 1]);
}
return area / 2;
}
// dist/lib/parsers/parse-shp.js
var LITTLE_ENDIAN3 = true;
var BIG_ENDIAN2 = false;
var SHP_HEADER_SIZE = 100;
var SHP_RECORD_HEADER_SIZE = 12;
var STATE = {
EXPECTING_HEADER: 0,
EXPECTING_RECORD: 1,
END: 2,
ERROR: 3
};
var SHPParser = class {
options = {};
binaryReader = new BinaryChunkReader({ maxRewindBytes: SHP_RECORD_HEADER_SIZE });
state = STATE.EXPECTING_HEADER;
result = {
geometries: [],
// Initialize with number values to make TS happy
// These are initialized for real in STATE.EXPECTING_HEADER
progress: {
bytesTotal: NaN,
bytesUsed: NaN,
rows: NaN
},
currentIndex: NaN
};
constructor(options) {
this.options = options;
}
write(arrayBuffer) {
this.binaryReader.write(arrayBuffer);
this.state = parseState(this.state, this.result, this.binaryReader, this.options);
}
end() {
this.binaryReader.end();
this.state = parseState(this.state, this.result, this.binaryReader, this.options);
if (this.state !== STATE.END) {
this.state = STATE.ERROR;
this.result.error = "SHP incomplete file";
}
}
};
function parseSHP(arrayBuffer, options) {
const shpParser = new SHPParser(options);
shpParser.write(arrayBuffer);
shpParser.end();
return shpParser.result;
}
async function* parseSHPInBatches(asyncIterator, options) {
const parser = new SHPParser(options);
let headerReturned = false;
for await (const arrayBuffer of (0, import_loader_utils.toArrayBufferIterator)(asyncIterator)) {
parser.write(arrayBuffer);
if (!headerReturned && parser.result.header) {
headerReturned = true;
yield parser.result.header;
}
if (parser.result.geometries.length > 0) {
yield parser.result.geometries;
parser.result.geometries = [];
}
}
parser.end();
if (parser.result.geometries.length > 0) {
yield parser.result.geometries;
}
return;
}
function parseState(state, result, binaryReader, options) {
var _a;
while (true) {
try {
switch (state) {
case STATE.ERROR:
case STATE.END:
return state;
case STATE.EXPECTING_HEADER:
const dataView = binaryReader.getDataView(SHP_HEADER_SIZE);
if (!dataView) {
return state;
}
result.header = parseSHPHeader(dataView);
result.progress = {
bytesUsed: 0,
bytesTotal: result.header.length,
rows: 0
};
result.currentIndex = 1;
state = STATE.EXPECTING_RECORD;
break;
case STATE.EXPECTING_RECORD:
while (binaryReader.hasAvailableBytes(SHP_RECORD_HEADER_SIZE)) {
const recordHeaderView = binaryReader.getDataView(SHP_RECORD_HEADER_SIZE);
const recordHeader = {
recordNumber: recordHeaderView.getInt32(0, BIG_ENDIAN2),
// 2 byte words; includes the four words of record header
byteLength: recordHeaderView.getInt32(4, BIG_ENDIAN2) * 2,
// This is actually part of the record, not the header...
type: recordHeaderView.getInt32(8, LITTLE_ENDIAN3)
};
if (!binaryReader.hasAvailableBytes(recordHeader.byteLength - 4)) {
binaryReader.rewind(SHP_RECORD_HEADER_SIZE);
return state;
}
const invalidRecord = recordHeader.byteLength < 4 || recordHeader.type !== ((_a = result.header) == null ? void 0 : _a.type) || recordHeader.recordNumber !== result.currentIndex;
if (invalidRecord) {
binaryReader.rewind(SHP_RECORD_HEADER_SIZE - 4);
} else {
binaryReader.rewind(4);
const recordView = binaryReader.getDataView(recordHeader.byteLength);
const geometry = parseRecord(recordView, options);
result.geometries.push(geometry);
result.currentIndex++;
result.progress.rows = result.currentIndex - 1;
}
}
if (binaryReader.ended) {
state = STATE.END;
}
return state;
default:
state = STATE.ERROR;
result.error = `illegal parser state ${state}`;
return state;
}
} catch (error) {
state = STATE.ERROR;
result.error = `SHP parsing failed: ${error == null ? void 0 : error.message}`;
return state;
}
}
}
// dist/shp-loader.js
var VERSION = true ? "4.4.4" : "latest";
var SHP_MAGIC_NUMBER2 = [0, 0, 39, 10];
var SHPWorkerLoader = {
dataType: null,
batchType: null,
name: "SHP",
id: "shp",
module: "shapefile",
version: VERSION,
worker: true,
category: "geometry",
extensions: ["shp"],
mimeTypes: ["application/octet-stream"],
// ISSUE: This also identifies SHX files, which are identical to SHP for the first 100 bytes...
tests: [new Uint8Array(SHP_MAGIC_NUMBER2).buffer],
options: {
shp: {
_maxDimensions: 4
}
}
};
var SHPLoader = {
...SHPWorkerLoader,
parse: async (arrayBuffer, options) => parseSHP(arrayBuffer, options),
parseSync: parseSHP,
parseInBatches: (arrayBufferIterator, options) => parseSHPInBatches(arrayBufferIterator, options)
};
// dist/lib/parsers/parse-shapefile.js
var import_loader_utils3 = require("@loaders.gl/loader-utils");
var import_gis = require("@loaders.gl/gis");
var import_proj4 = require("@math.gl/proj4");
// dist/lib/parsers/parse-shx.js
var SHX_HEADER_SIZE = 100;
var BIG_ENDIAN3 = false;
function parseShx(arrayBuffer) {
const headerView = new DataView(arrayBuffer, 0, SHX_HEADER_SIZE);
const header = parseSHPHeader(headerView);
const contentLength = header.length - SHX_HEADER_SIZE;
const contentView = new DataView(arrayBuffer, SHX_HEADER_SIZE, contentLength);
const offsets = new Int32Array(contentLength);
const lengths = new Int32Array(contentLength);
for (let i = 0; i < contentLength / 8; i++) {
offsets[i] = contentView.getInt32(i * 8, BIG_ENDIAN3);
lengths[i] = contentView.getInt32(i * 8 + 4, BIG_ENDIAN3);
}
return {
offsets,
lengths
};
}
// dist/lib/streaming/zip-batch-iterators.js
async function* zipBatchIterators(iterator1, iterator2, shape) {
const batch1Data = [];
const batch2Data = [];
let iterator1Done = false;
let iterator2Done = false;
while (!iterator1Done && !iterator2Done) {
if (batch1Data.length === 0 && !iterator1Done) {
const { value, done } = await iterator1.next();
if (done) {
iterator1Done = true;
} else {
batch1Data.push(...value);
}
}
if (batch2Data.length === 0 && !iterator2Done) {
const { value, done } = await iterator2.next();
if (done) {
iterator2Done = true;
} else {
batch2Data.push(...value);
}
}
const batchData = extractBatchData(batch1Data, batch2Data);
if (batchData) {
yield {
batchType: "data",
shape,
length: batchData.length,
data: batchData
};
}
}
}
function extractBatchData(batch1, batch2) {
const batchLength = Math.min(batch1.length, batch2.length);
if (batchLength === 0) {
return null;
}
const batch = [batch1.slice(0, batchLength), batch2.slice(0, batchLength)];
batch1.splice(0, batchLength);
batch2.splice(0, batchLength);
return batch;
}
// dist/lib/parsers/parse-dbf.js
var import_loader_utils2 = require("@loaders.gl/loader-utils");
var LITTLE_ENDIAN4 = true;
var DBF_HEADER_SIZE = 32;
var STATE2;
(function(STATE4) {
STATE4[STATE4["START"] = 0] = "START";
STATE4[STATE4["FIELD_DESCRIPTORS"] = 1] = "FIELD_DESCRIPTORS";
STATE4[STATE4["FIELD_PROPERTIES"] = 2] = "FIELD_PROPERTIES";
STATE4[STATE4["END"] = 3] = "END";
STATE4[STATE4["ERROR"] = 4] = "ERROR";
})(STATE2 || (STATE2 = {}));
var DBFParser = class {
binaryReader = new BinaryChunkReader();
textDecoder;
state = STATE2.START;
result = {
data: []
};
constructor(options) {
this.textDecoder = new TextDecoder(options.encoding);
}
/**
* @param arrayBuffer
*/
write(arrayBuffer) {
this.binaryReader.write(arrayBuffer);
this.state = parseState2(this.state, this.result, this.binaryReader, this.textDecoder);
}
end() {
this.binaryReader.end();
this.state = parseState2(this.state, this.result, this.binaryReader, this.textDecoder);
if (this.state !== STATE2.END) {
this.state = STATE2.ERROR;
this.result.error = "DBF incomplete file";
}
}
};
function parseDBF(arrayBuffer, options = {}) {
var _a;
const { encoding = "latin1" } = options.dbf || {};
const dbfParser = new DBFParser({ encoding });
dbfParser.write(arrayBuffer);
dbfParser.end();
const { data, schema } = dbfParser.result;
const shape = (_a = options == null ? void 0 : options.dbf) == null ? void 0 : _a.shape;
switch (shape) {
case "object-row-table": {
const table = {
shape: "object-row-table",
schema,
data
};
return table;
}
case "table":
return { schema, rows: data };
case "rows":
default:
return data;
}
}
async function* parseDBFInBatches(asyncIterator, options = {}) {
const { encoding = "latin1" } = options.dbf || {};
const parser = new DBFParser({ encoding });
let headerReturned = false;
for await (const arrayBuffer of (0, import_loader_utils2.toArrayBufferIterator)(asyncIterator)) {
parser.write(arrayBuffer);
if (!headerReturned && parser.result.dbfHeader) {
headerReturned = true;
yield parser.result.dbfHeader;
}
if (parser.result.data.length > 0) {
yield parser.result.data;
parser.result.data = [];
}
}
parser.end();
if (parser.result.data.length > 0) {
yield parser.result.data;
}
}
function parseState2(state, result, binaryReader, textDecoder) {
while (true) {
try {
switch (state) {
case STATE2.ERROR:
case STATE2.END:
return state;
case STATE2.START:
const dataView = binaryReader.getDataView(DBF_HEADER_SIZE);
if (!dataView) {
return state;
}
result.dbfHeader = parseDBFHeader(dataView);
result.progress = {
bytesUsed: 0,
rowsTotal: result.dbfHeader.nRecords,
rows: 0
};
state = STATE2.FIELD_DESCRIPTORS;
break;
case STATE2.FIELD_DESCRIPTORS:
const fieldDescriptorView = binaryReader.getDataView(
// @ts-ignore
result.dbfHeader.headerLength - DBF_HEADER_SIZE
);
if (!fieldDescriptorView) {
return state;
}
result.dbfFields = parseFieldDescriptors(fieldDescriptorView, textDecoder);
result.schema = {
fields: result.dbfFields.map((dbfField) => makeField(dbfField)),
metadata: {}
};
state = STATE2.FIELD_PROPERTIES;
binaryReader.skip(1);
break;
case STATE2.FIELD_PROPERTIES:
const { recordLength = 0, nRecords = 0 } = (result == null ? void 0 : result.dbfHeader) || {};
while (result.data.length < nRecords) {
const recordView = binaryReader.getDataView(recordLength - 1);
if (!recordView) {
return state;
}
binaryReader.skip(1);
const row = parseRow(recordView, result.dbfFields, textDecoder);
result.data.push(row);
result.progress.rows = result.data.length;
}
state = STATE2.END;
break;
default:
state = STATE2.ERROR;
result.error = `illegal parser state ${state}`;
return state;
}
} catch (error) {
state = STATE2.ERROR;
result.error = `DBF parsing failed: ${error.message}`;
return state;
}
}
}
function parseDBFHeader(headerView) {
return {
// Last updated date
year: headerView.getUint8(1) + 1900,
month: headerView.getUint8(2),
day: headerView.getUint8(3),
// Number of records in data file
nRecords: headerView.getUint32(4, LITTLE_ENDIAN4),
// Length of header in bytes
headerLength: headerView.getUint16(8, LITTLE_ENDIAN4),
// Length of each record
recordLength: headerView.getUint16(10, LITTLE_ENDIAN4),
// Not sure if this is usually set
languageDriver: headerView.getUint8(29)
};
}
function parseFieldDescriptors(view, textDecoder) {
const nFields = (view.byteLength - 1) / 32;
const fields = [];
let offset = 0;
for (let i = 0; i < nFields; i++) {
const name = textDecoder.decode(new Uint8Array(view.buffer, view.byteOffset + offset, 11)).replace(/\u0000/g, "");
fields.push({
name,
dataType: String.fromCharCode(view.getUint8(offset + 11)),
fieldLength: view.getUint8(offset + 16),
decimal: view.getUint8(offset + 17)
});
offset += 32;
}
return fields;
}
function parseRow(view, fields, textDecoder) {
const out = {};
let offset = 0;
for (const field of fields) {
const text = textDecoder.decode(new Uint8Array(view.buffer, view.byteOffset + offset, field.fieldLength));
out[field.name] = parseField(text, field.dataType);
offset += field.fieldLength;
}
return out;
}
function parseField(text, dataType) {
switch (dataType) {
case "B":
return parseNumber(text);
case "C":
return parseCharacter(text);
case "F":
return parseNumber(text);
case "N":
return parseNumber(text);
case "O":
return parseNumber(text);
case "D":
return parseDate(text);
case "L":
return parseBoolean(text);
default:
throw new Error("Unsupported data type");
}
}
function parseDate(str) {
return Date.UTC(str.slice(0, 4), parseInt(str.slice(4, 6), 10) - 1, str.slice(6, 8));
}
function parseBoolean(value) {
return /^[nf]$/i.test(value) ? false : /^[yt]$/i.test(value) ? true : null;
}
function parseNumber(text) {
const number = parseFloat(text);
return isNaN(number) ? null : number;
}
function parseCharacter(text) {
return text.trim() || null;
}
function makeField({ name, dataType, fieldLength, decimal }) {
switch (dataType) {
case "B":
return { name, type: "float64", nullable: true, metadata: {} };
case "C":
return { name, type: "utf8", nullable: true, metadata: {} };
case "F":
return { name, type: "float64", nullable: true, metadata: {} };
case "N":
return { name, type: "float64", nullable: true, metadata: {} };
case "O":
return { name, type: "float64", nullable: true, metadata: {} };
case "D":
return { name, type: "timestamp-millisecond", nullable: true, metadata: {} };
case "L":
return { name, type: "bool", nullable: true, metadata: {} };
default:
throw new Error("Unsupported data type");
}
}
// dist/dbf-loader.js
var VERSION2 = true ? "4.4.4" : "latest";
var DBFWorkerLoader = {
name: "DBF",
dataType: null,
batchType: null,
id: "dbf",
module: "shapefile",
version: VERSION2,
worker: true,
category: "table",
extensions: ["dbf"],
mimeTypes: ["application/x-dbf"],
options: {
dbf: {
encoding: "latin1"
}
}
};
var DBFLoader = {
...DBFWorkerLoader,
parse: async (arrayBuffer, options) => parseDBF(arrayBuffer, options),
parseSync: parseDBF,
parseInBatches(arrayBufferIterator, options) {
return parseDBFInBatches(arrayBufferIterator, options);
}
};
// dist/lib/parsers/parse-shapefile.js
async function* parseShapefileInBatches(asyncIterator, options, context) {
var _a, _b, _c;
const { reproject = false, _targetCrs = "WGS84" } = (options == null ? void 0 : options.gis) || {};
const { shx, cpg, prj } = await loadShapefileSidecarFiles(options, context);
const shapeIterable = await (0, import_loader_utils3.parseInBatchesFromContext)((0, import_loader_utils3.toArrayBufferIterator)(asyncIterator), SHPLoader, options, context);
const shapeIterator = ((_a = shapeIterable[Symbol.asyncIterator]) == null ? void 0 : _a.call(shapeIterable)) || ((_b = shapeIterable[Symbol.iterator]) == null ? void 0 : _b.call(shapeIterable));
let propertyIterator = null;
const dbfResponse = await (context == null ? void 0 : context.fetch(replaceExtension((context == null ? void 0 : context.url) || "", "dbf")));
if (dbfResponse == null ? void 0 : dbfResponse.ok) {
const propertyIterable = await (0, import_loader_utils3.parseInBatchesFromContext)(dbfResponse, DBFLoader, {
...options,
dbf: {
...options == null ? void 0 : options.dbf,
encoding: cpg || "latin1"
}
}, context);
propertyIterator = ((_c = propertyIterable[Symbol.asyncIterator]) == null ? void 0 : _c.call(propertyIterable)) || propertyIterable[Symbol.iterator]();
}
let shapeHeader = (await shapeIterator.next()).value;
if (shapeHeader && shapeHeader.batchType === "metadata") {
shapeHeader = (await shapeIterator.next()).value;
}
let dbfHeader = {};
if (propertyIterator) {
dbfHeader = (await propertyIterator.next()).value;
if (dbfHeader && dbfHeader.batchType === "metadata") {
dbfHeader = (await propertyIterator.next()).value;
}
}
const zippedIterator = propertyIterator ? zipBatchIterators(shapeIterator, propertyIterator, "object-row-table") : shapeIterator;
const zippedBatchIterable = {
[Symbol.asyncIterator]() {
return zippedIterator;
}
};
for await (const batch of zippedBatchIterable) {
let geometries;
let properties;
if (!propertyIterator) {
geometries = batch;
} else {
[geometries, properties] = batch.data;
}
const geojsonGeometries = parseGeometries(geometries);
let features = joinProperties(geojsonGeometries, properties);
if (reproject) {
features = reprojectFeatures(features, prj, _targetCrs);
}
yield {
encoding: cpg,
prj,
shx,
header: shapeHeader,
data: features
};
}
}
async function parseShapefile(arrayBuffer, options, context) {
var _a;
const { reproject = false, _targetCrs = "WGS84" } = (options == null ? void 0 : options.gis) || {};
const { shx, cpg, prj } = await loadShapefileSidecarFiles(options, context);
const { header, geometries } = await (0, import_loader_utils3.parseFromContext)(arrayBuffer, SHPLoader, options, context);
const geojsonGeometries = parseGeometries(geometries);
let propertyTable;
const dbfResponse = await (context == null ? void 0 : context.fetch(replaceExtension(context == null ? void 0 : context.url, "dbf")));
if (dbfResponse == null ? void 0 : dbfResponse.ok) {
const dbfOptions = {
...options,
dbf: {
...options == null ? void 0 : options.dbf,
shape: "object-row-table",
encoding: cpg || "latin1"
}
};
propertyTable = await (0, import_loader_utils3.parseFromContext)(dbfResponse, DBFLoader, dbfOptions, context);
}
let features = joinProperties(geojsonGeometries, (propertyTable == null ? void 0 : propertyTable.data) || []);
if (reproject) {
features = reprojectFeatures(features, prj, _targetCrs);
}
switch ((_a = options == null ? void 0 : options.shapefile) == null ? void 0 : _a.shape) {
case "geojson-table":
return {
// @ts-expect-error
shape: "geojson-table",
type: "FeatureCollection",
encoding: cpg,
schema: (propertyTable == null ? void 0 : propertyTable.schema) || { metadata: {}, fields: [] },
prj,
shx,
header,
features
};
default:
return {
encoding: cpg,
prj,
shx,
header,
data: features
};
}
}
function parseGeometries(geometries) {
const geojsonGeometries = [];
for (const geom of geometries) {
geojsonGeometries.push((0, import_gis.convertBinaryGeometryToGeometry)(geom));
}
return geojsonGeometries;
}
function joinProperties(geometries, properties) {
const features = [];
for (let i = 0; i < geometries.length; i++) {
const geometry = geometries[i];
const feature = {
type: "Feature",
geometry,
// properties can be undefined if dbfResponse above was empty
properties: properties && properties[i] || {}
};
features.push(feature);
}
return features;
}
function reprojectFeatures(features, sourceCrs, targetCrs) {
if (!sourceCrs && !targetCrs) {
return features;
}
const projection = new import_proj4.Proj4Projection({ from: sourceCrs || "WGS84", to: targetCrs || "WGS84" });
return (0, import_gis.transformGeoJsonCoords)(features, (coord) => projection.project(coord));
}
async function loadShapefileSidecarFiles(options, context) {
const { url, fetch } = context;
const shxPromise = fetch(replaceExtension(url, "shx"));
const cpgPromise = fetch(replaceExtension(url, "cpg"));
const prjPromise = fetch(replaceExtension(url, "prj"));
await Promise.all([shxPromise, cpgPromise, prjPromise]);
let shx;
let cpg;
let prj;
const shxResponse = await shxPromise;
if (shxResponse.ok) {
const arrayBuffer = await shxResponse.arrayBuffer();
shx = parseShx(arrayBuffer);
}
const cpgResponse = await cpgPromise;
if (cpgResponse.ok) {
cpg = await cpgResponse.text();
}
const prjResponse = await prjPromise;
if (prjResponse.ok) {
prj = await prjResponse.text();
}
return {
shx,
cpg,
prj
};
}
function replaceExtension(url, newExtension) {
const baseName = basename(url);
const extension = extname(url);
const isUpperCase = extension === extension.toUpperCase();
if (isUpperCase) {
newExtension = newExtension.toUpperCase();
}
return `${baseName}.${newExtension}`;
}
function basename(url) {
const extIndex = url && url.lastIndexOf(".");
if (typeof extIndex === "number") {
return extIndex >= 0 ? url.substr(0, extIndex) : "";
}
return extIndex;
}
function extname(url) {
const extIndex = url && url.lastIndexOf(".");
if (typeof extIndex === "number") {
return extIndex >= 0 ? url.substr(extIndex + 1) : "";
}
return extIndex;
}
// dist/shapefile-loader.js
var VERSION3 = true ? "4.4.4" : "latest";
var ShapefileLoader = {
name: "Shapefile",
id: "shapefile",
module: "shapefile",
version: VERSION3,
category: "geometry",
extensions: ["shp"],
mimeTypes: ["application/octet-stream"],
tests: [new Uint8Array(SHP_MAGIC_NUMBER2).buffer],
options: {
shapefile: {
shape: "v3"
},
shp: {
_maxDimensions: 4
}
},
// @ts-expect-error
parse: parseShapefile,
// @ts-expect-error
parseInBatches: parseShapefileInBatches
};
// dist/lib/parsers/parse-dbf-to-arrow.js
var import_loader_utils4 = require("@loaders.gl/loader-utils");
var import_schema_utils = require("@loaders.gl/schema-utils");
var LITTLE_ENDIAN5 = true;
var DBF_HEADER_SIZE2 = 32;
var STATE3;
(function(STATE4) {
STATE4[STATE4["START"] = 0] = "START";
STATE4[STATE4["FIELD_DESCRIPTORS"] = 1] = "FIELD_DESCRIPTORS";
STATE4[STATE4["FIELD_PROPERTIES"] = 2] = "FIELD_PROPERTIES";
STATE4[STATE4["END"] = 3] = "END";
STATE4[STATE4["ERROR"] = 4] = "ERROR";
})(STATE3 || (STATE3 = {}));
var DBFParser2 = class {
binaryReader = new BinaryChunkReader();
textDecoder;
state = STATE3.START;
result = {};
constructor(options) {
this.textDecoder = new TextDecoder(options.encoding);
}
/**
* @param arrayBuffer
*/
write(arrayBuffer) {
this.binaryReader.write(arrayBuffer);
this.state = parseState3(this.state, this.result, this.binaryReader, this.textDecoder);
}
end() {
this.binaryReader.end();
this.state = parseState3(this.state, this.result, this.binaryReader, this.textDecoder);
if (this.state !== STATE3.END) {
this.state = STATE3.ERROR;
this.result.error = "DBF incomplete file";
}
}
};
function parseDBF2(arrayBuffer, options = {}) {
const { encoding = "latin1" } = options.dbf || {};
const dbfParser = new DBFParser2({ encoding });
dbfParser.write(arrayBuffer);
dbfParser.end();
const tableBuilder = dbfParser.result.tableBuilder;
const arrowTable = tableBuilder.finishTable();
return arrowTable;
}
async function* parseDBFInBatches2(asyncIterator, options = {}) {
const { encoding = "latin1" } = options.dbf || {};
const parser = new DBFParser2({ encoding });
let headerReturned = false;
for await (const arrayBuffer of (0, import_loader_utils4.toArrayBufferIterator)(asyncIterator)) {
parser.write(arrayBuffer);
if (!headerReturned && parser.result.dbfHeader) {
headerReturned = true;
const tableBuilder3 = parser.result.tableBuilder;
const tableBatch3 = tableBuilder3.firstBatch();
if (tableBatch3) {
yield tableBatch3;
}
}
const tableBuilder2 = parser.result.tableBuilder;
const tableBatch2 = tableBuilder2.flushBatch();
if (tableBatch2) {
yield tableBatch2;
}
}
parser.end();
const tableBuilder = parser.result.tableBuilder;
const tableBatch = tableBuilder.finishBatch();
if (tableBatch) {
yield tableBatch;
}
}
function parseState3(state, result, binaryReader, textDecoder) {
while (true) {
try {
switch (state) {
case STATE3.ERROR:
case STATE3.END:
return state;
case STATE3.START:
const dataView = binaryReader.getDataView(DBF_HEADER_SIZE2);
if (!dataView) {
return state;
}
result.dbfHeader = parseDBFHeader2(dataView);
result.progress = {
bytesUsed: 0,
rowsTotal: result.dbfHeader.nRecords,
rows: 0
};
state = STATE3.FIELD_DESCRIPTORS;
break;
case STATE3.FIELD_DESCRIPTORS:
const fieldDescriptorView = binaryReader.getDataView(
// @ts-ignore
result.dbfHeader.headerLength - DBF_HEADER_SIZE2
);
if (!fieldDescriptorView) {
return state;
}
result.dbfFields = parseFieldDescriptors2(fieldDescriptorView, textDecoder);
const schema = {
fields: result.dbfFields.map((dbfField) => makeField2(dbfField)),
metadata: {}
};
result.tableBuilder = new import_schema_utils.ArrowTableBuilder(schema);
state = STATE3.FIELD_PROPERTIES;
binaryReader.skip(1);
break;
case STATE3.FIELD_PROPERTIES:
const { recordLength = 0, nRecords = 0 } = (result == null ? void 0 : result.dbfHeader) || {};
let rowCount = 0;
while (rowCount < nRecords) {
rowCount++;
const recordView = binaryReader.getDataView(recordLength - 1);
if (!recordView) {
return state;
}
binaryReader.skip(1);
const row = parseRow2(recordView, result.dbfFields, textDecoder);
result.tableBuilder.addObjectRow(row);
}
state = STATE3.END;
break;
default:
state = STATE3.ERROR;
result.error = `illegal parser state ${state}`;
return state;
}
} catch (error) {
state = STATE3.ERROR;
result.error = `DBF parsing failed: ${error.message}`;
return state;
}
}
}
function parseDBFHeader2(headerView) {
return {
// Last updated date
year: headerView.getUint8(1) + 1900,
month: headerView.getUint8(2),
day: headerView.getUint8(3),
// Number of records in data file
nRecords: headerView.getUint32(4, LITTLE_ENDIAN5),
// Length of header in bytes
headerLength: headerView.getUint16(8, LITTLE_ENDIAN5),
// Length of each record
recordLength: headerView.getUint16(10, LITTLE_ENDIAN5),
// Not sure if this is usually set
languageDriver: headerView.getUint8(29)
};
}
function parseFieldDescriptors2(view, textDecoder) {
const nFields = (view.byteLength - 1) / 32;
const fields = [];
let offset = 0;
for (let i = 0; i < nFields; i++) {
const name = textDecoder.decode(new Uint8Array(view.buffer, view.byteOffset + offset, 11)).replace(/\u0000/g, "");
fields.push({
name,
dataType: String.fromCharCode(view.getUint8(offset + 11)),
fieldLength: view.getUint8(offset + 16),
decimal: view.getUint8(offset + 17)
});
offset += 32;
}
return fields;
}
function parseRow2(view, fields, textDecoder) {
const out = {};
let offset = 0;
for (const field of fields) {
const text = textDecoder.decode(new Uint8Array(view.buffer, view.byteOffset + offset, field.fieldLength));
out[field.name] = parseField2(text, field.dataType);
offset += field.fieldLength;
}
return out;
}
function parseField2(text, dataType) {
switch (dataType) {
case "B":
return parseNumber2(text);
case "C":
return parseCharacter2(text);
case "F":
return parseNumber2(text);
case "N":
return parseNumber2(text);
case "O":
return parseNumber2(text);
case "D":
return parseDate2(text);
case "L":
return parseBoolean2(text);
default:
throw new Error("Unsupported data type");
}
}
function parseDate2(str) {
return Date.UTC(str.slice(0, 4), parseInt(str.slice(4, 6), 10) - 1, str.slice(6, 8));
}
function parseBoolean2(value) {
return /^[nf]$/i.test(value) ? false : /^[yt]$/i.test(value) ? true : null;
}
function parseNumber2(text) {
const number = parseFloat(text);
return isNaN(number) ? null : number;
}
function parseCharacter2(text) {
return text.trim() || null;
}
function makeField2({ name, dataType, fieldLength, decimal }) {
switch (dataType) {
case "B":
return { name, type: "float64", nullable: true, metadata: {} };
case "C":
return { name, type: "utf8", nullable: true, metadata: {} };
case "F":
return { name, type: "float64", nullable: true, metadata: {} };
case "N":
return { name, type: "float64", nullable: true, metadata: {} };
case "O":
return { name, type: "float64", nullable: true, metadata: {} };
case "D":
return { name, type: "timestamp-millisecond", nullable: true, metadata: {} };
case "L":
return { name, type: "bool", nullable: true, metadata: {} };
default:
throw new Error("Unsupported data type");
}
}
// dist/dbf-format.js
var DBFFormat = {
name: "DBF",
id: "dbf",
module: "shapefile",
category: "table",
extensions: ["dbf"],
mimeTypes: ["application/x-dbf"]
};
// dist/dbf-arrow-loader.js
var VERSION4 = true ? "4.4.4" : "latest";
var DBFArrowWorkerLoader = {
...DBFFormat,
dataType: null,
batchType: null,
version: VERSION4,
worker: true,
options: {
dbf: {
encoding: "latin1"
}
}
};
var DBFArrowLoader = {
...DBFArrowWorkerLoader,
parse: async (arrayBuffer, options) => parseDBF2(arrayBuffer, options),
parseSync: parseDBF2,
parseInBatches(arrayBufferIterator, options) {
return parseDBFInBatches2(arrayBufferIterator, options);
}
};
// dist/lib/streaming/binary-reader.js
var BinaryReader = class {
offset;
arrayBuffer;
constructor(arrayBuffer) {
this.offset = 0;
this.arrayBuffer = arrayBuffer;
}
/**
* Checks if there are available bytes in data
*
* @param bytes
* @returns boolean
*/
hasAvailableBytes(bytes) {
return this.arrayBuffer.byteLength - this.offset >= bytes;
}
/**
* Get the required number of bytes from the iterator
*
* @param bytes
* @returns Dataview
*/
getDataView(bytes) {
if (bytes && !this.hasAvailableBytes(bytes)) {
throw new Error("binary data exhausted");
}
const dataView = bytes ? new DataView(this.arrayBuffer, this.offset, bytes) : new DataView(this.arrayBuffer, this.offset);
this.offset += bytes;
return dataView;
}
/**
* Skipping
*
* @param bytes
*/
skip(bytes) {
this.offset += bytes;
}
/**
* Rewinding
*
* @param bytes
*/
rewind(bytes) {
this.offset -= bytes;
}
};
//# sourceMappingURL=index.cjs.map