open-vector-tile
Version:
This library reads/writes Open Vector Tiles
281 lines • 11.6 kB
JavaScript
import { deltaDecodeArray, deltaEncodeArray, dequantizeBBox, dequantizeBBox3D, quantizeBBox, unweaveAndDeltaDecode3DArray, unweaveAndDeltaDecodeArray, weaveAndDeltaEncode3DArray, weaveAndDeltaEncodeArray, } from '../util.js';
/**
* Column Cache Reader
* Stores all data in a column format.
* Upon construction, all columns are decoded from the protobuf.
* This allows for quick and easy access to data in a column format.
*/
export class ColumnCacheReader {
/** strings are stored in a column of strings */
[1 /* OColumnName.string */] = [];
/** unsigned whole numbers are stored in unsigned */
[2 /* OColumnName.unsigned */] = [];
/** negative numbers are stored in signed */
[3 /* OColumnName.signed */] = [];
/** non-whole 32-bit numbers are stored in float */
[4 /* OColumnName.float */] = [];
/** non-whole numbers greater than 32-bit are stored in double */
[5 /* OColumnName.double */] = [];
/** for geometry types each column is individually weaved and delta encoded */
[6 /* OColumnName.points */] = [];
/** for geometry types each column is individually weaved and delta encoded */
[7 /* OColumnName.points3D */] = [];
/** store M-Value indices, geometry indices, and geometry shapes */
[8 /* OColumnName.indices */] = [];
/** shapes and possibly value indices are stored in a number[] to be decoded by readShape */
[9 /* OColumnName.shapes */] = [];
/** Stores both BBox and BBox3D in a single column */
[10 /* OColumnName.bbox */] = [];
#pbf;
/**
* @param pbf - the pbf protocol we are reading from
* @param end - the position to stop at
*/
constructor(pbf, end = 0) {
this.#pbf = pbf;
pbf.readFields(this.#read.bind(this), this, end);
}
/**
* @param col - the column to read/store the parsed data
* @param index - the index in the column to read/store the parsed data
* @returns - the parsed data
*/
getColumn(col, index) {
let res;
const columnValue = this[col][index];
const hasPos = typeof columnValue === 'object' && 'pos' in columnValue;
const currPos = this.#pbf.pos;
if (hasPos)
this.#pbf.pos = columnValue.pos;
if (col === 9 /* OColumnName.shapes */) {
if (hasPos) {
this[col][index] = { data: this.#getColumnData(col) };
}
res = this[col][index].data;
}
else {
if (hasPos) {
this[col][index] = this.#getColumnData(col);
}
res = this[col][index];
}
// return to original position
this.#pbf.pos = currPos;
return res;
}
/**
* @param col - the column to store the parsed data
* @returns - the parsed data
*/
#getColumnData(col) {
switch (col) {
case 1 /* OColumnName.string */:
return this.#pbf.readString();
case 2 /* OColumnName.unsigned */:
return this.#pbf.readVarint();
case 3 /* OColumnName.signed */:
return this.#pbf.readSVarint();
case 4 /* OColumnName.float */:
return this.#pbf.readFloat();
case 5 /* OColumnName.double */:
return this.#pbf.readDouble();
case 6 /* OColumnName.points */:
return unweaveAndDeltaDecodeArray(this.#pbf.readPackedVarint());
case 7 /* OColumnName.points3D */:
return unweaveAndDeltaDecode3DArray(this.#pbf.readPackedVarint());
case 8 /* OColumnName.indices */:
return deltaDecodeArray(this.#pbf.readPackedVarint());
case 9 /* OColumnName.shapes */:
return this.#pbf.readPackedVarint();
case 10 /* OColumnName.bbox */: {
const data = this.#pbf.readBytes();
if (data.byteLength === 12) {
return dequantizeBBox(data);
}
else {
return dequantizeBBox3D(data);
}
}
default:
throw new Error('Unknown column type');
}
}
/**
* @param tag - the tag to explain how to read the following data
* @param reader - the column cache to read from
* @param pbf - the protobuf object to read from
*/
#read(tag, reader, pbf) {
if (tag <= 0 || tag > 10)
throw new Error('Unknown column type');
const columnType = tag;
reader[columnType].push({ pos: pbf.pos });
}
}
/**
* The cache where all data is stored in a column format.
* Each column type has its own array of data.
* Number types maintain their own index for sorting purposes.
*/
export class ColumnCacheWriter {
/** strings are grouped by their bytes. */
[1 /* OColumnName.string */] = new Map();
/** Unsigned integers are sorted prior to storing */
[2 /* OColumnName.unsigned */] = new Map();
/** Signed integers are sorted prior to storing */
[3 /* OColumnName.signed */] = new Map();
/** 32-bit partial values are sorted prior to storing */
[4 /* OColumnName.float */] = new Map();
/** 64-bit partial values are sorted prior to storing */
[5 /* OColumnName.double */] = new Map();
/** for geometry types each column is individually weaved and delta encoded */
[6 /* OColumnName.points */] = new Map();
/** for geometry types each column is individually weaved and delta encoded */
[7 /* OColumnName.points3D */] = new Map();
/** Indices track geometry indices, geometry shapes, or other indexing data */
[8 /* OColumnName.indices */] = new Map();
/** Contains number arrays of how to rebuild objects */
[9 /* OColumnName.shapes */] = new Map();
/** Features should be sorted by id prior to building a column */
[10 /* OColumnName.bbox */] = new Map();
/**
* @template T - one of the column types
* @param col - the column to add the value to
* @param value - the value to add
* @returns - the index of the value
*/
addColumnData(col, value) {
if (typeof value === 'number')
throw Error('Use addNumber instead.');
const type = typeof value;
const isString = type === 'string';
const isNumber = type === 'number';
const key = isString || isNumber
? value
: // we need to simplify value keys to only include the column and data for better matching.
col === 9 /* OColumnName.shapes */
? JSON.stringify(value.map((v) => typeof v === 'number' ? v : { col: v.col, data: v.data }))
: 'data' in value
? JSON.stringify(value.data)
: JSON.stringify(value);
// look for duplicates
const colData = this[col];
const data = colData.get(key);
if (data !== undefined) {
data.count++;
return data.index;
}
// otherwise add
// @ts-expect-error - i want to fix this later
colData.set(key, { col, data: value, index: colData.size, count: 1 });
return colData.size - 1;
}
/**
* This function is specifically designed for number types as they will be sorted later
* for better compression.
* @param value - the number
* @param cType - the column type if we already know its classification
* @returns - the ColumnValue reference which contains the index and data.
* Will be sorted before stored.
*/
addNumber(value, cType) {
// get column
const columnType = cType !== undefined
? cType
: value % 1 === 0
? value >= 0
? 2 /* OColumnName.unsigned */
: 3 /* OColumnName.signed */
: 5 /* OColumnName.double */;
// get index
const column = this[columnType];
let columnValue = column.get(value);
// add if not found
if (columnValue === undefined) {
columnValue = {
col: columnType,
data: value,
index: column.size,
count: 0,
};
column.set(value, columnValue);
}
// increment count
columnValue.count++;
return columnValue;
}
/**
* The whole column cache is a message at the tile level.
* all columns are stored as fields in the message
* @param column - the column cache we want to write from
* @param pbf - the pbf protocol we are writing to
*/
static write(column, pbf) {
// setup
const unsigned = sortColumn([...column[2 /* OColumnName.unsigned */].values()]);
const signed = sortColumn([...column[3 /* OColumnName.signed */].values()]);
const float = sortColumn([...column[4 /* OColumnName.float */].values()]);
const double = sortColumn([...column[5 /* OColumnName.double */].values()]);
// strings
// for each column, encode apropriately and send to pbf
for (const [string] of column[1 /* OColumnName.string */]) {
pbf.writeStringField(1 /* OColumnName.string */, string);
}
// numbers
for (const u of unsigned) {
pbf.writeVarintField(2 /* OColumnName.unsigned */, u.data);
}
for (const s of signed) {
pbf.writeSVarintField(3 /* OColumnName.signed */, s.data);
}
for (const f of float) {
pbf.writeFloatField(4 /* OColumnName.float */, f.data);
}
for (const d of double) {
pbf.writeDoubleField(5 /* OColumnName.double */, d.data);
}
// points
const points = [...column[6 /* OColumnName.points */].values()];
for (const p of points) {
pbf.writePackedVarint(6 /* OColumnName.points */, weaveAndDeltaEncodeArray(p.data));
}
// points 3D:
const points3D = [...column[7 /* OColumnName.points3D */].values()];
for (const p3D of points3D) {
pbf.writePackedVarint(7 /* OColumnName.points3D */, weaveAndDeltaEncode3DArray(p3D.data));
}
// indices
const allIndices = [...column[8 /* OColumnName.indices */].values()];
for (const indices of allIndices) {
pbf.writePackedVarint(8 /* OColumnName.indices */, deltaEncodeArray(indices.data));
}
// shapes
const allShapes = [...column[9 /* OColumnName.shapes */].values()];
for (const arr of allShapes) {
const packed = arr.data.map((v) => (typeof v === 'object' ? v.index : v));
pbf.writePackedVarint(9 /* OColumnName.shapes */, packed);
}
// bbox
const allBBox = [...column[10 /* OColumnName.bbox */].values()];
for (const bbox of allBBox) {
pbf.writeBytesField(10 /* OColumnName.bbox */, quantizeBBox(bbox.data));
}
}
}
/**
* Sort number types and value types by index then update the index of each row for better
* compression down the line.
* @param input - an interable array of number columns (contains data and index)
* @returns - the sorted array
*/
function sortColumn(input) {
input = input.sort((a, b) => {
const count = b.count - a.count;
if (count !== 0)
return count;
return a.data - b.data;
});
input.forEach((v, i) => (v.index = i));
return input;
}
//# sourceMappingURL=columnCache.js.map