terriajs
Version:
Geospatial data visualization platform.
757 lines • 30.9 kB
JavaScript
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
import countBy from "lodash-es/countBy";
import Mexp from "math-expression-evaluator";
import { computed, makeObservable } from "mobx";
import JulianDate from "terriajs-cesium/Source/Core/JulianDate";
import filterOutUndefined from "../Core/filterOutUndefined";
import isDefined from "../Core/isDefined";
import runLater from "../Core/runLater";
import createCombinedModel from "../Models/Definition/createCombinedModel";
import { THIS_COLUMN_EXPRESSION_TOKEN } from "../Traits/TraitsClasses/Table/ColumnTraits";
import TableColumnType, { stringToTableColumnType } from "./TableColumnType";
/**
* A column of tabular data.
*/
export default class TableColumn {
columnNumber;
tableModel;
constructor(tableModel, columnNumber) {
makeObservable(this);
this.columnNumber = columnNumber;
this.tableModel = tableModel;
}
/**
* Gets the raw, uninterpreted values in the column. This will not apply any transformations to data (eg transformation expressions).
*
*/
get values() {
const result = [];
if (this.tableModel.dataColumnMajor !== undefined) {
// Copy all but the first element (which is the header), and trim along the way.
const source = this.tableModel.dataColumnMajor[this.columnNumber];
for (let i = 1; i < source.length; ++i) {
result.push(source[i].trim());
}
}
return result;
}
/**
* Gets `math-expression-evaluator` tokens. These allow other column values to be used in an expression.
* Each token is generated by `transformation.dependencies` trait - which is an array of strings of column names.
* A token `THIS_COLUMN_EXPRESSION_TOKEN` is also added - which corresponds to the value in **this column**.
* For example:
* - if `dependencies = ['columnA']`
* - then a token will be generated for `columnA`
* - and then we can access the value in an expression (eg `THIS_COLUMN_EXPRESSION_TOKEN*columnA` will multiple each value in this column, by the value in `columnA`)
*/
get mexpColumnTokens() {
return [
{
type: 3, // This type defines a constant value - see https://bugwheels94.github.io/math-expression-evaluator/#how-to-define-a-token for types
token: THIS_COLUMN_EXPRESSION_TOKEN,
show: THIS_COLUMN_EXPRESSION_TOKEN,
value: THIS_COLUMN_EXPRESSION_TOKEN
},
...filterOutUndefined(this.traits.transformation?.dependencies?.map((colName) => {
if (this.tableModel.tableColumns.find((col) => col.name === colName)) {
return {
type: 3, // This type defines a constant value
token: colName,
show: colName,
value: colName
};
}
else {
// TODO: deal with error handling when we have it
console.log(`Failed to add column token - column "${colName}" doesn't exist.\nWith expression: ${this.traits.transformation.expression}\nWith dependencies: ${this.traits.transformation.dependencies?.join(", ")}`);
}
}) ?? [])
];
}
/**
* Gets a function which can be used to generate `math-expression-evaluator` pairs for a given row `i`.
* The function returns key-value pairs which map tokens to values.
* Each token (see `this.mexpColumnTokens`) represents another column in the table, each value represents corresponding cell for row i.
*
* @param rowIndex row number
* @param value value to transform
*/
mexpColumnValuePairs(rowIndex, value) {
return this.mexpColumnTokens.reduce((pairs, token) => {
if (token.token !== THIS_COLUMN_EXPRESSION_TOKEN)
pairs[token.value] =
this.tableModel.tableColumns.find((col) => col.name === token.token)
?.valuesAsNumbers.values[rowIndex] ?? null;
// Add column pair for this value (token `THIS_COLUMN_EXPRESSION_TOKEN`)
else
pairs[THIS_COLUMN_EXPRESSION_TOKEN] = value;
return pairs;
}, {});
}
/**
* Gets `math-expression-evaluator` expression in "postfix" notation. This will "bake" tokens from `this.mexpColumnTokens` into the expression.
* ColumnValuePairs from `this.mexpColumnValuePairs` are added each time the expression is evaluated (see `this.valuesAsNumbers`)
*/
get mexpPostfix() {
if (this.traits.transformation?.expression) {
try {
// Try to parse the expression and then add tokens
const lexed = Mexp.lex(this.traits.transformation.expression, this.mexpColumnTokens);
// Converts to postfix notation
return lexed.toPostfix();
}
catch (error) {
// TODO: deal with error handling when we have it
console.log(`Failed to setup column transformation: \n${this.traits.transformation.expression}\nWith dependencies: ${this.traits.transformation.dependencies?.join(", ")}`);
console.log(error);
}
}
}
/**
* Gets the column values as numbers, and returns information about how many
* rows were successfully converted to numbers and the range of values.
*/
get valuesAsNumbers() {
const numbers = [];
let minimum = Number.MAX_VALUE;
let maximum = -Number.MAX_VALUE;
let numberOfValidNumbers = 0;
let numberOfNonNumbers = 0;
const replaceWithZero = this.traits.replaceWithZeroValues;
const replaceWithNull = this.traits.replaceWithNullValues;
const values = this.values;
for (let i = 0; i < values.length; ++i) {
const value = values[i];
let n;
if (replaceWithZero && replaceWithZero.indexOf(value) >= 0) {
n = 0;
}
else if (replaceWithNull && replaceWithNull.indexOf(value) >= 0) {
n = null;
}
else if (value.length === 0) {
n = null;
}
else {
n = toNumber(values[i]);
// Only count as non number if value isn't actually null
if (value !== "null" && n === null) {
++numberOfNonNumbers;
}
}
if (n !== null) {
// If we have a `math-expression-evaluator` - use it to transform value
if (isDefined(this.mexpPostfix)) {
const columnPairs = this.mexpColumnValuePairs(i, n);
// Only transform value if all columnPairs have been set
// This means that if one of the columnPairs has a null values - the whole expression WON'T be evaluated
if (!Object.values(columnPairs).includes(null)) {
const result = this.mexpPostfix.postfixEval(columnPairs);
n = typeof result === "string" ? toNumber(result) : result;
}
}
}
if (n !== null) {
++numberOfValidNumbers;
minimum = Math.min(minimum, n);
maximum = Math.max(maximum, n);
}
numbers.push(n);
}
return {
values: numbers,
minimum: minimum === Number.MAX_VALUE ? undefined : minimum,
maximum: maximum === -Number.MAX_VALUE ? undefined : maximum,
numberOfValidNumbers: numberOfValidNumbers,
numberOfNonNumbers: numberOfNonNumbers
};
}
/**
* Gets the column values as dates, and returns information about how many
* rows were successfully converted to dates and the range of values.
*/
get valuesAsDates() {
// See ECMA-262 section 15.9.1.1
// http://ecma-international.org/ecma-262/5.1/#sec-15.9.1.1
const maxDate = new Date(8.64e15);
const minDate = new Date(-8.64e15);
const replaceWithNull = this.traits.replaceWithNullValues;
// Approach:
// * See how dd/mm/yyyy parsing goes
// * If mm/dd/yyyy parsing could work instead use that
// * Otherwise try `new Date` for everything
// Try dd/mm/yyyy, but look out for errors that would also make mm/dd/yyyy impossible
let skipMmddyyyy = false;
let parsingFailed = false;
const separators = ["/", "-"];
const centuryFix = (y) => y < 50 ? 2000 + y : y < 100 ? 1900 + y : y;
const ddmmyyyy = (value) => {
// Try dd/mm/yyyy and watch out for failures that would also cross out mm/dd/yyyy
for (const separator of separators) {
const sep1 = value.indexOf(separator);
if (sep1 === -1)
continue; // Try next separator
const sep2 = value.indexOf(separator, sep1 + 1);
if (sep2 === -1) {
// Neither ddmmyyyy nor mmddyyyy
parsingFailed = true;
skipMmddyyyy = true;
return null;
}
const dayString = value.slice(0, sep1);
const monthString = value.slice(sep1 + 1, sep2);
const yearString = value.slice(sep2 + 1);
const d = +dayString;
const m = +monthString;
const y = +yearString;
if (Number.isInteger(d) && Number.isInteger(m) && Number.isInteger(y)) {
if (d > 31 || y > 9999) {
// Neither ddmmyyyy nor mmddyyyy
parsingFailed = true;
skipMmddyyyy = true;
return null;
}
if (m > 12) {
// Probably mmddyyyy
parsingFailed = true;
return null;
}
return new Date(centuryFix(y), m - 1, d);
}
else {
// Neither ddmmyyyy nor mmddyyyy
parsingFailed = true;
skipMmddyyyy = true;
return null;
}
}
// Neither ddmmyyyy nor mmddyyyy
parsingFailed = true;
skipMmddyyyy = true;
return null;
};
const mmddyyyy = (value) => {
// This function only exists to allow mm-dd-yyyy dates
// mm/dd/yyyy dates could be picked up by `new Date`
const separator = "-";
const sep1 = value.indexOf(separator);
if (sep1 === -1) {
parsingFailed = true;
return null;
}
const sep2 = value.indexOf(separator, sep1 + 1);
if (sep2 === -1) {
parsingFailed = true;
return null;
}
const monthString = value.slice(0, sep1);
const dayString = value.slice(sep1 + 1, sep2);
const yearString = value.slice(sep2 + 1);
const d = +dayString;
const m = +monthString;
const y = +yearString;
if (Number.isInteger(d) && Number.isInteger(m) && Number.isInteger(y)) {
if (d > 31 || m > 12 || y > 9999) {
parsingFailed = true;
return null;
}
return new Date(centuryFix(y), m - 1, d);
}
else {
parsingFailed = true;
return null;
}
};
const yyyyQQ = (value) => {
// Is it quarterly data in the format yyyy-Qx ? (Ignoring null values, and failing on any purely numeric values)
if (value[4] === "-" && value[5] === "Q") {
const year = +value.slice(0, 4);
if (!Number.isInteger(year)) {
parsingFailed = true;
return null;
}
const quarter = value.slice(6);
let monthString;
if (quarter === "1") {
monthString = "01/01";
}
else if (quarter === "2") {
monthString = "04/01";
}
else if (quarter === "3") {
monthString = "07/01";
}
else if (quarter === "4") {
monthString = "10/01";
}
else {
parsingFailed = true;
return null;
}
return new Date(centuryFix(year) + "/" + monthString);
}
parsingFailed = true;
return null;
};
const dateConstructor = (value) => {
const ms = Date.parse(value);
if (!Number.isNaN(ms)) {
return new Date(ms);
}
return null;
};
function convertValuesToDates(values, toDate) {
let minimum = maxDate;
let maximum = minDate;
let numberOfValidDates = 0;
let numberOfNonDates = 0;
const dates = [];
for (let i = 0; i < values.length; ++i) {
const value = values[i];
let d;
if ((replaceWithNull && replaceWithNull.indexOf(value) >= 0) ||
value.length === 0) {
dates.push(null);
}
else {
d = toDate(values[i]);
if (d === null) {
++numberOfNonDates;
}
if (d !== null) {
++numberOfValidDates;
minimum = d < minimum ? d : minimum;
maximum = d > maximum ? d : maximum;
}
dates.push(d);
}
if (parsingFailed) {
break;
}
}
return {
values: dates,
minimum: minimum === maxDate ? undefined : minimum,
maximum: maximum === minDate ? undefined : maximum,
numberOfValidDates: numberOfValidDates,
numberOfNonDates: numberOfNonDates
};
}
let result = convertValuesToDates(this.values, ddmmyyyy);
if (!parsingFailed)
return result;
parsingFailed = false;
if (!skipMmddyyyy) {
result = convertValuesToDates(this.values, mmddyyyy);
if (!parsingFailed)
return result;
parsingFailed = false;
}
result = convertValuesToDates(this.values, yyyyQQ);
if (!parsingFailed)
return result;
parsingFailed = false;
return convertValuesToDates(this.values, dateConstructor);
}
get valuesAsJulianDates() {
const valuesAsDates = this.valuesAsDates;
return {
...this.valuesAsDates,
values: valuesAsDates.values.map((date) => date && JulianDate.fromDate(date)),
minimum: valuesAsDates.minimum && JulianDate.fromDate(valuesAsDates.minimum),
maximum: valuesAsDates.maximum && JulianDate.fromDate(valuesAsDates.maximum)
};
}
/**
* Gets the unique values in this column.
*/
get uniqueValues() {
const replaceWithNull = this.traits.replaceWithNullValues;
const values = this.values.map((value) => {
if (value.length === 0) {
return "";
}
else if (replaceWithNull && replaceWithNull.indexOf(value) >= 0) {
return "";
}
return value;
});
const count = countBy(values);
const nullCount = count[""] ?? 0;
delete count[""];
function toArray(key, value) {
return [key, value];
}
const countArray = Object.keys(count).map((key) => toArray(key, count[key]));
countArray.sort(function (a, b) {
return b[1] - a[1];
});
return {
values: countArray.map((a) => a[0]),
counts: countArray.map((a) => a[1]),
numberOfNulls: nullCount
};
}
get valuesAsRegions() {
const values = this.values;
const map = new Map();
const regionType = this.regionType;
if (!isDefined(regionType) || !regionType.loaded) {
// No regions.
return {
numberOfValidRegions: 0,
numberOfNonRegions: values.length,
numberOfRegionsWithMultipleRows: 0,
regionIds: values.map(() => null),
regionIdToRowNumbersMap: map,
uniqueRegionIds: []
};
}
const regionIds = [];
const uniqueRegionIds = new Set();
let numberOfValidRegions = 0;
let numberOfNonRegions = 0;
let numberOfRegionsWithMultipleRows = 0;
for (let i = 0; i < values.length; ++i) {
const value = values[i];
let regionIndex = this.regionType.findRegionIndex(value, this.regionDisambiguationColumn?.values?.[i]);
regionIndex = regionIndex === -1 ? null : regionIndex;
regionIds.push(regionIndex);
if (regionIndex !== null)
uniqueRegionIds.add(regionIndex);
if (regionIndex !== null) {
++numberOfValidRegions;
const rows = map.get(regionIndex);
if (rows === undefined) {
map.set(regionIndex, i);
}
else if (typeof rows === "number") {
numberOfRegionsWithMultipleRows++;
map.set(regionIndex, [rows, i]);
}
else {
rows.push(i);
}
}
else {
++numberOfNonRegions;
}
}
return {
regionIds: regionIds,
uniqueRegionIds: Array.from(uniqueRegionIds),
regionIdToRowNumbersMap: map,
numberOfValidRegions: numberOfValidRegions,
numberOfNonRegions: numberOfNonRegions,
numberOfRegionsWithMultipleRows: numberOfRegionsWithMultipleRows
};
}
/**
* Gets the name of this column. If the column's name is blank, this property
* will return `Column#` where `#` is the zero-based index of the column.
*/
get name() {
const data = this.tableModel.dataColumnMajor;
if (data === undefined ||
data.length < this.columnNumber ||
data[this.columnNumber].length < 1 ||
data[this.columnNumber].length === 0) {
return "Column" + this.columnNumber;
}
return data[this.columnNumber][0];
}
get title() {
return (this.tableModel.columnTitles[this.columnNumber] ??
this.traits.title ??
// If no title set, use `name` and:
// - un-camel case
// - remove underscores
// - capitalise
this.name
.replace(/[A-Z][a-z]/g, (letter) => ` ${letter.toLowerCase()}`)
.replace(/_/g, " ")
.trim()
.toLowerCase()
.replace(/(^\w|\s\w)/g, (m) => m.toUpperCase()));
}
get units() {
return this.tableModel.columnUnits[this.columnNumber]
? this.tableModel.columnUnits[this.columnNumber]
: this.traits.units;
}
/**
* Gets the {@link TableColumnTraits} for this column. The trait are derived
* from the default column plus this column layered on top of the default.
*/
get traits() {
// It is important to match on column name and not column number because the column numbers can vary between stratum
const thisColumn = this.tableModel.columns.find((column) => column.name === this.name);
if (thisColumn !== undefined) {
const result = createCombinedModel(thisColumn, this.tableModel.defaultColumn);
return result;
}
else {
return this.tableModel.defaultColumn;
}
}
/**
* Gets the type of this column. If {@link #traits} has an explicit
* {@link TableColumnTraits#type} specified, it is returned directly.
* Otherwise, the type is guessed from the column name and contents.
*/
get type() {
// Use the explicit column type, if any.
let type;
if (this.traits.type !== undefined &&
stringToTableColumnType(this.traits.type)) {
type = stringToTableColumnType(this.traits.type);
}
if (type) {
return type;
}
else if (this.regionType !== undefined) {
return TableColumnType.region;
}
return (this.guessColumnTypeFromName(this.name) ??
this.guessColumnTypeFromValues());
}
get isScalarBinary() {
if (this.type === TableColumnType.scalar) {
return (this.uniqueValues.values.length === 2 &&
this.uniqueValues.values[0] === "0" &&
this.uniqueValues.values[1] === "1");
}
}
/** Is column ready to be used.
* This will be false if regionType is not loaded
*/
get ready() {
return !isDefined(this.regionType) || this.regionType.loaded;
}
get regionType() {
let regionProvider;
const regionProviderLists = this.tableModel.regionProviderLists ?? [];
if (regionProviderLists.length === 0) {
return undefined;
}
const regionType = this.traits.regionType;
if (regionType !== undefined) {
// Explicit region type specified, we just need to resolve it.
// Return first match in regionProviderLists
regionProvider = regionProviderLists
.map((list) => list.getRegionProvider(regionType))
.find(isDefined);
}
if (!isDefined(regionProvider)) {
// No region type specified, so match the column name against the region aliases.
regionProvider = this.tableModel.matchRegionProvider(this.name);
}
// Load region IDs for region type
// Note: loadRegionIDs is called in TableMixin.forceLoadMapItems()
// So this will only load region IDs if style/regionType changes after initial loadMapItems
runLater(() => regionProvider?.loadRegionIDs());
return regionProvider;
}
get regionDisambiguationColumn() {
if (this.regionType === undefined) {
return undefined;
}
const columnName = this.traits.regionDisambiguationColumn;
if (columnName !== undefined) {
// Resolve the explicit disambiguation column.
return this.tableModel.tableColumns.find((column) => column.name === columnName);
}
// See if the region provider likes any of the table's other columns for
// disambiguation.
const disambigName = this.regionType.findDisambigVariable(this.tableModel.tableColumns.map((column) => column.name));
if (disambigName === undefined) {
return undefined;
}
return this.tableModel.tableColumns.find((column) => column.name === disambigName);
}
/**
* Gets a function that can be used to retrieve the value of this column for
* a given row as a type appropriate for the column {@link #type}. For
* example, if {@link #type} is {@link TableColumnType#scalar}, the value
* will be a number or null.
*/
get valueFunctionForType() {
if (this.type === TableColumnType.scalar) {
const values = this.valuesAsNumbers.values;
return function (rowIndex) {
return values[rowIndex];
};
}
const values = this.values;
return function (rowIndex) {
return values[rowIndex];
};
}
/** Gets value as a type appropriate for the column {@link #type}. For
* example, if {@link #type} is {@link TableColumnType#scalar}, the values
* will be number or null. */
get valuesForType() {
if (this.type === TableColumnType.scalar) {
return this.valuesAsNumbers.values;
}
return this.values;
}
guessColumnTypeFromValues() {
let type;
// We'll treat it as a scalar if _most_ of values can be successfully
// parsed as numbers, i.e. the number of successful parsings is ~10x
// the number of failed parsings. Note that replacements with null
// or zero are counted as neither failed nor successful.
// We need more than 1 number to create a `scalar` column
if (this.valuesAsNumbers.numberOfValidNumbers > 1 &&
this.valuesAsNumbers.numberOfNonNumbers <=
Math.ceil(this.valuesAsNumbers.numberOfValidNumbers * 0.1)) {
type = TableColumnType.scalar;
}
else {
// Lots of strings that can't be parsed as numbers.
// If there are relatively few different values, treat it as an enumeration.
// If there are heaps of different values, treat it as just ordinary
// free-form text.
const uniqueValues = this.uniqueValues.values;
if (
// We need more than 1 unique value (including nulls)
(this.uniqueValues.numberOfNulls ? 1 : 0) + uniqueValues.length > 1 &&
(uniqueValues.length <= 7 ||
// The number of unique values is less than 12% of total number of values
// Or, each value in the column exists 8.33 times on average
uniqueValues.length < this.values.length * 0.12)) {
type = TableColumnType.enum;
}
else {
type = TableColumnType.text;
}
}
return type;
}
guessColumnTypeFromName(name) {
const typeHintSet = [
{ hint: /^(lon|long|longitude|lng)$/i, type: TableColumnType.longitude },
{ hint: /^(lat|latitude)$/i, type: TableColumnType.latitude },
// Hide easting column if scalar
{
hint: /^(easting|eastings)$/i,
type: TableColumnType.hidden,
typeFromValues: TableColumnType.scalar
},
// Hide northing column if scalar
{
hint: /^(northing|northings)$/i,
type: TableColumnType.hidden,
typeFromValues: TableColumnType.scalar
},
// Hide ID columns if they are scalar
{
hint: /^(_id_|id|fid|objectid)$/i,
type: TableColumnType.hidden,
typeFromValues: TableColumnType.scalar
},
{ hint: /^(address|addr)$/i, type: TableColumnType.address },
// Disable until we actually do something with the height data
// {
// hint: /^(.*[_ ])?(depth|height|elevation|altitude)$/i,
// type: TableColumnType.height
// },
{ hint: /^(.*[_ ])?(time|date)/i, type: TableColumnType.time }, // Quite general, eg. matches "Start date (AEST)".
{ hint: /^(year)$/i, type: TableColumnType.time } // Match "year" only, not "Final year" or "0-4 years".
];
const match = typeHintSet.find((hint) => {
if (hint.hint.test(name)) {
if (hint.typeFromValues) {
return hint.typeFromValues === this.guessColumnTypeFromValues();
}
return true;
}
return false;
});
if (match !== undefined) {
return match.type;
}
return undefined;
}
}
__decorate([
computed
], TableColumn.prototype, "values", null);
__decorate([
computed
], TableColumn.prototype, "mexpColumnTokens", null);
__decorate([
computed
], TableColumn.prototype, "mexpPostfix", null);
__decorate([
computed
], TableColumn.prototype, "valuesAsNumbers", null);
__decorate([
computed
], TableColumn.prototype, "valuesAsDates", null);
__decorate([
computed
], TableColumn.prototype, "valuesAsJulianDates", null);
__decorate([
computed
], TableColumn.prototype, "uniqueValues", null);
__decorate([
computed
], TableColumn.prototype, "valuesAsRegions", null);
__decorate([
computed
], TableColumn.prototype, "name", null);
__decorate([
computed
], TableColumn.prototype, "title", null);
__decorate([
computed
], TableColumn.prototype, "units", null);
__decorate([
computed
], TableColumn.prototype, "traits", null);
__decorate([
computed
], TableColumn.prototype, "type", null);
__decorate([
computed
], TableColumn.prototype, "isScalarBinary", null);
__decorate([
computed
], TableColumn.prototype, "ready", null);
__decorate([
computed
], TableColumn.prototype, "regionType", null);
__decorate([
computed
], TableColumn.prototype, "regionDisambiguationColumn", null);
__decorate([
computed
], TableColumn.prototype, "valueFunctionForType", null);
__decorate([
computed
], TableColumn.prototype, "valuesForType", null);
const allCommas = /,/g;
function toNumber(value) {
// Remove commas and try to parse as a number.
const strippedValue = value.replace(allCommas, "").replace("$", "");
if (strippedValue.length === 0) {
// Treat an empty string as not a number rather than as zero.
return null;
}
// `Number()` requires that the entire string form a number, unlike
// parseInt and parseFloat which allow extra non-number characters
// at the end.
const asNumber = Number(strippedValue);
if (!Number.isNaN(asNumber)) {
return asNumber;
}
return null;
}
//# sourceMappingURL=TableColumn.js.map