terriajs
Version:
Geospatial data visualization platform.
1,070 lines (973 loc) • 88.8 kB
HTML
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>JSDoc: Source: Models/SdmxJsonCatalogItem.js</title>
<script src="scripts/prettify/prettify.js"> </script>
<script src="scripts/prettify/lang-css.js"> </script>
<!--[if lt IE 9]>
<script src="//html5shiv.googlecode.com/svn/trunk/html5.js"></script>
<![endif]-->
<link type="text/css" rel="stylesheet" href="styles/prettify-tomorrow.css">
<link type="text/css" rel="stylesheet" href="styles/jsdoc-default.css">
</head>
<body>
<div id="main">
<h1 class="page-title">Source: Models/SdmxJsonCatalogItem.js</h1>
<section>
<article>
<pre class="prettyprint source linenums"><code>'use strict';
/*global require*/
var Mustache = require('mustache');
var URI = require('urijs');
var naturalSort = require('javascript-natural-sort');
naturalSort.insensitive = true;
var clone = require('terriajs-cesium/Source/Core/clone');
var defaultValue = require('terriajs-cesium/Source/Core/defaultValue');
var defined = require('terriajs-cesium/Source/Core/defined');
var defineProperties = require('terriajs-cesium/Source/Core/defineProperties');
var deprecationWarning = require('terriajs-cesium/Source/Core/deprecationWarning');
var freezeObject = require('terriajs-cesium/Source/Core/freezeObject');
var knockout = require('terriajs-cesium/Source/ThirdParty/knockout');
var loadJson = require('../Core/loadJson');
var when = require('terriajs-cesium/Source/ThirdParty/when');
var arrayProduct = require('../Core/arrayProduct');
var DisplayVariablesConcept = require('../Map/DisplayVariablesConcept');
var inherit = require('../Core/inherit');
var overrideProperty = require('../Core/overrideProperty');
var proxyCatalogItemUrl = require('./proxyCatalogItemUrl');
var RegionMapping = require('../Models/RegionMapping');
var runLater = require('../Core/runLater');
var sdmxJsonLib = require('../ThirdParty/sdmxjsonlib');
var SummaryConcept = require('../Map/SummaryConcept');
var TableCatalogItem = require('./TableCatalogItem');
var TableColumn = require('../Map/TableColumn');
var TableStructure = require('../Map/TableStructure');
var TerriaError = require('../Core/TerriaError');
var VariableConcept = require('../Map/VariableConcept');
var VarType = require('../Map/VarType');
/**
* A {@link CatalogItem} representing region-mapped data obtained from SDMX-JSON format.
*
* Descriptions of this format are available at:
* - https://data.oecd.org/api/sdmx-json-documentation/
* - https://github.com/sdmx-twg/sdmx-json/tree/master/data-message/docs
* - https://sdmx.org/
* - http://stats.oecd.org/sdmx-json/ (hosts a handy query builder)
*
* The URL can be of two types, eg:
* 1. http://example.com/sdmx-json/data/DATASETID/BD1+BD2.LGA.1+2.A/all?startTime=2013&endTime=2013
* 2. http://example.com/sdmx-json/data/DATASETID
*
* For #2, the dimension names and codes come from (in json format):
* http://example.com/sdmx-json/dataflow/DATASETID
*
* @alias SdmxJsonCatalogItem
* @constructor
* @extends TableCatalogItem
*
* @param {Terria} terria The Terria instance.
* @param {String} [url] The base URL from which to retrieve the data.
*/
var SdmxJsonCatalogItem = function(terria, url) {
TableCatalogItem.call(this, terria, url);
// We will override item.url to show a custom URL in About This Dataset.
// So save the original URL here.
this._originalUrl = url;
// The options that should be passed to TableColumn when creating a new column.
this._columnOptions = undefined;
// Allows conversion between the dimensions and the table columns.
this._allDimensions = undefined;
this._loadedDimensions = undefined;
// Keep track of whether how many columns appear before the value columns (typically a time and a region column).
this._numberOfInitialColumns = undefined;
// Holds the time_period and region ids, ie. by default ["TIME_PERIOD", "REGION"].
this._suppressedIds = [];
// This is set to the dataflow URL for this data, if relevant.
this._dataflowUrl = undefined;
// The array of Concepts to display in the NowViewing panel.
this._concepts = [];
// An object containing all the region totals (eg. populations) required, keyed by the dimensionIdsRequestString used.
this._regionTotals = {};
/**
* Gets or sets the 'data' SDMX URL component, eg. 'data' in http://stats.oecd.org/sdmx-json/data/QNA.
* Defaults to 'data'.
* @type {String}
*/
this.dataUrlComponent = undefined;
/**
* Gets or sets the 'dataflow' SDMX URL component, eg. 'dataflow' in http://stats.oecd.org/sdmx-json/dataflow/QNA.
* Defaults to 'dataflow'.
* @type {String}
*/
this.dataflowUrlComponent = undefined;
/**
* Gets or sets the provider id in the SDMX URL, eg. the final 'all' in http://stats.oecd.org/sdmx-json/data/QNA/.../all.
* Defaults to 'all'.
* @type {String}
*/
this.providerId = undefined;
/**
* Gets or sets the SDMX region-type dimension id used with the region code to set the region type.
* Usually defaults to 'REGIONTYPE'.
* @type {String}
*/
this.regionTypeDimensionId = undefined;
/**
* Gets or sets the SDMX region dimension id, which is not displayed as a user-choosable dimension. Defaults to 'REGION'.
* @type {String}
*/
this.regionDimensionId = undefined;
/**
* Gets or sets the SDMX frequency dimension id. Defaults to 'FREQUENCY'.
* @type {String}
*/
this.frequencyDimensionId = undefined;
/**
* Gets or sets the SDMX time period dimension id, which is not displayed as a user-choosable dimension. Defaults to 'TIME_PERIOD'.
* @type {String}
*/
this.timePeriodDimensionId = undefined;
/**
* Gets or sets the regiontype directly, which is an alternative to including a regiontype in the data.
* Eg. "cnt3" would tell us that we should use cnt3 as the table column name.
* By default this is undefined.
* @type {String}
*/
this.regionType = undefined;
/**
* Gets or sets a Mustache template used to turn the name of the region provided in the "regionType" variable
* into a csv-geo-au-compliant column name. The Mustache variable "{{name}}" holds the original name.
* You can use this to specify a year in the name, even if it is absent on the server.
* Eg. "{{name}}_code_2016" converts STE to STE_code_2016.
* By default this is undefined. If it is undefined, the following rules are applied:
* - If there's a _, replace the last one with _code_; else append _code. So SA4 -> SA4_code; SA4_2011 -> SA4_code_2011.
* - If the name ends in 4 digits without an underscore, insert "_code_", eg. LGA2011 -> LGA_code_2011.
* @type {String}
*/
this.regionNameTemplate = undefined;
/**
* Gets or sets the concepts which are initially selected, eg. {"MEASURE": ["GDP", "GNP"], "FREQUENCY": ["A"]}.
* Defaults to the first value in each dimension (when undefined).
* @type {Object}
*/
this.selectedInitially = undefined;
/**
* Gets or sets the dimensions for which you can only select a single value at a time.
* The frequency and regiontype dimensions are added to this list in allSingleValuedDimensionIds.
* @type {String[]}
*/
this.singleValuedDimensionIds = [];
/**
* Gets or sets the startTime to use as part of the ?startTime=...&endTime=... query parameters.
* Currently a string, but could be extended to be an object with frequency codes as keys.
* By default this is undefined, and not used as part of the query.
* @type {String}
*/
this.startTime = undefined;
/**
* Gets or sets the endTime to use as part of the ?startTime=...&endTime=... query parameters.
* Currently a string, but could be extended to be an object with frequency codes as keys.
* By default this is undefined, and not used as part of the query.
* @type {String}
*/
this.endTime = undefined;
/**
* Gets or sets each dimension's allowed values, by id. Eg. {"SUBJECT": ["GDP", "GNP"], "FREQUENCY": ["A"]}.
* If not defined, all values are allowed.
* If a dimension is not present, all values for that dimension are allowed.
* Note this will not be applied to regions or time periods.
* The expression is first matched as a regular expression (sandwiched between ^ and &);
* if that fails, it is matched as a literal string. So eg. "[0-9]+" will match 015 but not A015.
* @type {Object}
*/
this.whitelist = {};
/**
* Gets or sets each dimension's non-allowed values, by id. Eg. {"COB": ["TOTAL", "1"], "FREQUENCY": ["Q"]}.
* If not defined, all values are allowed (subject to the whitelist).
* If a dimension is not present, all values for that dimension are allowed (subject to the whitelist).
* Note this will not be applied to regions or time periods.
* If the same value is in both the whitelist and the blacklist, the blacklist wins.
* The expression is first matched as a regular expression (sandwiched between ^ and &);
* if that fails, it is matched as a literal string. So eg. "[0-9]+" will match 015 but not A015.
* @type {Object}
*/
this.blacklist = {};
/**
* Gets or sets an array of dimension ids whose values should not be shown in the Now Viewing panel;
* instead, their values should be aggregated and treated as a single value.
* Eg. useful if a dimension is repeated (eg. STATE and REGION).
* NOTE: Currently only a single aggregatedDimensionId is supported.
* This should not be applied to regions or time periods.
* @type {Object}
*/
this.aggregatedDimensionIds = [];
/**
* Gets or sets how to re-sort the values that appear in the SDMX-JSON response, in the Now Viewing panel.
* The default is null, so that the order is maintained (except for totalValueIds, which are moved to the top).
* By setting this to 'name' or 'id', the values are sorted into alphabetical and/or numerical order either by name or by id,
* respectively.
* @type {String}
*/
this.sortValues = null;
/**
* Gets or sets value ids for each dimension which correspond to total values.
* Place the grand total first.
* If all dimensions (except region-type, region and frequency) have totals
* available, then a "Display as a percentage of regional total" option becomes available.
* Eg. Suppose AGE had "10" for 10 year olds, etc, plus "ALL" for all ages, "U21" and "21PLUS" for under and over 21 year olds.
* Then you would want to specify {"AGE": ["ALL", "U21", "21PLUS"]}.
* In this case, when the user selects one of these values, any other values will be unselected.
* And when the user selects any other value (eg. "10"), if any of these values were selected, they will be unselected.
* In addition, any values provided under a wildcard "*" key are used for _all_ dimensions, and are shown first in the list,
* if present, eg. {"*": ["ALL"], "AGE": ["U21", "21PLUS"]}.
* @type {Object}
*/
this.totalValueIds = {};
/**
* Gets or sets whether to remove trailing "(x)"s from the values that appear in the SDMX-JSON response.
* If true, for example, "Total responses(c)" would be replaced with "Total responses".
* This is a workaround for an ABS-specific issue.
* Default false.
* @type {Boolean}
*/
this.cleanFootnotes = false;
/**
* Gets or sets whether this item can show percentages instead of raw values.
* This is set to true automatically if total value ids are available on all necessary columns.
* This property is observable.
* @type {Boolean}
* @default false
*/
this.canDisplayPercent = false;
/**
* Gets or sets whether to show percentages or raw values. This property is observable.
* @type {Boolean}
* @default false
*/
this.displayPercent = false;
/**
* Gets or sets a mapping of concept ids to arrays of values which, if selected, mean the results cannot be summed.
* If one of these values is chosen:
* - Does not show the "canDisplayPercent" option.
* - Explains to the user that it can't show multiple values of concepts.
* eg. {"MEASURE": ["rate"]}.
* Can also be the boolean "true", if it should apply to all selections.
* Defaults to none.
* @type {Object|Boolean}
*/
this.cannotSum = undefined;
/**
* Deprecated. Use cannotSum instead.
* Defaults to none.
* @type {Object}
*/
this.cannotDisplayPercentMap = undefined;
/**
* Gets or sets a flag which determines whether the legend comes before (false) or after (true) the display variable choice.
* Default true.
* @type {Boolean}
*/
this.displayChoicesBeforeLegend = true;
/**
* Gets or sets an array of dimension ids which, if present, should be shown to the user, even if there is only one value.
* This is useful if the name of the dataset doesn't convey what is in it, but one of the dimension values does. Eg. ['MEASURE'].
* Default [].
* @type {Boolean}
*/
this.forceShowDimensionIds = [];
// Tracking _concepts makes this a circular object.
// _concepts (via concepts) is both set and read in rebuildData.
// A solution to this would be to make concepts a Promise, but that would require changing the UI side.
knockout.track(this, ['_concepts', 'displayPercent', 'canDisplayPercent']);
overrideProperty(this, 'concepts', {
get: function() {
return this._concepts;
}
});
// See explanation in the comments for TableCatalogItem.
overrideProperty(this, 'dataViewId', {
get: function() {
// We need an id that depends on the selected concepts. Just use the dimensionRequestString.
return calculateDimensionRequestString(this, calculateActiveConceptIds(this) || [], this._fullDimensions || []);
}
});
knockout.defineProperty(this, 'activeConcepts', {
get: function() {
const isActive = concept => concept.isActive;
if (defined(this._concepts) && this._concepts.length > 0) {
return this._concepts.map(concept => concept.getNodes(isActive));
}
}
});
knockout.getObservable(this, 'activeConcepts').subscribe(function() {
if (!this.isLoading) {
// Defer the execution of this so that other knockout observables are updated when we look at them.
// In particular, DisplayVariablesConcept's activeItems.
runLater(() => changedActiveItems(this));
}
}, this);
knockout.getObservable(this, 'canDisplayPercent').subscribe(function(canDisplayPercent) {
// If canDisplayPercent becomes false, must also turn off displayPercent.
if (!canDisplayPercent) {
this.displayPercent = false;
}
}, this);
knockout.getObservable(this, 'displayPercent').subscribe(function(displayPercent) {
var item = this;
if (defined(item._tableStructure)) {
item._tableStructure.columns.forEach(function(column) {
if (displayPercent) {
column.isActive = (column.id === 'region percent');
} else {
if (item._concepts.length > 0) {
column.isActive = (column.id === 'total selected');
}
// An example without concepts can only display one thing, so cannot calculate any regional totals.
}
});
}
}, this);
};
inherit(TableCatalogItem, SdmxJsonCatalogItem);
defineProperties(SdmxJsonCatalogItem.prototype, {
/**
* Gets the type of data member represented by this instance.
* @memberOf SdmxJsonCatalogItem.prototype
* @type {String}
*/
type: {
get: function() {
return 'sdmx-json';
}
},
/**
* Gets a human-readable name for this type of data source, 'SDMX-JSON'.
* @memberOf SdmxJsonCatalogItem.prototype
* @type {String}
*/
typeName: {
get: function() {
return 'SDMX-JSON';
}
},
/**
* Gets the set of names of the properties to be serialized for this object for a share link.
* @memberOf ImageryLayerCatalogItem.prototype
* @type {String[]}
*/
propertiesForSharing: {
get: function() {
return SdmxJsonCatalogItem.defaultPropertiesForSharing;
}
},
/**
* Gets the set of functions used to serialize individual properties in {@link CatalogMember#serializeToJson}.
* When a property name on the model matches the name of a property in the serializers object literal,
* the value will be called as a function and passed a reference to the model, a reference to the destination
* JSON object literal, and the name of the property.
* @memberOf SdmxJsonCatalogItem.prototype
* @type {Object}
*/
serializers: {
get: function() {
return SdmxJsonCatalogItem.defaultSerializers;
}
},
/**
* Gets the list of singleValuedDimensionIds with the frequency and region type included.
* @memberOf SdmxJsonCatalogItem.prototype
* @type {String[]}
*/
allSingleValuedDimensionIds: {
get: function() {
return [this.regionTypeDimensionId, this.frequencyDimensionId].concat(this.singleValuedDimensionIds);
}
},
/**
* Gets the original URL of this item.
* @memberOf SdmxJsonCatalogItem.prototype
* @type {String}
*/
originalUrl: {
get: function() {
return defaultValue(this._originalUrl, this.url);
}
}
});
/**
* Gets or sets the default set of properties that are serialized when serializing a {@link CatalogItem}-derived for a
* share link.
* @type {String[]}
*/
SdmxJsonCatalogItem.defaultPropertiesForSharing = clone(TableCatalogItem.defaultPropertiesForSharing);
SdmxJsonCatalogItem.defaultPropertiesForSharing.push('selectedInitially');
SdmxJsonCatalogItem.defaultPropertiesForSharing.push('displayPercent');
freezeObject(SdmxJsonCatalogItem.defaultPropertiesForSharing);
SdmxJsonCatalogItem.defaultSerializers = clone(TableCatalogItem.defaultSerializers);
SdmxJsonCatalogItem.defaultSerializers.selectedInitially = function(item, json) {
// Create the 'selectedInitially' that would start us off with the same active items as are currently shown.
json.selectedInitially = {};
if (item._concepts.length === 0) {
return;
}
item._concepts[0].items.forEach(function(displayConcept) {
json.selectedInitially[displayConcept.id] = displayConcept.items.filter(function(concept) {
return concept.isActive;
}).map(function(concept) {
return concept.id;
});
});
};
SdmxJsonCatalogItem.defaultSerializers.activeConcepts = function() {
// Don't serialize.
};
SdmxJsonCatalogItem.defaultSerializers.url = function(item, json) {
// Put the original URL back in as the url when serializing.
json.url = item.originalUrl;
};
freezeObject(SdmxJsonCatalogItem.defaultSerializers);
// Just the items that would influence the load from the server or the file
SdmxJsonCatalogItem.prototype._getValuesThatInfluenceLoad = function() {
// Reply with the item's url, which is saved into originalUrl during the load process, and overwritten.
return [this.originalUrl];
};
// The URL can have two different forms, which require different handling.
// 1. http://stat.abs.gov.au/sdmx-json/data/ABS_REGIONAL_LGA/CABEE_2.LGA2013.1+.A/all?startTime=2013&endTime=2013
// Read data from this URL directly and construct the table and concepts from it.
// 2. http://stat.abs.gov.au/sdmx-json/data/ABS_REGIONAL_LGA
// Do not attempt to hit this URL directly.
// Instead get the concepts from .../dataflow/ABS_REGIONAL_LGA, and then, whenever the active concepts are changed,
// construct a specific URL like in #1 from those concepts, load the data from it, and construct a table.
// If no 'dataflow' URL is recognizable, revert to #1 behaviour.
// If the URL fits neither form, assume it is a datafile to be handled like #1.
// You can also force the #1 behaviour by blanking out item.dataflowUrlComponent.
// This function returns undefined for #1, and the dataflow URL for #2.
function getDataflowUrl(item) {
if (!item.dataflowUrlComponent) {
return;
}
var dataUrlComponent = '/' + item.dataUrlComponent + '/';
var dataUrlIndex = (item._originalUrl.lastIndexOf(dataUrlComponent));
// If the URL contains /data/, look for how many / terms come after it.
if (dataUrlIndex >= 0) {
var suffix = item._originalUrl.slice(dataUrlIndex + dataUrlComponent.length);
// eg. suffix would be ABS_REGIONAL_LGA/CABEE_2.LGA2013.1+.A/all...
// If it contains a /, and anything after the /, then treat it as #1.
if (suffix.indexOf('/') >= 0 && suffix.indexOf('/') < suffix.length - 1) {
return;
} else {
// return the same URL but with /data/ replaced with /dataflow/.
var dataflowUrlComponent = '/' + item.dataflowUrlComponent + '/';
return item._originalUrl.replace(dataUrlComponent, dataflowUrlComponent);
}
}
}
/*
* We access:
* - result.structure.dimensions.observation[k] for {keyPosition, id, name, values[]}
* to get the name & id of dimension keyPosition and its array of allowed values (with {id, name}).
* - result.structure.dimensions.attributes.dataSet
* can have units, unit multipliers, reference periods (eg. http://stats.oecd.org/sdmx-json/dataflow/QNA).
* - result.structure.dimensions.attributes.observation
* can have time formats and status (eg. estimated value, forecast value).
*
* (Alternatively, in xml format):
* http://stats.oecd.org/restsdmx/sdmx.ashx/GetDataStructure/<dataset id> (eg. QNA).
*
* Data comes from:
* http://example.com/sdmx-json/data/<dataset identifier>/<filter expression>/<agency name>[ ?<additional parameters>]
*
* Eg.
* http://stats.oecd.org/sdmx-json/data/QNA/AUS+AUT.GDP+B1_GE.CUR+VOBARSA.Q/all?startTime=2009-Q2&endTime=2011-Q4
*
* An example from the ABS could be:
* http://stat.abs.gov.au/sdmx-json/data/ABS_REGIONAL_LGA/CABEE_2.LGA2013.1+.A/all?startTime=2013&endTime=2013
*
* Then access:
* - result.structure.dimensions.series[i] for {keyPosition, id, name, values[]}
* to get the name & id of dimension keyPosition and its array of allowed values (with {id, name}).
* - result.structure.dimensions.observation[i] for {role, id, name, values[]}
* to get the name & id of the observations and its array of allowed values (with {id, name}).
* - result.dataSets[0].series[key].observations[t][0] with key = "xx:yy:zz"
* where xx is the index of a value from dimension 0, etc, and t is the time index (eg. 0 for a single time).
*
* Currently, we only parse the first "dataSet" object provided. (This covers all situations of interest to us so far.)
*
* Time seems to be handled specially, at least by the OECD.
* Eg.
* http://stats.oecd.org/sdmx-json/dataflow/QNA shows there are 5 dimensions (result.structure.dimensions.observation): LOCATION, SUBJECT, MEASURE, FREQUENCY, TIME_PERIOD.
* But http://stats.oecd.org/sdmx-json/data/QNA/.B1_GE.VOBARSA.Q/all only returns 4 dimensions (result.structure.dimensions.series): TIME_PERIOD is gone.
* Instead, it has become an observation: result.structure.dimensions.observation[0] has property "values" with lots of {id, name} fields, eg. {id: "1960-Q1", name: "Q1-1960"}.
* And result.dataSets[0].series[key].observations[t] has lots of values for different t, not necessarily including t = 0. (eg. key = "21:0:0:0" starts at t = 140).
*/
SdmxJsonCatalogItem.prototype._load = function() {
// Set some defaults.
this._originalUrl = this.originalUrl; // Since `this.url` is often set after initialization.
this.regionTypeDimensionId = defaultValue(this.regionTypeDimensionId, 'REGIONTYPE');
this.regionDimensionId = defaultValue(this.regionDimensionId, 'REGION');
this.frequencyDimensionId = defaultValue(this.frequencyDimensionId, 'FREQUENCY');
this.timePeriodDimensionId = defaultValue(this.timePeriodDimensionId, 'TIME_PERIOD');
this.providerId = defaultValue(this.providerId, 'all');
this.dataUrlComponent = defaultValue(this.dataUrlComponent, 'data');
this.dataflowUrlComponent = defaultValue(this.dataflowUrlComponent, 'dataflow');
// cannotDisplayPercentMap is deprecated. Replace it with cannotSum.
if (defined(this.cannotDisplayPercentMap)) {
deprecationWarning('cannotDisplayPercentMap is deprecated. Use cannotSum instead.');
if (!defined(this.cannotSum)) {
this.cannotSum = this.cannotDisplayPercentMap;
}
}
this._suppressedIds = [this.regionDimensionId, this.timePeriodDimensionId];
var tableStyle = this._tableStyle;
this._columnOptions = {
displayDuration: tableStyle.displayDuration,
displayVariableTypes: TableStructure.defaultDisplayVariableTypes,
replaceWithNullValues: tableStyle.replaceWithNullValues,
replaceWithZeroValues: tableStyle.replaceWithZeroValues
};
// We pass column options to TableStructure too, but they only do anything if TableStructure itself (eg. via fromJson) adds the columns,
// which is not the case here. We will need to pass them to each call to new TableColumn as well.
this._tableStructure = new TableStructure(this.name, this._columnOptions);
this._regionMapping = new RegionMapping(this, this._tableStructure, tableStyle);
this._dataflowUrl = getDataflowUrl(this);
if (!defined(this.metadataUrl)) {
this.metadataUrl = this._dataflowUrl; // So a link to the metadata appears in About This Dataset.
}
if (this._dataflowUrl) {
return loadDataflow(this); // This eventually triggers loadAndBuildTable too, via changedActiveItem.
} else {
return loadAndBuildTable(this);
}
};
// Sets the tableStructure's columns to the new columns, redraws the map, and closes the feature info panel.
function updateColumns(item, newColumns) {
item._tableStructure.columns = newColumns;
if (item._tableStructure.columns.length === 0) {
// Nothing to show, so the attempt to redraw will fail; need to explicitly hide the existing regions.
item._regionMapping.hideImageryLayer();
item.terria.currentViewer.notifyRepaintRequired();
}
// Close any picked features, as the description of any associated with this catalog item may change.
item.terria.pickedFeatures = undefined;
}
// Adds the wildcard's exclusive values to the front of those for this dimension id, if any.
function getTotalValueIdsForDimensionId(item, dimensionId) {
return (item.totalValueIds['*'] || []).concat(item.totalValueIds[dimensionId] || []);
}
// Trims spaces off rawName
// If cleanFootnotes is true, also removes trailing (x)'s, eg. Total(c) => Total.
function renameValue(item, rawName) {
var trimmedName = rawName.trim();
if (item.cleanFootnotes) {
var length = trimmedName.length;
if ((trimmedName.indexOf('(') === length - 3) && (trimmedName.indexOf(')') === length - 1)) {
return trimmedName.slice(0, length - 3);
}
}
return trimmedName;
}
/**
* Returns an array whose elements are objects describing each dimension.
* The array has length structureSeries.length (assuming the keyPositions are correct),
* and the index of each element is its keyPosition.
* Each element is an object with the properties:
* - dimensionId
* - dimensionName
* - values: An array whose elements describe each allowed value of the dimension (eg. countries, measurement types).
* Each element is an object with the properties:
* - id
* - name
* If there is a whitelist, only the whitelisted values are included.
* If there is a blacklist, blacklisted values are excluded.
* @private
* @param {SdmxJsonCatalogItem} item The SDMX-JSON catalog item.
* @param {Array} structureSeries The structure's series property, json.structure.dimensions.series.
* @return {Object[]} A description of the dimensions.
*/
function buildDimensions(item, structureSeries) {
// getFilter returns a function which can be used in list.filter().
// It tests if the value is in the given list, using regexps if possible.
// filterList can be either item.whitelist or item.blacklist.
// set isWhiteList false if is blacklist, so that the return values are negated (except for a missing list).
function getFilter(filterList, dimensionId, isWhiteList) {
var thisIdsFilterList = filterList[dimensionId];
if (!defined(thisIdsFilterList)) {
return function() { return true; };
}
try {
var thisIdsRegExps = thisIdsFilterList.map(string => new RegExp('^' + string + '$'));
return function(value) {
// Test as a straight string, and if that fails, as a regular expression.
var isPresent = thisIdsFilterList.indexOf(value.id) >= 0 || thisIdsRegExps.map(regExp => value.id.match(regExp)).some(defined);
if (isWhiteList) {
return isPresent;
}
return !isPresent;
};
} catch (e) {
// Cannot intepret as a regular expression.
// Eg. "[" causes Uncaught SyntaxError: Invalid regular expression: /[/: Unterminated character class(…)),
// So just test as a string.
return function(value) {
var isPresent = thisIdsFilterList.indexOf(value.id) >= 0;
if (isWhiteList) {
return isPresent;
}
return !isPresent;
};
}
}
var result = [];
for (var i = 0; i < structureSeries.length; i++) {
var thisSeries = structureSeries[i];
var keyPosition = defined(thisSeries.keyPosition) ? thisSeries.keyPosition : i; // Since time_period can be an observation, without a keyPosition.
var values = thisSeries.values
.filter(getFilter(item.whitelist, thisSeries.id, true))
.filter(getFilter(item.blacklist, thisSeries.id, false));
if (item.sortValues === 'id') {
values = values.sort((a, b) => naturalSort(a.id, b.id));
} else if (item.sortValues === 'name' || item.sortValues === true) {
values = values.sort((a, b) => naturalSort(a.name, b.name));
}
moveTotalValueIdsToFront(values, getTotalValueIdsForDimensionId(item, thisSeries.id));
result[keyPosition] = {
id: thisSeries.id,
name: thisSeries.name,
// Eg. values: [{id: "BD_2", name: "Births"}, {id: "BD_4", name: "Deaths"}].
values: values
};
}
return result;
}
function moveTotalValueIdsToFront(values, totalValueIds) {
if (defined(totalValueIds)) {
// Go in reverse order so the first one in the list ends up in the front at the end.
// Not all exclusive values need be present.
for (var j = totalValueIds.length - 1; j >= 0; j--) {
var exclusiveValue = totalValueIds[j];
var currentIndex = values.map(value => value.id).indexOf(exclusiveValue);
if (currentIndex >= 0) {
// Move it to the top.
values.splice(0, 0, values.splice(currentIndex, 1)[0]);
}
}
}
}
// Return dimensions, but removing:
// - suppressed dimensions,
// - dimensions with only one value in fullDimensions (unless they are in the force-show list)
// - aggregated dimensions.
// Dimensions and fullDimensions must have the same ordering of dimensions.
function getShownDimensions(item, dimensions, fullDimensions) {
return dimensions.filter(function(dimension, i) {
return (item._suppressedIds.indexOf(dimension.id) === -1) &&
// note the logic of the next line is repeated in calculateDimensionRequestString
(fullDimensions[i].values.length > 1 || item.forceShowDimensionIds.indexOf(dimension.id) >= 0) &&
(item.aggregatedDimensionIds.indexOf(dimension.id) === -1);
});
}
/**
* Calculates all the combinations of values that should appear as either:
* - columns in our table (by passing the "loadedDimensions" for a given dataset), or
* - concepts in the Now Viewing panel (by passing the "fullDimensions", ie. those from the dataflow.)
* Does not include suppressed (ie. region or time_period) values.
* Returns an object with properties:
* names: An array, each element of which is an array of the names of each relevant dimension value.
* ids: An array, each element of which is an array of the ids of each relevant dimension value.
* @private
* @param {SdmxJsonCatalogItem} item The catalog item.
* @param {Object[]} dimensions The output of buildDimensions, either fullDimensions or loadedDimensions.
* @param {Object[]} fullDimensions The output of buildDimensions on the dataflow result (or data if no dataflow). Defaults to dimensions.
* @return {Object} The values and names of the dimensions to be shown.
*/
function calculateShownDimensionCombinations(item, dimensions, fullDimensions) {
// Note we need to suppress the time dimension from the dimension list, if any; it appears as an observation instead.
// We also need to suppress the regions.
// Convert the values into all the combinations we'll need to load into columns,
// eg. [[0], [0], [0, 1, 2], [0, 1]] => [[0, 0, 0, 0], [0, 0, 0, 1], [0, 0, 1, 0], [0, 0, 1, 1], [0, 0, 2, 0], [0, 0, 2, 1]].
if (!defined(fullDimensions)) {
fullDimensions = dimensions;
}
var valuesArrays = getShownDimensions(item, dimensions, fullDimensions).map(function(dimension) {
return dimension.values;
});
var idsArrays = valuesArrays.map(function(values) { return values.map(function(value) { return value.id; }); });
var namesArrays = valuesArrays.map(function(values) { return values.map(function(value) { return value.name; }); });
return {
ids: arrayProduct(idsArrays),
names: arrayProduct(namesArrays)
};
}
function getDimensionById(dimensions, id) {
var result;
for (var i = 0; i < dimensions.length; i++) {
if (dimensions[i].id === id) {
result = dimensions[i];
}
}
return result;
}
function getDimensionIndexById(dimensions, id) {
var result;
for (var i = 0; i < dimensions.length; i++) {
if (dimensions[i].id === id) {
result = i;
}
}
return result;
}
function getRegionColumnName(item, dimensions, regionTypeIndex) {
var regionTypeDimension = getDimensionById(dimensions, item.regionTypeDimensionId);
var regionDimension = getDimensionById(dimensions, item.regionDimensionId);
if (defined(regionTypeDimension)) {
// If there is a REGIONTYPE dimension, use its id.
var regionTypeId = regionTypeDimension.values[regionTypeIndex || 0].id;
// If there is a regionNameTemplate, apply it to the id.
if (defined(item.regionNameTemplate)) {
return Mustache.render(item.regionNameTemplate, {"name": regionTypeId});
}
// Fall back to this default approach to convert it to csv-geo-au:
// Assume the raw data is just missing the word "code", eg. SA4 or SA4_2013 should be SA4_code or SA4_code_2013.
// So, if there's a _, replace the last one with _code_; else append _code.
// Also handle the case that the raw data ends in 4 digits without the underscore, eg. LGA2011 -> LGA_code_2011.
var underscoreIndex = regionTypeId.lastIndexOf('_');
if (underscoreIndex >= 0) {
return regionTypeId.slice(0, underscoreIndex) + '_code' + regionTypeId.slice(underscoreIndex);
} else {
var fourDigitSuffixMatch = regionTypeId.match(/(.+)([0-9]{4})$/);
if (defined(fourDigitSuffixMatch)) {
return fourDigitSuffixMatch[1] + '_code_' + fourDigitSuffixMatch[2];
}
return regionTypeId + '_code';
}
} else if (defined(regionDimension)) {
// Else, if there is a REGION dimension and item.regionType has been defined, return item.regionType (and don't append anything).
if (defined(item.regionType) && defined(item.regionType)) {
return item.regionType;
}
// Else, use the REGION dimension id, if present.
return regionDimension.id;
}
}
// If there are times 2010, 2011, and regions AUS, MEX,
// then the table has rows in this order:
// date, region, ...
// 2010, AUS
// 2010, MEX
// 2011, AUS ... etc.
function buildRegionAndTimeColumns(item, dimensions) {
var regionDimension = getDimensionById(dimensions, item.regionDimensionId);
var timePeriodDimension = getDimensionById(dimensions, item.timePeriodDimensionId);
if (!defined(regionDimension) && !defined(timePeriodDimension)) {
// No region dimension (with the actual region values in it) AND no time dimension - we're done.
return [];
}
var regionValues = [];
var timePeriodValues = [];
var regionCount = defined(regionDimension) ? regionDimension.values.length : 1;
var timePeriodCount = defined(timePeriodDimension) ? timePeriodDimension.values.length : 1;
for (var timePeriodIndex = 0; timePeriodIndex < timePeriodCount; timePeriodIndex++) {
for (var regionIndex = 0; regionIndex < regionCount; regionIndex++) {
if (defined(regionDimension)) {
regionValues.push(regionDimension.values[regionIndex].id);
}
if (defined(timePeriodDimension)) {
timePeriodValues.push(timePeriodDimension.values[timePeriodIndex].id);
}
}
}
var timePeriodColumn;
if (defined(timePeriodDimension)) {
var thisColumnOptions = clone(item._columnOptions);
if (timePeriodCount === 1) {
thisColumnOptions.type = VarType.ENUM; // Don't trigger timeline off a single-valued time dimension.
}
timePeriodColumn = new TableColumn('date', timePeriodValues, thisColumnOptions);
}
if (!defined(regionDimension)) {
return [timePeriodColumn];
}
// If there are multiple region types in the data, only use the first region type.
var regionColumnName = getRegionColumnName(item, dimensions, 0);
var regionColumn = new TableColumn(regionColumnName, regionValues, item._columnOptions);
if (defined(timePeriodDimension) && defined(regionDimension)) {
return [timePeriodColumn, regionColumn];
} else {
return [regionColumn];
}
}
// Sums an array, treating undefined's as 0 in the sum, but leaving undefined + undefined = undefined.
// (Note the "defined" function catches null and defined.)
function sumArray(array) {
return array.filter(defined).reduce((x, y) => x + y, null);
}
// Eg. ids = ['GDP', 'METHOD-B'].
// We want to map this to an array of ids, eg. ['GDP, 'METHOD-B', undefined, 'LGA', 'A'] for SUBJECT, METHOD, REGION, REGIONTYPE, FREQUENCY (for example),
// with undefined for any suppressed dimensions and values[0] for all single-valued dimensions.
// This can then easily be turned into a colon-separated key.
function mapDimensionValueIdsToKeyValues(item, loadedDimensions, ids) {
var result = [];
var shownDimensions = getShownDimensions(item, loadedDimensions, item._fullDimensions);
var shownDimensionIds = shownDimensions.map(function(dimension) { return dimension.id; });
for (var dimensionIndex = 0; dimensionIndex < loadedDimensions.length; dimensionIndex++) {
var outputDimension = loadedDimensions[dimensionIndex];
var i = shownDimensionIds.indexOf(outputDimension.id);
if (i >= 0) {
result.push(ids[i]);
} else if (item._suppressedIds.indexOf(outputDimension.id) >= 0) {
result.push(undefined);
} else {
result.push(outputDimension.values[0].id);
}
}
return result;
}
// Slightly generalise dataSets[key].obsValue to handle undefined data, and the
// possibility of aggregating across some dimensions.
function getObsValue(item, loadedDimensions, dimensionIndices, dataSets) {
// Usually no dimensions need to be aggregated, so just return dataSets[key].obsValue.
var key = dimensionIndices.join(':');
var valueObject = dataSets[key];
if (item.aggregatedDimensionIds.length === 0) {
return defined(valueObject) ? valueObject.obsValue : null;
}
// This implementation can handle at most a single aggregated dimension id.
if (item.aggregatedDimensionIds.length === 1) {
var aggregatedDimensionId = item.aggregatedDimensionIds[0];
var valuesToAggregate = [];
var aggregatedDimensionIndex = getDimensionIndexById(loadedDimensions, aggregatedDimensionId);
var dimension = getDimensionById(loadedDimensions, aggregatedDimensionId);
if (!defined(dimension)) {
console.warn('Tried to aggregate on a dimension that doesn\'t exist, ' + aggregatedDimensionId);
return defined(valueObject) ? valueObject.obsValue : null;
}
dimension.values.forEach(function(thisValueObject) {
dimensionIndices[aggregatedDimensionIndex] = thisValueObject.id;
var thisKey = dimensionIndices.join(':');
thisValueObject = dataSets[thisKey];
if (defined(thisValueObject)) {
valuesToAggregate.push(thisValueObject.obsValue);
}
});
return sumArray(valuesToAggregate);
}
console.warn('SDMX-JSON aggregatedDimensionIds - only a single dimension id is implemented.');
}
// Create a column for each combination of (non-region) dimension values.
// The column has values for each region.
function buildValueColumns(item, loadedDimensions, columnCombinations, dataSets) {
var thisColumnOptions = clone(item._columnOptions);
thisColumnOptions.tableStructure = item._tableStructure;
var columns = [];
var hasNoConcepts = (item._concepts.length === 0);
var regionDimension = getDimensionById(loadedDimensions, item.regionDimensionId);
var regionDimensionIndex = getDimensionIndexById(loadedDimensions, item.regionDimensionId);
var timePeriodDimension = getDimensionById(loadedDimensions, item.timePeriodDimensionId);
var timePeriodDimensionIndex = getDimensionIndexById(loadedDimensions, item.timePeriodDimensionId);
var regionCount = defined(regionDimension) ? regionDimension.values.length : 1;
var timePeriodCount = defined(timePeriodDimension) ? timePeriodDimension.values.length : 1;
for (var combinationIndex = 0; combinationIndex < columnCombinations.ids.length; combinationIndex++) {
var ids = columnCombinations.ids[combinationIndex];
var dimensionIndices = mapDimensionValueIdsToKeyValues(item, loadedDimensions, ids);
// The name is just the joined names of all the columns involved, or 'value' if no columns still have names.
var combinationName = columnCombinations.names[combinationIndex].filter(function(name) {return !!name; }).join(' ') || 'Value';
var combinationId = ids.join(' ') || 'Value';
var values = [];
for (var timePeriodIndex = 0; timePeriodIndex < timePeriodCount; timePeriodIndex++) {
for (var regionIndex = 0; regionIndex < regionCount; regionIndex++) {
if (defined(regionDimensionIndex)) {
dimensionIndices[regionDimensionIndex] = regionDimension.values[regionIndex].id;
}
if (defined(timePeriodDimensionIndex)) {
dimensionIndices[timePeriodDimensionIndex] = timePeriodDimension.values[timePeriodIndex].id;
}
values.push(getObsValue(item, loadedDimensions, dimensionIndices, dataSets));
}
}
thisColumnOptions.id = combinationId; // So we can refer to the dimension in a template by a sequence of ids or names.
var column = new TableColumn(combinationName, values, thisColumnOptions);
if (hasNoConcepts) {
// If there are no concepts displayed to the user, there is only one value column, and we won't add a "total" column.
// So make this column active.
column.isActive = true;
}
columns.push(column);
}
return columns;
}
// Map the active concepts into arrays of arrays of ids.
// Eg. Return [['GDP', 'GNP'], ['Q']].
function calculateActiveConceptIds(item) {
if (item._concepts.length === 0) {
return [];
}
var conceptItems = item._concepts[0].items;
return conceptItems.map(parent =>
// Note this wouldn't work whilst following through on the activeItems knockout subscription,
// if we hadn't wrapped that in a runLater. You would have had to explicitly call
// parent.items.filter(concept => concept.isActive).map(concept => concept.id)
parent.activeItems.map(concept => concept.id)
);
}
// Map the _total_ concept ids into arrays of (arrays of length 1).
// If a total is not available for a dimension, but it is in singleValuedDimensionIds,
// then use its current value instead - eg. region type, frequency, and some measure types.
// Returns undefined if any concepts do not have a total available.
// Eg. Return [['TOT'], ['3'], ['TOT']].
function calculateTotalConceptIds(item) {
if (item._concepts.length === 0) {
return [];
}
var conceptItems = item._concepts[0].items;
var totalConceptIds = conceptItems.map(function(parent) {
// Because any available totals are sorted to the top of the list of children,
// and the grand total is always the first one, just check if the first child
// is in the list of exclusiveChildIds for the parent, and return it if so.
var firstChildId = parent.items[0].id;
if (parent.exclusiveChildIds.indexOf(firstChildId) >= 0) {
return [firstChildId];
}
// Dimensions in singleValuedDimensionIds became concepts with allowMultiple false.
if (!parent.allowMultiple) {
// Needs runLater for activeItems to work correctly - see comment above.
return parent.activeItems.map(concept => concept.id);
}
});
// totalConceptIds will have undefined elements if it couldn't find a total for a dimension.
// Only return the array if every element is defined. Otherwise, return undefined.
if (totalConceptIds.every(defined)) {
return totalConceptIds;
}
return undefined;
}
// Check if item.selectedInitially has at least one value that exists in dimension.values,
// and if it doesn't, reset item.selectedInitially.
function fixSelectedInitially(item, conceptDimensions) {
conceptDimensions.forEach(dimension => {
if (defined(item.selectedInitially)) {
var thisSelectedInitially = item.selectedInitially[dimension.id];
if (thisSelectedInitially) {
var valueIds = dimension.values.map(value => value.id);
if (!thisSelectedInitially.some(initialValue => valueIds.indexOf(initialValue) >= 0)) {
console.warn('Ignoring invalid initial selection ' + thisSelectedInitially + ' on ' + dimension.name);
item.selectedInitially[dimension.id] = undefined;
}
}
}
});
}
// Build out the concepts displayed in the NowViewing panel. Also fixes selectedInitially, if broken.
function buildConcepts(item, fullDimensions) {
function isInitiallyActive(dimensionId, value, index) {
if (!defined(item.selectedInitially)) {
return index === 0;
}
var dimensionSelectedInitially = item.selectedInitially[dimensionId];
if (!defined(dimensionSelectedInitially)) {
return index === 0;
}
return dimensionSelectedInitially.indexOf(value.id) >= 0;
}
var conceptDimensions = getShownDimensions(item, fullDimensions, fullDimensions);
fixSelectedInitially(item, conceptDimensions); // Note side-eff