@d3plus/data
Version:
JavaScript data loading, manipulation, and analysis functions.
395 lines (362 loc) • 16.3 kB
JavaScript
/*
@d3plus/data v3.0.6
JavaScript data loading, manipulation, and analysis functions.
Copyright (c) 2025 D3plus - https://d3plus.org
@license MIT
*/
(function (factory) {
typeof define === 'function' && define.amd ? define(factory) :
factory();
})((function () { 'use strict';
if (typeof window !== "undefined") {
(function () {
try {
if (typeof SVGElement === 'undefined' || Boolean(SVGElement.prototype.innerHTML)) {
return;
}
} catch (e) {
return;
}
function serializeNode (node) {
switch (node.nodeType) {
case 1:
return serializeElementNode(node);
case 3:
return serializeTextNode(node);
case 8:
return serializeCommentNode(node);
}
}
function serializeTextNode (node) {
return node.textContent.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>');
}
function serializeCommentNode (node) {
return '<!--' + node.nodeValue + '-->'
}
function serializeElementNode (node) {
var output = '';
output += '<' + node.tagName;
if (node.hasAttributes()) {
[].forEach.call(node.attributes, function(attrNode) {
output += ' ' + attrNode.name + '="' + attrNode.value + '"';
});
}
output += '>';
if (node.hasChildNodes()) {
[].forEach.call(node.childNodes, function(childNode) {
output += serializeNode(childNode);
});
}
output += '</' + node.tagName + '>';
return output;
}
Object.defineProperty(SVGElement.prototype, 'innerHTML', {
get: function () {
var output = '';
[].forEach.call(this.childNodes, function(childNode) {
output += serializeNode(childNode);
});
return output;
},
set: function (markup) {
while (this.firstChild) {
this.removeChild(this.firstChild);
}
try {
var dXML = new DOMParser();
dXML.async = false;
var sXML = '<svg xmlns=\'http://www.w3.org/2000/svg\' xmlns:xlink=\'http://www.w3.org/1999/xlink\'>' + markup + '</svg>';
var svgDocElement = dXML.parseFromString(sXML, 'text/xml').documentElement;
[].forEach.call(svgDocElement.childNodes, function(childNode) {
this.appendChild(this.ownerDocument.importNode(childNode, true));
}.bind(this));
} catch (e) {
throw new Error('Error parsing markup string');
}
}
});
Object.defineProperty(SVGElement.prototype, 'innerSVG', {
get: function () {
return this.innerHTML;
},
set: function (markup) {
this.innerHTML = markup;
}
});
})();
}
}));
(function (global, factory) {
typeof exports === 'object' && typeof module !== 'undefined' ? factory(exports, require('d3-request'), require('@d3plus/dom'), require('d3-array'), require('d3-collection')) :
typeof define === 'function' && define.amd ? define('@d3plus/data', ['exports', 'd3-request', '@d3plus/dom', 'd3-array', 'd3-collection'], factory) :
(global = typeof globalThis !== 'undefined' ? globalThis : global || self, factory(global.d3plus = {}, global.d3Request, global.dom, global.d3Array, global.d3Collection));
})(this, (function (exports, d3Request, dom, d3Array, d3Collection) { 'use strict';
/**
@function isData
@desc Returns true/false whether the argument provided to the function should be loaded using an internal XHR request. Valid data can either be a string URL or an Object with "url" and "headers" keys.
@param {*} dataItem The value to be tested
*/ var isData = ((dataItem)=>typeof dataItem === "string" || typeof dataItem === "object" && dataItem.url && dataItem.headers);
/**
@function dataFold
@desc Given a JSON object where the data values and headers have been split into separate key lookups, this function will combine the data values with the headers and returns one large array of objects.
@param {Object} json A JSON data Object with `data` and `headers` keys.
@param {String} [data = "data"] The key used for the flat data array inside of the JSON object.
@param {String} [headers = "headers"] The key used for the flat headers array inside of the JSON object.
*/ var fold = ((json, data = "data", headers = "headers")=>json[data].map((data)=>json[headers].reduce((obj, header, i)=>(obj[header] = data[i], obj), {})));
/**
@function dataConcat
@desc Reduce and concat all the elements included in arrayOfArrays if they are arrays. If it is a JSON object try to concat the array under given key data. If the key doesn't exists in object item, a warning message is lauched to the console. You need to implement DataFormat callback to concat the arrays manually.
@param {Array} arrayOfArray Array of elements
@param {String} [data = "data"] The key used for the flat data array if exists inside of the JSON object.
*/ var concat = ((arrayOfArrays, data = "data")=>arrayOfArrays.reduce((acc, item)=>{
let dataArray = [];
if (Array.isArray(item)) {
dataArray = item;
} else {
if (item[data]) {
dataArray = item[data];
}
// else {
// console.warn(`d3plus-viz: Please implement a "dataFormat" callback to concat the arrays manually (consider using the d3plus.dataConcat method in your callback). Currently unable to concatenate (using key: "${data}") the following response:`, item);
// }
}
return acc.concat(dataArray);
}, []));
/**
@function dataLoad
@desc Loads data from a filepath or URL, converts it to a valid JSON object, and returns it to a callback function.
@param {Array|String} path The path to the file or url to be loaded. Also support array of paths strings. If an Array of objects is passed, the xhr request logic is skipped.
@param {Function} [formatter] An optional formatter function that is run on the loaded data.
@param {String} [key] The key in the `this` context to save the resulting data to.
@param {Function} [callback] A function that is called when the final data is loaded. It is passed 2 variables, any error present and the data loaded.
*/ function load(path, formatter, key, callback) {
let parser;
const getParser = (path)=>{
const ext = path.slice(path.length - 4);
switch(ext){
case ".csv":
return d3Request.csv;
case ".tsv":
return d3Request.tsv;
case ".txt":
return d3Request.text;
default:
return d3Request.json;
}
};
const validateData = (err, parser, data)=>{
if (parser !== d3Request.json && !err && data && data instanceof Array) {
data.forEach((d)=>{
for(const k in d){
if (!isNaN(d[k])) d[k] = parseFloat(d[k]);
else if (d[k].toLowerCase() === "false") d[k] = false;
else if (d[k].toLowerCase() === "true") d[k] = true;
else if (d[k].toLowerCase() === "null") d[k] = null;
else if (d[k].toLowerCase() === "undefined") d[k] = undefined;
}
});
}
return data;
};
const loadedLength = (loadedArray)=>loadedArray.reduce((prev, current)=>current ? prev + 1 : prev, 0);
const getPathIndex = (url, array)=>array.indexOf(url);
// If path param is a not an Array then convert path to a 1 element Array to re-use logic
if (!(path instanceof Array)) path = [
path
];
const needToLoad = path.find(isData);
let loaded = new Array(path.length);
const toLoad = [];
// If there is a string I'm assuming is a Array to merge, urls or data
if (needToLoad) {
path.forEach((dataItem, ix)=>{
if (isData(dataItem)) toLoad.push(dataItem);
else loaded[ix] = dataItem;
});
} else {
loaded[0] = path;
}
// Load all urls an combine them with data arrays
const alreadyLoaded = loadedLength(loaded);
toLoad.forEach((dataItem)=>{
let headers = {}, url = dataItem;
if (typeof dataItem === "object") {
url = dataItem.url;
headers = dataItem.headers;
}
parser = getParser(url);
const request = parser(url);
for(const key in headers){
if (({}).hasOwnProperty.call(headers, key)) {
request.header(key, headers[key]);
}
}
request.get((err, data)=>{
data = err ? [] : data;
if (data && !(data instanceof Array) && data.data && data.headers) data = fold(data);
data = validateData(err, parser, data);
loaded[getPathIndex(url, path)] = data;
if (loadedLength(loaded) - alreadyLoaded === toLoad.length) {
// Format data
data = loadedLength(loaded) === 1 ? loaded[0] : loaded;
if (this._cache) this._lrucache.set(`${key}_${url}`, data);
if (formatter) {
const formatterResponse = formatter(loadedLength(loaded) === 1 ? loaded[0] : loaded);
if (key === "data" && dom.isObject(formatterResponse)) {
data = formatterResponse.data || [];
delete formatterResponse.data;
this.config(formatterResponse);
} else data = formatterResponse || [];
} else if (key === "data") {
data = concat(loaded, "data");
}
if (key && `_${key}` in this) this[`_${key}`] = data;
if (callback) callback(err, data);
}
});
});
// If there is no data to Load response is immediately
if (toLoad.length === 0) {
loaded = loaded.map((data)=>{
if (data && !(data instanceof Array) && data.data && data.headers) data = fold(data);
return data;
});
// Format data
let data = loadedLength(loaded) === 1 ? loaded[0] : loaded;
if (formatter) {
const formatterResponse = formatter(loadedLength(loaded) === 1 ? loaded[0] : loaded);
if (key === "data" && dom.isObject(formatterResponse)) {
data = formatterResponse.data || [];
delete formatterResponse.data;
this.config(formatterResponse);
} else data = formatterResponse || [];
} else if (key === "data") {
data = concat(loaded, "data");
}
if (key && `_${key}` in this) this[`_${key}`] = data;
if (callback) callback(null, data);
}
}
/**
@function isData
@desc Adds the provided value to the internal queue to be loaded, if necessary. This is used internally in new d3plus visualizations that fold in additional data sources, like the nodes and links of Network or the topojson of Geomap.
@param {Array|String|Object} data The data to be loaded
@param {Function} [data] An optional data formatter/callback
@param {String} data The internal Viz method to be modified
*/ function addToQueue(_, f, key) {
if (!(_ instanceof Array)) _ = [
_
];
const needToLoad = _.find(isData);
if (needToLoad) {
const prev = this._queue.find((q)=>q[3] === key);
const d = [
load.bind(this),
_,
f,
key
];
if (prev) this._queue[this._queue.indexOf(prev)] = d;
else this._queue.push(d);
} else {
this[`_${key}`] = _;
}
}
/**
@function unique
@desc ES5 implementation to reduce an Array of values to unique instances.
@param {Array} arr The Array of objects to be filtered.
@param {Function} [accessor] An optional accessor function used to extract data points from an Array of Objects.
@example <caption>this</caption>
unique(["apple", "banana", "apple"]);
@example <caption>returns this</caption>
["apple", "banana"]
*/ function unique(arr, accessor = (d)=>d) {
const values = arr.map(accessor).map((d)=>d instanceof Date ? +d : d);
return arr.filter((obj, i)=>{
const d = accessor(obj);
return values.indexOf(d instanceof Date ? +d : d) === i;
});
}
/**
@function merge
@desc Combines an Array of Objects together and returns a new Object.
@param {Array} objects The Array of objects to be merged together.
@param {Object} aggs An object containing specific aggregation methods (functions) for each key type. By default, numbers are summed and strings are returned as an array of unique values.
@example <caption>this</caption>
merge([
{id: "foo", group: "A", value: 10, links: [1, 2]},
{id: "bar", group: "A", value: 20, links: [1, 3]}
]);
@example <caption>returns this</caption>
{id: ["bar", "foo"], group: "A", value: 30, links: [1, 2, 3]}
*/ function objectMerge(objects, aggs = {}) {
const availableKeys = unique(d3Array.merge(objects.map((o)=>Object.keys(o)))), newObject = {};
availableKeys.forEach((k)=>{
let value;
if (aggs[k]) value = aggs[k](objects, (o)=>o[k]);
else {
const values = objects.map((o)=>o[k]);
const types = values.map((v)=>v || v === false ? v.constructor : v).filter((v)=>v !== void 0);
if (!types.length) value = undefined;
else if (types.indexOf(Array) >= 0) {
value = d3Array.merge(values.map((v)=>v instanceof Array ? v : [
v
]));
value = unique(value);
if (value.length === 1) value = value[0];
} else if (types.indexOf(String) >= 0) {
value = unique(values);
if (value.length === 1) value = value[0];
} else if (types.indexOf(Number) >= 0) value = d3Array.sum(values);
else if (types.indexOf(Object) >= 0) {
value = unique(values.filter((v)=>v));
if (value.length === 1) value = value[0];
else value = objectMerge(value);
} else {
value = unique(values.filter((v)=>v !== void 0));
if (value.length === 1) value = value[0];
}
}
newObject[k] = value;
});
return newObject;
}
/**
@function nest
@summary Extends the base behavior of d3.nest to allow for multiple depth levels.
@param {Array} *data* The data array to be nested.
@param {Array} *keys* An array of key accessors that signify each nest level.
@private
*/ function nest(data, keys) {
if (!(keys instanceof Array)) keys = [
keys
];
const dataNest = d3Collection.nest();
for(let i = 0; i < keys.length; i++)dataNest.key(keys[i]);
const nestedData = dataNest.entries(data);
return bubble(nestedData);
}
/**
Bubbles up values that do not nest to the furthest key.
@param {Array} *values* The "values" of a nest object.
@private
*/ function bubble(values) {
return values.map((d)=>{
if (d.key && d.values) {
if (d.values[0].key === "undefined") return d.values[0].values[0];
else d.values = bubble(d.values);
}
return d;
});
}
exports.addToQueue = addToQueue;
exports.concat = concat;
exports.fold = fold;
exports.isData = isData;
exports.load = load;
exports.merge = objectMerge;
exports.nest = nest;
exports.unique = unique;
}));
//# sourceMappingURL=d3plus-data.js.map