@alinex/datastore
Version:
Read, work and write data structures from and to differents locations and formats.
388 lines • 14.1 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.compressions = exports.formats = exports.DataStore = void 0;
const path_1 = require("path");
const url_1 = require("url");
const objectPath = require("object-path");
const async_1 = require("@alinex/async");
const data = require("@alinex/data");
const debug_1 = require("debug");
const util = require("util");
const protocol_1 = require("./protocol");
const compressor = require("./compression");
const formatter = require("./format");
const debug = debug_1.default("datastore");
// enable details debugging for core debug usages
const debugDetails = debug_1.default("datastore:details");
let debuglog = util.debuglog;
if (debugDetails.enabled) {
process.env.NODE_DEBUG = process.env.NODE_DEBUG ? `${process.env.NODE_DEBUG},request` : "request";
// @ts-ignore
util.debuglog = (set) => {
//if (set === 'https') return debugDetails;
debugDetails(`See more using NODE_DEBUG=${set}`);
return debuglog(set);
};
}
function parseUri(uri) {
if (!uri.match(/^[a-z]+:/))
uri = `file:${path_1.resolve(uri)}`;
let parsed = new url_1.URL(uri);
return parsed;
}
/**
* DataStore can be used to load, save and access different data structures.
*/
class DataStore {
/**
* Initialize a new data store.
* @param input one or multiple data sources to later be used in load/save
*/
constructor(...input) {
/**
* Real content, ready to be used.
*/
this.data = {};
/**
* Options to be used for load/save and parsing of data formats.
*/
this.options = {};
this._source = [];
this._map = {};
this._load = undefined;
this.changed = false;
if (!input)
return;
this._source = input;
}
/**
* Create a new data store and load it from single source.
* @param input one or multiple data sources
*/
static async load(...input) {
const ds = new DataStore();
return ds.load(...input).then(() => Promise.resolve(ds));
}
/**
* Create a new data store and load it from single source.
* @param source URL to load
* @param options options to load source
*/
static async url(source, options) {
const ds = new DataStore();
return ds.load({ source, options }).then(() => Promise.resolve(ds));
}
/**
* Create a new data store with preset data.
* @param source data structure to be set
*/
static async data(data, source) {
const ds = new DataStore();
ds.data = data;
if (source && !source.startsWith('preset:'))
source = `preset:${source}`;
ds.source = source;
return Promise.resolve(ds);
}
/**
* Load from persistent storage and parse content.
* @param input URL or list to read from
* @return parsed data structure (same like ds.data)
*/
async load(...input) {
// check if already loaded
if (!input.length && this._load)
return Promise.resolve(this.data);
if (input.length)
this._source = input;
// load array
debug("Loading: %o", this._source);
if (this._source.filter(e => e.source && !e.source.startsWith('preset:')).length)
await async_1.default.map(this._source, this.multiload);
// get result
if (this._source.length === 1)
this.data = data.clone(this._source[0].data);
else if (this._source.length) {
const res = data.clone(data.merge(...this._source));
this.data = res.data;
this._map = res.map;
}
debug("Result: %o", this.data);
// all loaded
this.changed = false;
return Promise.resolve(this.data);
}
async multiload(entry, index, list) {
if (!entry.source)
return;
const source = parseUri(entry.source);
const options = Object.assign({}, entry.options); // copy which may be changed in compression
return (protocol_1.default
.load(source, options)
.then(([buffer, meta]) => {
entry.meta = meta;
return compressor.uncompress(source, buffer, options);
})
.then(buffer => {
entry.meta.raw = buffer;
return formatter.parse(source, buffer, options);
})
.then(data => {
entry.data = data;
})
// ignore errors if multiple sources are used
.catch(e => {
debug("ERROR: %s", entry.source, e.message || e);
if (list && list.length == 1)
throw e;
}));
}
/**
* Reload and parse already existing content again.
* @param time number of seconds to wait till reload
* @return flag if reload was done (only if changed)
*/
async reload(time = 0) {
if (!this._source.length)
throw new Error("No source defined to reload DataStore from.");
if (!this._source.filter(e => e.source && !e.source.startsWith('preset:')).length)
throw Error("The datastore is not associated with a path to load from.");
if (!this.changed)
return false;
// don't reload for time seconds
let cache = new Date();
cache.setSeconds(cache.getSeconds() + time);
if (this._load && cache < this._load)
return false;
// check for changes
const list = await async_1.default.map(this._source, e => {
if (!e.source)
return Promise.resolve(undefined);
return protocol_1.default.modified(parseUri(e.source));
});
const change = list.reduce((prev, cur) => {
if (!cur)
return prev;
if (!prev)
return prev;
if (prev < cur)
return cur;
return prev;
});
if (this._load && change && change < this._load)
return false;
await this.load();
return true;
}
/**
* Save data structure to persistent store.
* @param output URL to be stored to
* @return true after storing
*/
async save(output) {
if (output)
this.source = [output];
if (this._source.length > 1)
throw new Error("Storing in multiple source list not possible, yet.");
if (!this._source.length || !this._source[0].source)
throw new Error("No source defined to save DataStore to.");
const source = parseUri(this._source[0].source);
const options = Object.assign({}, this._source[0].options); // copy which may be changed in compression
return formatter
.format(source, this.data, options)
.then(buffer => compressor.compress(source, buffer, options))
.then(buffer => protocol_1.default.save(source, buffer, options))
.then(() => {
this._load = new Date();
this.changed = false;
return true;
});
}
/**
* Only parse an olready loaded buffer into data structure.
* @param uri URL specifying where the data is stored, also used to read the format from if not specified as option
* @param buffer byte array buffer with the file to be parsed
* @param options additional settings
* @return parsed data structure (same like ds.data)
*/
async parse(uri, buffer, options) {
if (uri)
this.source = uri;
if (options)
this._source[0].options = Object.assign(this._source[0].options, options);
if (typeof buffer == "string")
buffer = Buffer.from(buffer);
if (!this._source.length)
this._source = [{ source: "file:/dev/null" }];
const source = parseUri(this._source[0].source || "file:/dev/null");
this.data = await formatter.parse(source, buffer, this.options);
return this.data;
}
/**
* Only format the data structure to be stored later.
* @param uri URL specifying where the data is stored, also used to read the format from if not specified as option
* @param options additional settings
* @return byte array buffer with the file to be stored
*/
async format(uri, options) {
let c_uri = parseUri((this._source.length && this._source[0].source) || "file:/dev/null");
if (uri)
c_uri = parseUri(uri);
let c_options = this.options;
if (options)
c_options = Object.assign({}, this.options, options);
return await formatter.format(c_uri, this.data, c_options);
}
/**
* Get the DataSource list or source URL.
*/
get source() {
if (this._source.length === 1)
return this._source[0].source;
if (this._source.length > 1)
return this._source;
return undefined;
}
/**
* Set the DataSource list or source URL.
* @param data URL specifying where the data is/will be stored
*/
set source(data) {
if (data) {
delete this._load;
if (typeof data === "string")
this._source = [{ source: data }];
else if (Array.isArray(data) && data.length)
this.source = data;
else
throw new Error("Need a DataSource list or source string for DataStore.");
}
else {
this._source = [];
}
}
/**
* Get meta information from the DataSource list or source URL.
*/
get meta() {
if (this._source.length === 1)
return this._source[0].meta;
if (this._source.length > 1)
return this._source.map(e => e.meta);
return undefined;
}
/**
* Get mapping of data elements to it's sources.
*/
get map() {
return this._map;
}
/**
* Check the given path and return true if this element is defined else false.
* @param command filter command to select elements
*/
has(command) {
const res = data.filter(this.data, command || "");
return res ? true : false;
}
/**
* Get the element at the defined path.
* @param command filter command to select elements
* @param fallback default value to be used if not found
*/
get(command, fallback) {
const res = data.filter(this.data, command || "");
return typeof res === undefined ? fallback : res;
}
/**
* Like get but return the result as new DataStore.
* @param command filter command to select elements
* @param fallback default value to be used if not found
*/
filter(command, fallback) {
const res = data.filter(this.data, command || "");
const ds = new DataStore();
ds.data = typeof res === undefined ? fallback : res;
return ds;
}
/**
* Set the value of an element by path specification.
* @param path full way to the selected element in the data structure
* @param value new content to be set
* @param doNotReplace if set to true, only set if it is currently not set
* @return value before or undefined if nothing was set
*/
set(path, value, doNotReplace) {
this.changed = true;
if (path === "" || (Array.isArray(path) && path.length == 0)) {
if (this.data === undefined)
return;
const old = this.data;
this.data = value;
return old;
}
let p = typeof path === "number" ? path : path.split(/\[([^\]]*)\]|\./g).filter(e => e);
return objectPath.set(this.data, p, value, doNotReplace);
}
/**
* Insert elements into array.
* @param path full way to the selected element in the data structure
* @param value content to be added
* @param pos position in the array (0=first, 9999=last, -1=before last)
*/
insert(path, value, pos) {
this.changed = true;
let p = typeof path === "number" ? path : path.split(/\[([^\]]*)\]|\./g).filter(e => e);
objectPath.insert(this.data, p, value, pos);
}
/**
* Add element to the end of the array.
* @param path full way to the selected element in the data structure
* @param values content to be added
*/
push(path, ...values) {
this.changed = true;
let p = typeof path === "number" ? path : path.split(/\[([^\]]*)\]|\./g).filter(e => e);
values.unshift(p);
values.unshift(this.data);
// @ts-ignore
objectPath.push.apply(objectPath, values);
}
/**
* Keep the specified element but make it an empty array or object.
* @param path full way to the selected element in the data structure
* @return the value which was deleted
*/
empty(path) {
this.changed = true;
if (path === "" || (Array.isArray(path) && path.length == 0)) {
const old = this.data;
this.data = {};
return old;
}
let p = typeof path === "number" ? path : path.split(/\[([^\]]*)\]|\./g).filter(e => e);
return objectPath.empty(this.data, p);
}
/**
* Removes a element from the data structure, it will be undefined afterwards.
* @param path full way to the selected element in the data structure
* @return the value which was deleted
*/
delete(path) {
this.changed = true;
if (path === "" || (Array.isArray(path) && path.length == 0)) {
const old = this.data;
delete this.data;
return old;
}
let p = typeof path === "number" ? path : path.split(/\[([^\]]*)\]|\./g).filter(e => e);
return objectPath.del(this.data, p);
}
}
exports.DataStore = DataStore;
const formats = formatter.formats;
exports.formats = formats;
const compressions = compressor.compressions;
exports.compressions = compressions;
exports.default = DataStore;
//# sourceMappingURL=index.js.map