UNPKG

@nymphjs/nymph

Version:

Nymph.js - Nymph ORM

886 lines 38 kB
import fs from 'node:fs'; import { difference } from 'lodash-es'; import ReadLines from 'n-readlines'; import strtotime from 'locutus/php/datetime/strtotime.js'; import { InvalidParametersError, UnableToConnectError, } from '../errors/index.js'; import { xor } from '../utils.js'; // from: https://stackoverflow.com/a/6969486/664915 function escapeRegExp(string) { return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string } /** * A Nymph database driver. */ export default class NymphDriver { nymph = new Proxy({}, { get() { throw new Error('Attempted access of Nymph instance before driver initialization!'); }, }); /** * A cache to make entity retrieval faster. */ entityCache = {}; /** * A counter for the entity cache to determine the most accessed entities. */ entityCount = {}; /** * Protect against infinite loops. */ putDataCounter = 0; /** * Initialize the Nymph driver. * * This is meant to be called internally by Nymph. Don't call this directly. * * @param nymph The Nymph instance. */ init(nymph) { this.nymph = nymph; } posixRegexMatch(pattern, subject, caseInsensitive = false) { const posixClasses = [ // '[[:<:]]', // '[[:>:]]', '[:alnum:]', '[:alpha:]', '[:ascii:]', '[:blank:]', '[:cntrl:]', '[:digit:]', '[:graph:]', '[:lower:]', '[:print:]', '[:punct:]', '[:space:]', '[:upper:]', '[:word:]', '[:xdigit:]', ]; const jsClasses = [ // '\b(?=\w)', // '(?<=\w)\b', '[A-Za-z0-9]', '[A-Za-z]', '[\x00-\x7F]', '[ \t]', // '\s', '[\x00-\x1F\x7F]', // '[\000\001\002\003\004\005\006\007\008\009\010\011\012\013\014'. // '\015\016\017\018\019\020\021\022\023\024\025\026\027\028\029'. // '\030\031\032\033\034\035\036\037\177]', '[0-9]', // '\d', '[\x21-\x7E]', // '[A-Za-z0-9!"#$%&\'()*+,\-./:;<=>?@[\\\]^_`{|}\~]', '[a-z]', '[\x20-\x7E]', // '[A-Za-z0-9!"#$%&\'()*+,\-./:;<=>?@[\\\]^_`{|}\~]', '[!"#$%&\'()*+,-./:;<=>?@[\\\\\\]^_‘{|}~]', // '[!"#$%&\'()*+,\-./:;<=>?@[\\\]^_`{|}\~]', '[ \t\r\n\v\f]', // '[\t\n\x0B\f\r ]', '[A-Z]', '[A-Za-z0-9_]', '[0-9A-Fa-f]', ]; let newPattern = pattern; for (let i = 0; i < posixClasses.length; i++) { newPattern = newPattern.replace(posixClasses[i], () => jsClasses[i]); } let re = new RegExp(newPattern, caseInsensitive ? 'mi' : 'm'); return re.test(subject); } async export(filename) { try { const fhandle = fs.openSync(filename, 'w'); if (!fhandle) { throw new InvalidParametersError('Provided filename is not writeable.'); } for await (let entry of this.exportDataIterator()) { fs.writeSync(fhandle, entry.content); } fs.closeSync(fhandle); } catch (e) { return false; } return true; } async exportPrint() { for await (let entry of this.exportDataIterator()) { console.log(entry.content); } return true; } async importDataIterator(lines, transaction) { const first = lines[Symbol.iterator]().next(); if (first.done || first.value !== '#nex2') { throw new Error('Tried to import a file that is not a NEX v2 file.'); } if (transaction) { await this.internalTransaction('nymph-import'); } try { let guid = null; let sdata = {}; let tags = []; let etype = '__undefined'; for (let line of lines) { if (line.match(/^\s*#/)) { continue; } const entityMatch = line.match(/^\s*{([0-9A-Fa-f]+)}<([-\w_]+)>\[([^\]]*)\]\s*$/); const propMatch = line.match(/^\s*([^=]+)\s*=\s*(.*\S)\s*$/); const uidMatch = line.match(/^\s*<([^>]+)>\[(\d+)\]\s*$/); if (uidMatch) { // Add the UID. await this.importUID({ name: uidMatch[1], value: Number(uidMatch[2]), }); } else if (entityMatch) { // Save the current entity. if (guid) { const cdate = Number(JSON.parse(sdata.cdate)); delete sdata.cdate; const mdate = Number(JSON.parse(sdata.mdate)); delete sdata.mdate; await this.importEntity({ guid, cdate, mdate, tags, sdata, etype }); guid = null; tags = []; sdata = {}; etype = '__undefined'; } // Record the new entity's info. guid = entityMatch[1]; etype = entityMatch[2]; tags = entityMatch[3].split(','); } else if (propMatch) { // Add the variable to the new entity. if (guid) { sdata[propMatch[1]] = propMatch[2]; } } // Clear the entity cache. this.entityCache = {}; } // Save the last entity. if (guid) { const cdate = Number(JSON.parse(sdata.cdate)); delete sdata.cdate; const mdate = Number(JSON.parse(sdata.mdate)); delete sdata.mdate; await this.importEntity({ guid, cdate, mdate, tags, sdata, etype }); } if (transaction) { await this.commit('nymph-import'); } } catch (e) { await this.rollback('nymph-import'); throw e; } return true; } async importData(text, transaction) { return await this.importDataIterator(text.split('\n'), transaction); } async import(filename, transaction) { let rl; try { rl = new ReadLines(filename); } catch (e) { throw new InvalidParametersError('Provided filename is unreadable.'); } const lines = { *[Symbol.iterator]() { let line; while ((line = rl.next())) { yield line.toString('utf8'); } }, }; return await this.importDataIterator(lines, transaction); } checkData(data, sdata, selectors, guid = null, tags = null) { try { const formattedSelectors = this.formatSelectors(selectors).selectors; for (const curSelector of formattedSelectors) { const type = curSelector.type; const typeIsNot = type === '!&' || type === '!|'; const typeIsOr = type === '|' || type === '!|'; let pass = !typeIsOr; for (const k in curSelector) { const key = k; const value = curSelector[key]; if (key === 'type' || value == null) { continue; } const clauseNot = key.substring(0, 1) === '!'; if (key === 'selector' || key === '!selector') { const tmpArr = (Array.isArray(value) ? value : [value]); pass = xor(this.checkData(data, sdata, tmpArr, guid, tags), xor(typeIsNot, clauseNot)); } else { if (key === 'qref' || key === '!qref') { throw new Error("Can't use checkData on qref clauses."); } else { // Check if it doesn't pass any for &, check if it passes any for |. for (const curValue of value) { if (curValue == null) { continue; } const propName = curValue[0]; // Unserialize the data for this variable. if (propName in sdata) { data[propName] = JSON.parse(sdata[propName]); delete sdata[propName]; } switch (key) { case 'guid': case '!guid': pass = xor(guid == propName, xor(typeIsNot, clauseNot)); break; case 'tag': case '!tag': pass = xor((tags ?? []).indexOf(propName) !== -1, xor(typeIsNot, clauseNot)); break; case 'defined': case '!defined': pass = xor(propName in data, xor(typeIsNot, clauseNot)); break; case 'truthy': case '!truthy': pass = xor(propName in data && data[propName], xor(typeIsNot, clauseNot)); break; case 'ref': case '!ref': const testRefValue = curValue[1]; pass = xor(propName in data && this.entityReferenceSearch(data[propName], testRefValue), xor(typeIsNot, clauseNot)); break; case 'equal': case '!equal': const testEqualValue = curValue[1]; pass = xor(propName in data && JSON.stringify(data[propName]) === JSON.stringify(testEqualValue), xor(typeIsNot, clauseNot)); break; case 'contain': case '!contain': const testContainValue = curValue[1]; pass = xor(propName in data && JSON.stringify(data[propName]).indexOf(JSON.stringify(testContainValue)) !== -1, xor(typeIsNot, clauseNot)); break; case 'like': case '!like': const testLikeValue = curValue[1]; pass = xor(propName in data && new RegExp('^' + escapeRegExp(testLikeValue) .replace('%', () => '.*') .replace('_', () => '.') + '$').test(data[propName]), xor(typeIsNot, clauseNot)); break; case 'ilike': case '!ilike': const testILikeValue = curValue[1]; pass = xor(propName in data && new RegExp('^' + escapeRegExp(testILikeValue) .replace('%', () => '.*') .replace('_', () => '.') + '$', 'i').test(data[propName]), xor(typeIsNot, clauseNot)); break; case 'match': case '!match': const testMatchValue = curValue[1]; // Convert a POSIX regex to a JS regex. pass = xor(propName in data && this.posixRegexMatch(testMatchValue, data[propName]), xor(typeIsNot, clauseNot)); break; case 'imatch': case '!imatch': const testIMatchValue = curValue[1]; // Convert a POSIX regex to a JS regex. pass = xor(propName in data && this.posixRegexMatch(testIMatchValue, data[propName], true), xor(typeIsNot, clauseNot)); break; case 'gt': case '!gt': const testGTValue = curValue[1]; pass = xor(propName in data && data[propName] > testGTValue, xor(typeIsNot, clauseNot)); break; case 'gte': case '!gte': const testGTEValue = curValue[1]; pass = xor(propName in data && data[propName] >= testGTEValue, xor(typeIsNot, clauseNot)); break; case 'lt': case '!lt': const testLTValue = curValue[1]; pass = xor(propName in data && data[propName] < testLTValue, xor(typeIsNot, clauseNot)); break; case 'lte': case '!lte': const testLTEValue = curValue[1]; pass = xor(propName in data && data[propName] <= testLTEValue, xor(typeIsNot, clauseNot)); break; } if (!xor(typeIsOr, pass)) { break; } } } } if (!xor(typeIsOr, pass)) { break; } } if (!pass) { return false; } } return true; } catch (e) { this.nymph.config.debugError('nymph', `Failed to check entity data: ${e}`); throw e; } } /** * Remove all copies of an entity from the cache. * * @param guid The GUID of the entity to remove. */ cleanCache(guid) { delete this.entityCache[guid]; } async deleteEntity(entity) { const className = entity.constructor.class; if (entity.guid == null) { return false; } const ret = await this.deleteEntityByID(entity.guid, className); if (ret) { entity.guid = null; entity.cdate = null; entity.mdate = null; } return ret; } /** * Search through a value for an entity reference. * * @param value Any value to search. * @param entity An entity, GUID, or array of either to search for. * @returns True if the reference is found, false otherwise. */ entityReferenceSearch(value, entity) { // Get the GUID, if the passed $entity is an object. const guids = []; if (Array.isArray(entity)) { if (entity[0] === 'nymph_entity_reference') { guids.push(value[1]); } else { for (const curEntity of entity) { if (typeof curEntity === 'string') { guids.push(curEntity); } else if (Array.isArray(curEntity)) { guids.push(curEntity[1]); } else if (typeof curEntity.guid === 'string') { guids.push(curEntity.guid); } } } } else if (typeof entity === 'string') { guids.push(entity); } else if (typeof entity.guid === 'string') { guids.push(entity.guid); } if (Array.isArray(value) && value[0] === 'nymph_entity_reference' && guids.indexOf(value[1]) !== -1) { return true; } // Search through arrays and objects looking for the reference. if (value != null && (Array.isArray(value) || typeof value === 'object')) { for (const key in value) { if (this.entityReferenceSearch(value[key], guids)) { return true; } } } return false; } formatSelectors(selectors, options = {}) { const newSelectors = []; const qrefs = []; for (const curSelector of selectors) { const newSelector = { type: curSelector.type, }; for (const k in curSelector) { const key = k; const value = curSelector[key]; if (key === 'type') { continue; } if (value === undefined) { continue; } if (key === 'qref' || key === '!qref') { const tmpArr = (Array.isArray((value ?? [])[0]) ? value : [value]); const formatArr = []; for (let i = 0; i < tmpArr.length; i++) { const name = tmpArr[i][0]; const [qrefOptions, ...qrefSelectors] = tmpArr[i][1]; const QrefEntityClass = qrefOptions.class ? this.nymph.getEntityClass(qrefOptions.class) : this.nymph.getEntityClass('Entity'); const newOptions = { ...qrefOptions, class: QrefEntityClass, source: options.source, }; const newSelectors = this.formatSelectors(qrefSelectors, options); qrefs.push([newOptions, ...newSelectors.selectors], ...newSelectors.qrefs); formatArr[i] = [name, [newOptions, ...newSelectors.selectors]]; } newSelector[key] = formatArr; } else if (key === 'selector' || key === '!selector') { const tmpArr = (Array.isArray(value) ? value : [value]); const newSelectors = this.formatSelectors(tmpArr, options); newSelector[key] = newSelectors.selectors; qrefs.push(...newSelectors.qrefs); } else if (!Array.isArray(value)) { // @ts-ignore: ts doesn't know what value is here. newSelector[key] = [[value]]; } else if (!Array.isArray(value[0])) { if (value.length === 3 && value[1] == null && typeof value[2] === 'string') { // @ts-ignore: ts doesn't know what value is here. newSelector[key] = [[value[0], strtotime(value[2]) * 1000]]; } else { // @ts-ignore: ts doesn't know what value is here. newSelector[key] = [value]; } } else { // @ts-ignore: ts doesn't know what value is here. newSelector[key] = value.map((curValue) => { if (curValue.length === 3 && curValue[1] == null && typeof curValue[2] === 'string') { return [curValue[0], strtotime(curValue[2]) * 1000]; } return curValue; }); } } newSelectors.push(newSelector); } return { selectors: newSelectors, qrefs }; } iterateSelectorsForQuery(selectors, callback) { const queryParts = []; for (const curSelector of selectors) { let curSelectorQuery = ''; let type = curSelector.type; let typeIsNot = type === '!&' || type === '!|'; let typeIsOr = type === '|' || type === '!|'; for (const key in curSelector) { const value = curSelector[key]; if (key === 'type') { continue; } let curQuery = callback({ key, value, typeIsOr, typeIsNot }); if (curQuery) { if (curSelectorQuery) { curSelectorQuery += typeIsOr ? ' OR ' : ' AND '; } curSelectorQuery += curQuery; } } if (curSelectorQuery) { queryParts.push(curSelectorQuery); } } return queryParts; } getEntitiesRowLike(options, selectors, performQueryCallback, rowFetchCallback, freeResultCallback, getCountCallback, getGUIDCallback, getTagsAndDatesCallback, getDataNameAndSValueCallback) { if (!this.isConnected()) { throw new UnableToConnectError('not connected to DB'); } for (const selector of selectors) { if (!selector || Object.keys(selector).length === 1 || !('type' in selector) || ['&', '!&', '|', '!|'].indexOf(selector.type) === -1) { throw new InvalidParametersError('Invalid query selector passed: ' + JSON.stringify(selector)); } } let entities = []; let count = 0; const EntityClass = options.class ?? this.nymph.getEntityClass('Entity'); const etype = EntityClass.ETYPE; if (options.source === 'client' && options.return === 'object') { throw new InvalidParametersError('Object return type not allowed from client.'); } // Check if the requested entity is cached. if (this.nymph.config.cache && !options.skipCache && selectors.length && 'guid' in selectors[0] && typeof selectors[0].guid === 'number') { // Only safe to use the cache option with no other selectors than a GUID // and tags. if (selectors.length === 1 && selectors[0].type === '&' && (Object.keys(selectors[0]).length === 2 || (Object.keys(selectors[0]).length === 3 && 'tag' in selectors[0]))) { const cacheEntity = this.pullCache(selectors[0]['guid']); if (cacheEntity != null && cacheEntity.guid != null && (!('tag' in selectors[0]) || !(Array.isArray(selectors[0].tag) ? selectors[0].tag : [selectors[0].tag]).find((tag) => tag == null || !cacheEntity.tags.includes(tag)))) { // Return the value from the cache. const { guid, cdate, mdate, tags, data, sdata } = cacheEntity; if (options.return === 'count') { return { result: Promise.resolve(null), process: () => 1, }; } else if (options.return === 'guid') { return { result: Promise.resolve(null), process: () => [guid], }; } else if (options.return === 'object') { return { result: Promise.resolve(null), process: () => [ { guid, cdate, mdate, tags, ...data, ...Object.fromEntries(Object.entries(sdata).map(([name, value]) => [ name, JSON.parse(value), ])), }, ], }; } else { const entity = this.nymph .getEntityClass(EntityClass.class) .factorySync(); if (entity) { entity.$nymph = this.nymph; if (options.skipAc != null) { entity.$useSkipAc(!!options.skipAc); } entity.guid = guid; entity.cdate = cdate; entity.mdate = mdate; entity.tags = tags; entity.$putData(data, sdata, 'server'); return { result: Promise.resolve(null), process: () => [entity], }; } return { result: Promise.resolve(null), process: () => [], }; } } } } try { const formattedSelectors = this.formatSelectors(selectors, options); this.nymph.runQueryCallbacks(options, formattedSelectors.selectors); for (let i = 0; i < formattedSelectors.qrefs.length; i++) { const [options, ...selectors] = formattedSelectors.qrefs[i]; this.nymph.runQueryCallbacks(options, selectors); formattedSelectors.qrefs[i] = [options, ...selectors]; } const { result } = performQueryCallback({ options, selectors: formattedSelectors.selectors, etype, }); return { result, process: () => { let row = rowFetchCallback(); if (options.return === 'count') { while (row != null) { count += getCountCallback(row); row = rowFetchCallback(); } } else if (options.return === 'guid') { while (row != null) { entities.push(getGUIDCallback(row)); row = rowFetchCallback(); } } else { while (row != null) { const guid = getGUIDCallback(row); const tagsAndDates = getTagsAndDatesCallback(row); const tags = tagsAndDates.tags; const cdate = tagsAndDates.cdate; const mdate = tagsAndDates.mdate; let dataNameAndSValue = getDataNameAndSValueCallback(row); // Data. const data = {}; // Serialized data. const sdata = {}; if (dataNameAndSValue.name !== '') { // This do will keep going and adding the data until the // next entity is reached. $row will end on the next entity. do { dataNameAndSValue = getDataNameAndSValueCallback(row); sdata[dataNameAndSValue.name] = dataNameAndSValue.svalue; row = rowFetchCallback(); } while (row != null && getGUIDCallback(row) === guid); } else { // Make sure that $row is incremented :) row = rowFetchCallback(); } if (options.return === 'object') { entities.push({ guid, cdate, mdate, tags, ...data, ...Object.fromEntries(Object.entries(sdata).map(([name, value]) => [ name, JSON.parse(value), ])), }); } else { const entity = EntityClass.factorySync(); entity.$nymph = this.nymph; if (options.skipAc != null) { entity.$useSkipAc(!!options.skipAc); } entity.guid = guid; entity.cdate = cdate; entity.mdate = mdate; entity.tags = tags; this.putDataCounter++; if (this.putDataCounter == 100) { throw new Error('Infinite loop detected in Entity loading.'); } entity.$putData(data, sdata, 'server'); this.putDataCounter--; entities.push(entity); } if (this.nymph.config.cache) { this.pushCache(guid, cdate, mdate, tags, data, sdata); } } } freeResultCallback(); return options.return === 'count' ? count : entities; }, }; } catch (e) { return { result: Promise.resolve(e), process: () => { return e; }, }; } } async saveEntityRowLike(entity, saveNewEntityCallback, saveExistingEntityCallback, startTransactionCallback = null, commitTransactionCallback = null) { const originalGuid = entity.guid; const originalCdate = entity.cdate; const originalMdate = entity.mdate; // Get a modified date. let mdate = Date.now(); if (entity.mdate != null) { if (this.nymph.config.updateMDate === false) { mdate = entity.mdate; } else if (typeof this.nymph.config.updateMDate === 'number') { mdate = entity.mdate + this.nymph.config.updateMDate; } } const tags = difference(entity.tags, ['']); const data = entity.$getData(); const sdata = entity.$getSData(); const uniques = await entity.$getUniques(); const EntityClass = entity.constructor; const etype = EntityClass.ETYPE; if (startTransactionCallback) { await startTransactionCallback(); } let success = false; try { if (entity.guid == null) { const cdate = mdate; const newId = entity.$getGuaranteedGUID(); success = await saveNewEntityCallback({ entity, guid: newId, tags, data, sdata, uniques, cdate, etype, }); if (success) { entity.guid = newId; entity.cdate = cdate; entity.mdate = mdate; } } else { // Removed any cached versions of this entity. if (this.nymph.config.cache) { this.cleanCache(entity.guid); } success = await saveExistingEntityCallback({ entity, guid: entity.guid, tags, data, sdata, uniques, mdate, etype, }); if (success) { entity.mdate = mdate; } } if (commitTransactionCallback) { success = await commitTransactionCallback(success); } // Cache the entity. if (success && this.nymph.config.cache) { this.pushCache(entity.guid, entity.cdate, entity.mdate, entity.tags, data, sdata); } } catch (e) { entity.guid = originalGuid; entity.cdate = originalCdate; entity.mdate = originalMdate; throw e; } if (!success) { entity.guid = originalGuid; entity.cdate = originalCdate; entity.mdate = originalMdate; } return success; } /** * Pull an entity from the cache. * * @param guid The entity's GUID. * @returns The entity's data or null if it's not cached. */ pullCache(guid) { // Increment the entity access count. if (!(guid in this.entityCount)) { this.entityCount[guid] = 0; } this.entityCount[guid]++; if (guid in this.entityCache) { return { guid, cdate: this.entityCache[guid]['cdate'], mdate: this.entityCache[guid]['mdate'], tags: this.entityCache[guid]['tags'], data: this.entityCache[guid]['data'], sdata: this.entityCache[guid]['sdata'], }; } return null; } /** * Push an entity onto the cache. * * @param guid The entity's GUID. * @param cdate The entity's cdate. * @param mdate The entity's mdate. * @param tags The entity's tags. * @param data The entity's data. * @param sdata The entity's sdata. */ pushCache(guid, cdate, mdate, tags, data, sdata) { if (guid == null) { return; } // Increment the entity access count. if (!(guid in this.entityCount)) { this.entityCount[guid] = 0; } this.entityCount[guid]++; // Check the threshold. if (this.entityCount[guid] < this.nymph.config.cacheThreshold) { return; } // Cache the entity. this.entityCache[guid] = { cdate, mdate, tags, data, sdata, }; while (this.nymph.config.cacheLimit && Object.keys(this.entityCache).length >= this.nymph.config.cacheLimit) { // Find which entity has been accessed the least. const least = Object.entries(this.entityCount) .sort(([_guidA, countA], [_guidB, countB]) => countA - countB) .pop()?.[0] ?? null; if (least == null) { // This should never happen. return; } // Remove it. delete this.entityCache[least]; delete this.entityCount[least]; } } findReferences(svalue) { const re = /\["nymph_entity_reference","([0-9a-fA-F]+)",/g; const matches = svalue.match(re); if (matches == null) { return []; } return matches.map((match) => match.replace(re, '$1')); } } //# sourceMappingURL=NymphDriver.js.map