UNPKG

@netlify/content-engine

Version:
268 lines 10.2 kB
"use strict"; Object.defineProperty(exports, "__esModule", { value: true }); exports.getDefaultDbPath = getDefaultDbPath; exports.getLmdbStore = getLmdbStore; const lmdb_1 = require("lmdb"); const nodes_1 = require("./updates/nodes"); const nodes_by_type_1 = require("./updates/nodes-by-type"); const iterable_1 = require("../common/iterable"); const run_query_1 = require("./query/run-query"); const run_fast_filters_1 = require("../in-memory/run-fast-filters"); const fs_extra_1 = require("fs-extra"); const redux_1 = require("../../redux"); function getDefaultDbPath(cwd = process.cwd()) { const dbFileName = process.env.NODE_ENV === `test` ? `test-datastore-${ // FORCE_TEST_DATABASE_ID will be set if this gets executed in worker context // when running jest tests. JEST_WORKER_ID will be set when this gets executed directly // in test context (jest will use jest-worker internally). process.env.FORCE_TEST_DATABASE_ID ?? process.env.JEST_WORKER_ID}` : `datastore`; return cwd + `/.cache/data/` + dbFileName; } const stores = new Map(); function getLmdbStore({ dbPath = getDefaultDbPath(), } = {}) { if (stores.has(dbPath)) return stores.get(dbPath); const lmdbDatastore = { resetCache, getNode, getTypes, countNodes, iterateNodes, iterateNodesByType, updateDataStore, ready, runQuery, clearIndexes, // deprecated: getNodes, getNodesByType, }; const preSyncDeletedNodeIdsCache = new Set(); let rootDb; let databases; function getRootDb() { if (!rootDb) { if (!dbPath) { throw new Error(`LMDB path is not set!`); } if (!globalThis.__GATSBY_OPEN_ROOT_LMDBS) { globalThis.__GATSBY_OPEN_ROOT_LMDBS = new Map(); } rootDb = globalThis.__GATSBY_OPEN_ROOT_LMDBS.get(dbPath); if (rootDb) { return rootDb; } rootDb = (0, lmdb_1.open)({ name: `root`, path: dbPath, compression: true, }); globalThis.__GATSBY_OPEN_ROOT_LMDBS.set(dbPath, rootDb); } return rootDb; } function getDatabases() { if (!databases) { // __GATSBY_OPEN_LMDBS tracks if we already opened given db in this process // In `gatsby serve` case we might try to open it twice - once for engines // and second to get access to `SitePage` nodes (to power trailing slashes // redirect middleware). This ensure there is single instance within a process. // Using more instances seems to cause weird random errors. if (!globalThis.__GATSBY_OPEN_LMDBS) { globalThis.__GATSBY_OPEN_LMDBS = new Map(); } databases = globalThis.__GATSBY_OPEN_LMDBS.get(dbPath); if (databases) { return databases; } rootDb = getRootDb(); databases = { nodes: rootDb.openDB({ name: `nodes`, // FIXME: sharedStructuresKey breaks tests - probably need some cleanup for it on DELETE_CACHE // sharedStructuresKey: Symbol.for(`structures`), cache: { // expirer: false disables LRU part and only take care of WeakRefs // this way we don't retain nodes strongly, but will continue to // reuse them if they are loaded already expirer: false, }, }), nodesByType: rootDb.openDB({ name: `nodesByType`, dupSort: true, }), metadata: rootDb.openDB({ name: `metadata`, useVersions: true, }), indexes: rootDb.openDB({ name: `indexes`, // TODO: use dupSort when this is ready: https://github.com/DoctorEvidence/lmdb-store/issues/66 // dupSort: true }), }; globalThis.__GATSBY_OPEN_LMDBS.set(dbPath, databases); } return databases; } /** * @deprecated */ function getNodes() { // const start = performance.now() const result = Array.from(iterateNodes()); // const timeTotal = performance.now() - start // console.warn( // `getNodes() is deprecated, use iterateNodes() instead; ` + // `array length: ${result.length}; time(ms): ${timeTotal}` // ) return result ?? []; } /** * @deprecated */ function getNodesByType(type) { // const start = performance.now() const result = Array.from(iterateNodesByType(type)); // const timeTotal = performance.now() - start // console.warn( // `getNodesByType() is deprecated, use iterateNodesByType() instead; ` + // `array length: ${result.length}; time(ms): ${timeTotal}` // ) return result ?? []; } function iterateNodes() { // Additionally fetching items by id to leverage lmdb-store cache const nodesDb = getDatabases().nodes; return new iterable_1.GatsbyIterable(nodesDb .getKeys({ snapshot: false }) .map((nodeId) => typeof nodeId === `string` ? getNode(nodeId) : undefined) .filter(Boolean)); } function iterateNodesByType(type) { const nodesByType = getDatabases().nodesByType; return new iterable_1.GatsbyIterable(nodesByType .getValues(type) .map((nodeId) => getNode(nodeId)) .filter(Boolean)); } function getNode(id) { if (!id || preSyncDeletedNodeIdsCache.has(id)) { return undefined; } const { nodes } = getDatabases(); const node = nodes.get(id); return node; } function getTypes() { return getDatabases().nodesByType.getKeys({}).asArray; } function countNodes(typeName) { if (!typeName) { const stats = getDatabases().nodes.getStats(); return Math.max(Number(stats.entryCount) - preSyncDeletedNodeIdsCache.size, 0); // FIXME: add -1 when restoring shared structures key } const { nodesByType } = getDatabases(); return nodesByType.getValuesCount(typeName); } async function runQuery(args) { if (process.env.GATSBY_EXPERIMENTAL_LMDB_INDEXES) { return await (0, run_query_1.doRunQuery)({ datastore: lmdbDatastore, databases: getDatabases(), ...args, }); } return Promise.resolve((0, run_fast_filters_1.runFastFiltersAndSort)(args)); } async function resetCache() { redux_1.store.dispatch({ type: `DELETE_CACHE`, dbPath, }); await (0, fs_extra_1.rm)(dbPath, { recursive: true, force: true }); await restartDbs(); await ready(); } async function restartDbs() { const dbs = getDatabases(); await Promise.all([ dbs.nodes.close(), dbs.nodesByType.close(), dbs.metadata.close(), dbs.indexes.close(), rootDb.close(), ]); globalThis.__GATSBY_OPEN_ROOT_LMDBS.delete(dbPath); globalThis.__GATSBY_OPEN_LMDBS.delete(dbPath); rootDb = undefined; databases = undefined; getDatabases(); } let lastOperationPromise = Promise.resolve(); function updateDataStore(action) { switch (action.type) { case `DELETE_CACHE`: { const dbs = getDatabases(); // Force sync commit dbs.nodes.transactionSync(() => { dbs.nodes.clearSync(); dbs.nodesByType.clearSync(); dbs.metadata.clearSync(); dbs.indexes.clearSync(); }); break; } case `SET_PROGRAM`: { // TODO: remove this when we have support for incremental indexes in lmdb clearIndexes(); break; } case `CREATE_NODE`: case `DELETE_NODE`: case `ADD_FIELD_TO_NODE`: case `ADD_CHILD_NODE_TO_PARENT_NODE`: { const dbs = getDatabases(); const operationPromise = Promise.all([ (0, nodes_1.updateNodes)(dbs.nodes, action), (0, nodes_by_type_1.updateNodesByType)(dbs.nodesByType, action), ]); lastOperationPromise = operationPromise; // if create is used in the same transaction as delete we should remove it from cache if (action.type === `CREATE_NODE`) { preSyncDeletedNodeIdsCache.delete(action.payload.id); } if (action.type === `DELETE_NODE` && action.payload?.id) { preSyncDeletedNodeIdsCache.add(action.payload.id); operationPromise.then(() => { // only clear if no other operations have been done in the meantime if (lastOperationPromise === operationPromise) { preSyncDeletedNodeIdsCache.clear(); } }); } } } } function clearIndexes() { const dbs = getDatabases(); dbs.nodes.transactionSync(() => { dbs.metadata.clearSync(); dbs.indexes.clearSync(); }); } /** * Resolves when all the data is synced */ async function ready() { await lastOperationPromise; } // TODO: remove this when we have support for incremental indexes in lmdb clearIndexes(); stores.set(dbPath, lmdbDatastore); return lmdbDatastore; } //# sourceMappingURL=lmdb-datastore.js.map