@netlify/content-engine
Version:
268 lines • 10.2 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.getDefaultDbPath = getDefaultDbPath;
exports.getLmdbStore = getLmdbStore;
const lmdb_1 = require("lmdb");
const nodes_1 = require("./updates/nodes");
const nodes_by_type_1 = require("./updates/nodes-by-type");
const iterable_1 = require("../common/iterable");
const run_query_1 = require("./query/run-query");
const run_fast_filters_1 = require("../in-memory/run-fast-filters");
const fs_extra_1 = require("fs-extra");
const redux_1 = require("../../redux");
function getDefaultDbPath(cwd = process.cwd()) {
const dbFileName = process.env.NODE_ENV === `test`
? `test-datastore-${
// FORCE_TEST_DATABASE_ID will be set if this gets executed in worker context
// when running jest tests. JEST_WORKER_ID will be set when this gets executed directly
// in test context (jest will use jest-worker internally).
process.env.FORCE_TEST_DATABASE_ID ?? process.env.JEST_WORKER_ID}`
: `datastore`;
return cwd + `/.cache/data/` + dbFileName;
}
const stores = new Map();
function getLmdbStore({ dbPath = getDefaultDbPath(), } = {}) {
if (stores.has(dbPath))
return stores.get(dbPath);
const lmdbDatastore = {
resetCache,
getNode,
getTypes,
countNodes,
iterateNodes,
iterateNodesByType,
updateDataStore,
ready,
runQuery,
clearIndexes,
// deprecated:
getNodes,
getNodesByType,
};
const preSyncDeletedNodeIdsCache = new Set();
let rootDb;
let databases;
function getRootDb() {
if (!rootDb) {
if (!dbPath) {
throw new Error(`LMDB path is not set!`);
}
if (!globalThis.__GATSBY_OPEN_ROOT_LMDBS) {
globalThis.__GATSBY_OPEN_ROOT_LMDBS = new Map();
}
rootDb = globalThis.__GATSBY_OPEN_ROOT_LMDBS.get(dbPath);
if (rootDb) {
return rootDb;
}
rootDb = (0, lmdb_1.open)({
name: `root`,
path: dbPath,
compression: true,
});
globalThis.__GATSBY_OPEN_ROOT_LMDBS.set(dbPath, rootDb);
}
return rootDb;
}
function getDatabases() {
if (!databases) {
// __GATSBY_OPEN_LMDBS tracks if we already opened given db in this process
// In `gatsby serve` case we might try to open it twice - once for engines
// and second to get access to `SitePage` nodes (to power trailing slashes
// redirect middleware). This ensure there is single instance within a process.
// Using more instances seems to cause weird random errors.
if (!globalThis.__GATSBY_OPEN_LMDBS) {
globalThis.__GATSBY_OPEN_LMDBS = new Map();
}
databases = globalThis.__GATSBY_OPEN_LMDBS.get(dbPath);
if (databases) {
return databases;
}
rootDb = getRootDb();
databases = {
nodes: rootDb.openDB({
name: `nodes`,
// FIXME: sharedStructuresKey breaks tests - probably need some cleanup for it on DELETE_CACHE
// sharedStructuresKey: Symbol.for(`structures`),
cache: {
// expirer: false disables LRU part and only take care of WeakRefs
// this way we don't retain nodes strongly, but will continue to
// reuse them if they are loaded already
expirer: false,
},
}),
nodesByType: rootDb.openDB({
name: `nodesByType`,
dupSort: true,
}),
metadata: rootDb.openDB({
name: `metadata`,
useVersions: true,
}),
indexes: rootDb.openDB({
name: `indexes`,
// TODO: use dupSort when this is ready: https://github.com/DoctorEvidence/lmdb-store/issues/66
// dupSort: true
}),
};
globalThis.__GATSBY_OPEN_LMDBS.set(dbPath, databases);
}
return databases;
}
/**
* @deprecated
*/
function getNodes() {
// const start = performance.now()
const result = Array.from(iterateNodes());
// const timeTotal = performance.now() - start
// console.warn(
// `getNodes() is deprecated, use iterateNodes() instead; ` +
// `array length: ${result.length}; time(ms): ${timeTotal}`
// )
return result ?? [];
}
/**
* @deprecated
*/
function getNodesByType(type) {
// const start = performance.now()
const result = Array.from(iterateNodesByType(type));
// const timeTotal = performance.now() - start
// console.warn(
// `getNodesByType() is deprecated, use iterateNodesByType() instead; ` +
// `array length: ${result.length}; time(ms): ${timeTotal}`
// )
return result ?? [];
}
function iterateNodes() {
// Additionally fetching items by id to leverage lmdb-store cache
const nodesDb = getDatabases().nodes;
return new iterable_1.GatsbyIterable(nodesDb
.getKeys({ snapshot: false })
.map((nodeId) => typeof nodeId === `string` ? getNode(nodeId) : undefined)
.filter(Boolean));
}
function iterateNodesByType(type) {
const nodesByType = getDatabases().nodesByType;
return new iterable_1.GatsbyIterable(nodesByType
.getValues(type)
.map((nodeId) => getNode(nodeId))
.filter(Boolean));
}
function getNode(id) {
if (!id || preSyncDeletedNodeIdsCache.has(id)) {
return undefined;
}
const { nodes } = getDatabases();
const node = nodes.get(id);
return node;
}
function getTypes() {
return getDatabases().nodesByType.getKeys({}).asArray;
}
function countNodes(typeName) {
if (!typeName) {
const stats = getDatabases().nodes.getStats();
return Math.max(Number(stats.entryCount) - preSyncDeletedNodeIdsCache.size, 0); // FIXME: add -1 when restoring shared structures key
}
const { nodesByType } = getDatabases();
return nodesByType.getValuesCount(typeName);
}
async function runQuery(args) {
if (process.env.GATSBY_EXPERIMENTAL_LMDB_INDEXES) {
return await (0, run_query_1.doRunQuery)({
datastore: lmdbDatastore,
databases: getDatabases(),
...args,
});
}
return Promise.resolve((0, run_fast_filters_1.runFastFiltersAndSort)(args));
}
async function resetCache() {
redux_1.store.dispatch({
type: `DELETE_CACHE`,
dbPath,
});
await (0, fs_extra_1.rm)(dbPath, { recursive: true, force: true });
await restartDbs();
await ready();
}
async function restartDbs() {
const dbs = getDatabases();
await Promise.all([
dbs.nodes.close(),
dbs.nodesByType.close(),
dbs.metadata.close(),
dbs.indexes.close(),
rootDb.close(),
]);
globalThis.__GATSBY_OPEN_ROOT_LMDBS.delete(dbPath);
globalThis.__GATSBY_OPEN_LMDBS.delete(dbPath);
rootDb = undefined;
databases = undefined;
getDatabases();
}
let lastOperationPromise = Promise.resolve();
function updateDataStore(action) {
switch (action.type) {
case `DELETE_CACHE`: {
const dbs = getDatabases();
// Force sync commit
dbs.nodes.transactionSync(() => {
dbs.nodes.clearSync();
dbs.nodesByType.clearSync();
dbs.metadata.clearSync();
dbs.indexes.clearSync();
});
break;
}
case `SET_PROGRAM`: {
// TODO: remove this when we have support for incremental indexes in lmdb
clearIndexes();
break;
}
case `CREATE_NODE`:
case `DELETE_NODE`:
case `ADD_FIELD_TO_NODE`:
case `ADD_CHILD_NODE_TO_PARENT_NODE`: {
const dbs = getDatabases();
const operationPromise = Promise.all([
(0, nodes_1.updateNodes)(dbs.nodes, action),
(0, nodes_by_type_1.updateNodesByType)(dbs.nodesByType, action),
]);
lastOperationPromise = operationPromise;
// if create is used in the same transaction as delete we should remove it from cache
if (action.type === `CREATE_NODE`) {
preSyncDeletedNodeIdsCache.delete(action.payload.id);
}
if (action.type === `DELETE_NODE` && action.payload?.id) {
preSyncDeletedNodeIdsCache.add(action.payload.id);
operationPromise.then(() => {
// only clear if no other operations have been done in the meantime
if (lastOperationPromise === operationPromise) {
preSyncDeletedNodeIdsCache.clear();
}
});
}
}
}
}
function clearIndexes() {
const dbs = getDatabases();
dbs.nodes.transactionSync(() => {
dbs.metadata.clearSync();
dbs.indexes.clearSync();
});
}
/**
* Resolves when all the data is synced
*/
async function ready() {
await lastOperationPromise;
}
// TODO: remove this when we have support for incremental indexes in lmdb
clearIndexes();
stores.set(dbPath, lmdbDatastore);
return lmdbDatastore;
}
//# sourceMappingURL=lmdb-datastore.js.map