@netlify/content-engine
Version:
718 lines (710 loc) • 30 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __exportStar = (this && this.__exportStar) || function(m, exports) {
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.contentEngine = exports.setFrameworkHook = void 0;
exports.throwOutsideTestEnv = throwOutsideTestEnv;
const inspector_1 = __importDefault(require("inspector"));
const signal_exit_1 = __importDefault(require("signal-exit"));
const reporter_1 = __importDefault(require("./reporter"));
const controllable_script_1 = require("./utils/controllable-script");
const crypto_1 = __importDefault(require("crypto"));
const fs_extra_1 = require("fs-extra");
const path_1 = __importDefault(require("path"));
const create_require_from_path_1 = require("./core-utils/create-require-from-path");
const lmdb_datastore_1 = require("./datastore/lmdb/lmdb-datastore");
__exportStar(require("./types"), exports);
var index_1 = require("./framework-hooks/index");
Object.defineProperty(exports, "setFrameworkHook", { enumerable: true, get: function () { return index_1.setFrameworkHook; } });
const contentEngine = (engineOptions) => {
const { directory, frameworkHooks } = engineOptions || {};
if (directory && !(0, fs_extra_1.existsSync)(directory)) {
throw new Error(`Content Engine directory does not exist: ${directory}`);
}
if (frameworkHooks && typeof frameworkHooks !== `string`) {
throw new Error(`The framework file passed to contentEngine({ frameworkHooks: "..." }) must be a string.`);
}
if (frameworkHooks && !(0, fs_extra_1.existsSync)(frameworkHooks)) {
throw new Error(`The framework file passed to contentEngine({ frameworkHooks: "${frameworkHooks}" }) does not exist. Make sure the path to the framework file is correct.`);
}
engineOptions = {
directory: process.cwd(),
...(engineOptions || {}),
};
if (`runInSubProcess` in engineOptions &&
engineOptions.runInSubProcess === false) {
process.chdir(engineOptions.directory);
const defaultContentEnginePath = `@netlify/content-engine/dist/services/content-engine.js`;
const contentEngineDir = (0, create_require_from_path_1.resolveFromContentEngine)(`/services/content-engine`, engineOptions?.directory) || defaultContentEnginePath;
if (!contentEngineDir ||
(contentEngineDir &&
contentEngineDir !== defaultContentEnginePath &&
!(0, fs_extra_1.existsSync)(contentEngineDir))) {
throw new Error(`Content Engine directory does not exist: ${contentEngineDir}`);
}
const { contentEngine } = require(contentEngineDir);
return contentEngine(engineOptions);
}
const state = {
exited: false,
};
const internalState = {
env: engineOptions?.env || {},
processListenersWereSet: false,
stdErrListeners: [],
stdOutListeners: [],
messageListeners: [],
};
const log = (message) => {
if (engineOptions?.printLogs !== false) {
console.log(message);
}
};
function onceReceived(message) {
return new Promise((res) => {
const subprocess = internalState.subProcess;
if (!subprocess) {
new Error(`Content Engine process is not running but should be.`);
}
const exitListener = () => {
// @ts-ignore if subProcess is accessed from outside it will be recreated automatically. setting it to null here allows it to be recreated as needed
internalState.subProcess = null;
res(false);
};
const listener = (receivedMessage) => {
if (receivedMessage.type === message) {
subprocess.offMessage(listener);
subprocess.offExit(exitListener);
res(receivedMessage.payload || true);
}
};
subprocess.onMessage(listener);
subprocess.onExit(exitListener);
});
}
const createSubProcessIfNoneExists = async ({ env, }) => {
if (!internalState.subProcess) {
state.exited = false;
state.exitCode = undefined;
internalState.env = env || internalState.env;
reporter_1.default.verbose(`[content-engine] starting subprocess`);
const contentEngineDir = (0, create_require_from_path_1.resolveFromContentEngine)(`/services/content-engine`, engineOptions?.directory) || require.resolve(`./services/content-engine`);
const reduxDir = (0, create_require_from_path_1.resolveFromContentEngine)(`/redux`, engineOptions?.directory) ||
require.resolve(`./redux`);
internalState.subProcess = new controllable_script_1.ControllableScript(`
const engineOptions = ${JSON.stringify(engineOptions)}
const { contentEngine } = require("${contentEngineDir}")
const { saveState } = require("${reduxDir}")
const engine = contentEngine(engineOptions)
if (!process.send) {
throw new Error(
'Started Content Engine as a subprocess, but no parent was found.'
)
}
process.send({
type: 'CONTENT_ENGINE_CHILD_RUNNING',
})
process.on('message', async message => {
if (message.type === 'COMMAND' && message.action?.type === 'EXIT') {
saveState()
const code = typeof message.action?.payload === 'number'
? message.action.payload
: 0
process.exit(code)
} else if (message.type === 'CONTENT_ENGINE_CHILD_SYNC_DATA') {
engine.sync(message.payload).then(() => {
process.send({
type: 'CONTENT_ENGINE_CHILD_FINISHED_SYNCING_DATA',
})
}).catch(e => {
process.send({
type: 'CONTENT_ENGINE_CHILD_FINISHED_SYNCING_DATA',
payload: {
error: {
message: e.message,
stack: e.stack
}
}
})
})
} else if (message.type === "CONTENT_ENGINE_CHILD_START_GRAPHQL_SERVER") {
engine.startGraphQLServer().then(() => {
process.send({
type: 'CONTENT_ENGINE_CHILD_FINISHED_STARTING_GRAPHQL_SERVER',
})
}).catch(e => {
process.send({
type: 'CONTENT_ENGINE_CHILD_FINISHED_STARTING_GRAPHQL_SERVER',
payload: {
error: {
message: e.message,
stack: e.stack
}
}
})
})
} else if (message.type === 'CONTENT_ENGINE_CHILD_SYNCHRONIZE') {
engine.syncLedger({
...message.payload,
onAction: message.shouldSendActionsToParentProcess ?
(action) => {
process.send({
type: 'CONTENT_ENGINE_CHILD_SYNCHRONIZE_ACTION--' + message.messageId,
payload: action
})
}
: undefined
}).then((totalActions) => {
process.send({
type: 'CONTENT_ENGINE_CHILD_FINISHED_SYNCHRONIZE',
payload: {
totalActions
}
})
}).catch(e => {
process.send({
type: 'CONTENT_ENGINE_CHILD_FINISHED_SYNCHRONIZE',
payload: {
error: {
message: e.message,
stack: e.stack
}
}
})
})
} else if (message.type === 'CONTENT_ENGINE_CHILD_BUILD_SCHEMA') {
engine.buildSchema()
.then(() => {
process.send({
type: 'CONTENT_ENGINE_CHILD_FINISHED_BUILD_SCHEMA',
})
}).catch(e => {
process.send({
type: 'CONTENT_ENGINE_CHILD_FINISHED_BUILD_SCHEMA',
payload: {
error: {
message: e.message,
stack: e.stack
}
}
})
})
} else if (message.type === 'CONTENT_ENGINE_CHILD_QUERY') {
engine.query(message.payload.query, message.payload.variables, message.payload.querySettings)
.then((results) => {
process.send({
type: 'CONTENT_ENGINE_CHILD_FINISHED_QUERY--' + message.payload.messageId,
payload: {
data: results
}
})
}).catch(e => {
process.send({
type: 'CONTENT_ENGINE_CHILD_FINISHED_QUERY--' + message.payload.messageId,
payload: {
error: {
message: e.message,
stack: e.stack
}
}
})
})
} else if (message.type === 'CONTENT_ENGINE_CHILD_INITIALIZE') {
engine.initialize(message.payload).then(() => {
process.send({
type: 'CONTENT_ENGINE_CHILD_FINISHED_INITIALIZING',
})
}).catch(e => {
process.send({
type: 'CONTENT_ENGINE_CHILD_FINISHED_INITIALIZING',
error: {
message: e.message,
stack: e.stack
}
})
})
} else if (message.type === 'CONTENT_ENGINE_CHILD_INVOKE_TEST_UTIL') {
const responseType = 'CONTENT_ENGINE_CHILD_INVOKE_TEST_UTIL_RESULT--' + message.payload.messageId
function sendError(e) {
console.error(e)
process.send({
type: 'CONTENT_ENGINE_CHILD_INVOKE_TEST_UTIL_RESULT--' + message.payload.messageId,
payload: {
error: e.message,
stack: e.stack
}
})
}
try {
const promiseOrResult = engine.test[message.payload.utilName](...message.payload.args)
if (promiseOrResult && 'then' in promiseOrResult) {
promiseOrResult.then(result => {
process.send({
type: responseType,
payload: {
result
}
})
}).catch(sendError)
} else {
process.send({
type: responseType,
payload: {
result: promiseOrResult
}
})
}
} catch (e) {
sendError(e)
}
}
})
`, null, {
printLogs: engineOptions?.printLogs,
});
internalState.subProcess.start({
env: internalState.env,
directory: engineOptions?.directory,
});
internalState.subProcess.process.stdout?.on(`data`, (data) => {
internalState.stdOutListeners.forEach((listener) => listener(data));
});
internalState.subProcess.process.stderr?.on(`data`, (data) => {
internalState.stdErrListeners.forEach((listener) => listener(data));
});
internalState.subProcess.onMessage((message) => {
internalState.messageListeners.forEach((listener) => listener(message));
});
internalState.subProcess.onExit((exitCode) => {
state.exitCode = exitCode || undefined;
const proc = internalState.subProcess?.process;
if (proc?.killed || typeof exitCode === `number`) {
state.exited = true;
}
// @ts-ignore if subProcess is accessed from outside it will be recreated automatically
internalState.subProcess = null;
if (typeof exitCode === `number` && exitCode !== 0) {
console.error(`[content-engine] Process stopped with exit code ${exitCode}`);
}
});
if (!internalState.processListenersWereSet) {
internalState.processListenersWereSet = true;
function shutdownService({ subProcess, code, }, signal) {
if (!subProcess) {
return Promise.resolve();
}
return subProcess
.stop(signal, code || 0)
.catch(() => { })
.then(() => { });
}
process.on(`message`, (msg) => {
internalState.subProcess?.send(msg);
});
process.on(`SIGINT`, async () => {
await shutdownService(internalState, `SIGINT`);
process.exit(0);
});
process.on(`SIGTERM`, async () => {
await shutdownService(internalState, `SIGTERM`);
process.exit(0);
});
(0, signal_exit_1.default)((code, signal) => {
shutdownService({
subProcess: internalState.subProcess,
code,
}, signal);
});
}
reporter_1.default.verbose(`[content-engine] waiting for subprocess to start`);
const isRunning = await onceReceived(`CONTENT_ENGINE_CHILD_RUNNING`);
return {
isRunning,
};
}
return {
isRunning: true,
};
};
const sendFrameworkContext = (context) => {
if (internalState.subProcess) {
internalState.subProcess.send({
type: `CONTENT_ENGINE_FRAMEWORK_CONTEXT`,
context,
});
}
};
const initialize = async ({ clearCache, env, context } = {
clearCache: false,
}) => {
if (env && internalState.subProcess) {
log(`restarting process since env vars were passed to initialize`);
await stop();
}
if (clearCache) {
await deleteCache();
}
const { isRunning } = await createSubProcessIfNoneExists({
env: env || engineOptions?.env,
});
if (context) {
sendFrameworkContext(context);
}
if (!isRunning) {
reporter_1.default.verbose(`[content-engine] subprocess exited.`);
// return early if not running. this means the process exited.
return;
}
internalState.subProcess.send({
type: `CONTENT_ENGINE_CHILD_INITIALIZE`,
});
const result = await onceReceived(`CONTENT_ENGINE_CHILD_FINISHED_INITIALIZING`);
if (result?.error) {
const newErr = new Error(result.error.message);
newErr.stack = result.error.stack;
throw newErr;
}
};
const deleteCache = async () => {
await stop();
const cachePath = path_1.default.join(engineOptions.directory, `.cache`);
if ((0, fs_extra_1.existsSync)(cachePath)) {
await (0, fs_extra_1.rm)(cachePath, {
recursive: true,
force: true,
});
}
await getStore().resetCache();
};
const sync = async ({ clearCache = false, runServer, webhookBody, env, context, buildSchema = true, connector, } = {
buildSchema: true,
}) => {
state.error = undefined;
if (runServer && !buildSchema) {
throw new Error(`Cannot run server without building schema. Set "runServer: true" or "buildSchema: false" to fix this.`);
}
reporter_1.default.verbose(`[content-engine] sync, ${JSON.stringify(env, null, 2)}`);
if (env &&
internalState.subProcess &&
JSON.stringify(env) !== JSON.stringify(internalState.env)) {
log(`restarting process since new env vars were passed to sync`);
await stop();
}
if (clearCache) {
await deleteCache();
}
const { isRunning } = await createSubProcessIfNoneExists({
env: env || engineOptions?.env,
});
if (context) {
sendFrameworkContext(context);
}
if (!isRunning) {
reporter_1.default.verbose(`[content-engine] subprocess exited.`);
// return early if not running. this means the process exited.
return state;
}
internalState.subProcess.send({
type: `CONTENT_ENGINE_CHILD_SYNC_DATA`,
payload: {
runServer,
webhookBody,
buildSchema,
connector,
},
});
reporter_1.default.verbose(`[content-engine] waiting for subprocess to finish sync`);
const { error } = (await onceReceived(`CONTENT_ENGINE_CHILD_FINISHED_SYNCING_DATA`)) || {};
if (error) {
state.error = {
message: error.message,
stack: error.stack,
};
reporter_1.default.verbose(`[content-engine] subprocess finished syncing with errors.`);
}
else {
reporter_1.default.verbose(`[content-engine] subprocess finished sync`);
}
return { ...state, error: state.error ? { ...state.error } : undefined };
};
const restart = async ({ env, clearCache, buildSchema = true, ...options } = {
buildSchema: true,
}) => {
await stop();
if (clearCache) {
await deleteCache();
}
log(`restarting subprocess`);
await createSubProcessIfNoneExists({
env,
});
return sync({ ...options, buildSchema });
};
const stop = async (signal, code = 0) => {
if (internalState.subProcess) {
// if a debugger is attached, the process must be killed with SIGKILL or it will hang indefinitely
const inDebugMode = inspector_1.default.url() !== undefined;
const exitSignal = inDebugMode ? `SIGKILL` : signal || null;
await internalState.subProcess.stop(exitSignal, code);
// @ts-ignore if subProcess is accessed from outside it will be recreated automatically
internalState.subProcess = null;
}
};
async function deferTestUtilToSubProcess(utilName, args) {
if (!internalState.subProcess) {
throw new Error(`contentEngine().test.${utilName}() can only be called when content engine is running.`);
}
// @ts-ignore
const messageId = crypto_1.default.randomUUID();
internalState.subProcess.send({
type: `CONTENT_ENGINE_CHILD_INVOKE_TEST_UTIL`,
payload: {
args,
messageId,
utilName,
},
});
const message = await onceReceived(`CONTENT_ENGINE_CHILD_INVOKE_TEST_UTIL_RESULT--${messageId}`);
if (message?.error) {
if (message.error.startsWith(`Error:`)) {
message.error = message.error.replace(`Error: `, ``);
}
const newErr = new Error(message.error.message);
newErr.stack = message.error.stack;
throw newErr;
}
return message?.result;
}
function getStore() {
internalState.store ||= (0, lmdb_datastore_1.getLmdbStore)({
dbPath: (0, lmdb_datastore_1.getDefaultDbPath)(engineOptions?.directory),
});
return internalState.store;
}
const engineApi = {
initialize,
sync,
restart,
stop,
config: async (newConfig) => {
const normConfig = {
directory: newConfig.directory || process.cwd(),
...(newConfig || {}),
};
if (JSON.stringify(normConfig) !== JSON.stringify(engineOptions)) {
reporter_1.default.info(`[content-engine] restarting process since config changed`);
await stop();
const newEngine = (0, exports.contentEngine)(normConfig);
// update the engineApi with the new engine's methods
// so that calling engine.sync() will use the new engine config instead of the old one
Object.keys(engineApi).forEach((key) => {
engineApi[key] = newEngine[key];
});
return newEngine;
}
else {
reporter_1.default.verbose(`[content-engine] reusing existing engine since the config didn't change`);
return engineApi;
}
},
test: throwOutsideTestEnv({
getNodes: () => deferTestUtilToSubProcess(`getNodes`, []),
getNodesByType: (...args) => deferTestUtilToSubProcess(`getNodesByType`, [...args]),
getNode: (...args) => deferTestUtilToSubProcess(`getNode`, [...args]),
query: (...args) => deferTestUtilToSubProcess(`query`, [...args]),
}),
store: {
getNode: (id) => getStore().getNode(id),
getTypes: () => getStore().getTypes(),
countNodes: (type) => getStore().countNodes(type),
getNodes: () => getStore().iterateNodes(),
getNodesByType: (type) => getStore().iterateNodesByType(type),
runQuery: (args) => {
process.env.GATSBY_EXPERIMENTAL_LMDB_INDEXES ||= `1`;
return getStore().runQuery(args);
},
},
getProcess() {
createSubProcessIfNoneExists({
env: internalState.env,
});
return internalState.subProcess.process;
},
onStdOut(callback) {
internalState.stdOutListeners.push(callback);
},
onStdErr(callback) {
internalState.stdErrListeners.push(callback);
},
onMessage(callback) {
internalState.messageListeners.push(callback);
},
sendMessage(message) {
if (internalState.subProcess) {
internalState.subProcess.send(message);
}
else {
throw new Error(`contentEngine().sendMessage() can only be called when content engine is running.`);
}
},
clearListeners() {
internalState.stdErrListeners = [];
internalState.stdOutListeners = [];
internalState.messageListeners = [];
},
async startGraphQLServer() {
const { isRunning } = await createSubProcessIfNoneExists({
env: engineOptions?.env,
});
if (!isRunning) {
reporter_1.default.verbose(`[content-engine] subprocess exited.`);
// return early if not running. this means the process exited.
return;
}
internalState.subProcess.send({
type: `CONTENT_ENGINE_CHILD_START_GRAPHQL_SERVER`,
});
const { error } = (await onceReceived(`CONTENT_ENGINE_CHILD_FINISHED_STARTING_GRAPHQL_SERVER`)) || {};
if (error) {
state.error = {
message: error.message,
stack: error.stack,
};
reporter_1.default.verbose(`[content-engine] subprocess finished starting the GraphQL server with errors.`);
}
else {
reporter_1.default.verbose(`[content-engine] subprocess finished starting the GraphQL server`);
}
},
async buildSchema() {
const { isRunning } = await createSubProcessIfNoneExists({
env: engineOptions?.env,
});
if (!isRunning) {
reporter_1.default.verbose(`[content-engine] subprocess exited.`);
// return early if not running. this means the process exited.
return;
}
internalState.subProcess.send({
type: `CONTENT_ENGINE_CHILD_BUILD_SCHEMA`,
});
const { error } = (await onceReceived(`CONTENT_ENGINE_CHILD_FINISHED_BUILD_SCHEMA`)) ||
{};
if (error) {
state.error = {
message: error.message,
stack: error.stack,
};
reporter_1.default.verbose(`[content-engine] subprocess finished building schema with errors.`);
}
else {
reporter_1.default.verbose(`[content-engine] subprocess finished building schema`);
}
},
syncLedger: async (args) => {
const { isRunning } = await createSubProcessIfNoneExists({
env: engineOptions?.env,
});
if (!isRunning) {
reporter_1.default.verbose(`[content-engine] subprocess exited.`);
// return early if not running. this means the process exited.
return 0;
}
const messageId = crypto_1.default.randomUUID();
const shouldSendActionsToParentProcess = Boolean(args.onAction);
internalState.subProcess.send({
type: `CONTENT_ENGINE_CHILD_SYNCHRONIZE`,
payload: {
...args,
directory: engineOptions?.directory,
},
shouldSendActionsToParentProcess,
messageId,
});
reporter_1.default.verbose(`[content-engine] waiting for subprocess to finish ledger sync`);
if (shouldSendActionsToParentProcess) {
internalState.subProcess.onMessage((message) => {
if (message.type ===
`CONTENT_ENGINE_CHILD_SYNCHRONIZE_ACTION--${messageId}`) {
args.onAction?.(message.payload);
}
});
}
const { error, totalActions } = (await onceReceived(`CONTENT_ENGINE_CHILD_FINISHED_SYNCHRONIZE`)) || {};
if (error) {
state.error = {
message: error.message,
stack: error.stack,
};
reporter_1.default.verbose(`[content-engine] subprocess finished syncing with errors.`);
throw new Error(JSON.stringify(state.error, null, 2));
}
else {
reporter_1.default.verbose(`[content-engine] subprocess finished sync`);
}
return totalActions > 0 ? totalActions - 2 : 0;
},
query: async (query, variables, querySettings) => {
const { isRunning } = await createSubProcessIfNoneExists({
env: engineOptions?.env,
});
if (!isRunning) {
reporter_1.default.verbose(`[content-engine] subprocess exited.`);
// return early if not running. this means the process exited.
return;
}
// we need to send/receive messages with an ID so that we don't receive the wrong GraphQL response when multiple requests are being made simultaneously
const messageId = crypto_1.default.randomUUID();
internalState.subProcess.send({
type: `CONTENT_ENGINE_CHILD_QUERY`,
payload: {
query,
variables,
messageId,
querySettings,
},
});
const { error, data } = (await onceReceived(`CONTENT_ENGINE_CHILD_FINISHED_QUERY--${messageId}`)) || {};
if (error) {
state.error = {
message: error.message,
stack: error.stack,
};
reporter_1.default.verbose(`[content-engine] subprocess finished querying with errors.`);
}
else {
reporter_1.default.verbose(`[content-engine] subprocess finished query`);
}
return data;
},
};
return engineApi;
};
exports.contentEngine = contentEngine;
function throwOutsideTestEnv(fns) {
return Object.fromEntries(Object.entries(fns).map(([name, fn]) => [
name,
(...args) => {
if (process.env.NODE_ENV !== `test`) {
throw new Error(`contentEngine().test.${name}() can only be called from within tests.`);
}
return fn(...args);
},
]));
}
//# sourceMappingURL=index.js.map