@netlify/content-engine
Version:
488 lines • 21.9 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.contentEngine = void 0;
const async_mutex_1 = require("async-mutex");
const graphql_1 = require("../graphql");
const index_1 = require("./../datastore/index");
const express_1 = __importDefault(require("express"));
const inspector_1 = __importDefault(require("inspector"));
const opentracing_1 = require("opentracing");
const reporter_1 = __importDefault(require("../reporter"));
const tracer_1 = require("../utils/tracer");
const detect_port_in_use_and_prompt_1 = require("../utils/detect-port-in-use-and-prompt");
const _1 = require(".");
const redux_1 = require("../redux");
const datastore_1 = require("../datastore");
const print_instructions_1 = require("../utils/print-instructions");
const __1 = require("..");
const context_1 = __importDefault(require("../schema/context"));
const framework_hooks_1 = require("../framework-hooks");
const ledger_writer_1 = require("./replication/ledger-writer");
const ledger_reader_1 = require("./replication/ledger-reader");
const ledger_dependency_manager_1 = require("./replication/ledger-dependency-manager");
const configuration_hash_1 = require("./replication/configuration-hash/configuration-hash");
const cache_lmdb_1 = __importDefault(require("../utils/cache-lmdb"));
const tracer = (0, opentracing_1.globalTracer)();
const syncDataMutex = new async_mutex_1.Mutex();
const openDebuggerPort = (debugInfo) => {
if (inspector_1.default.url() !== undefined) {
return; // fixes #26708
}
if (debugInfo.break) {
inspector_1.default.open(debugInfo.port, undefined, true);
// eslint-disable-next-line no-debugger
debugger;
}
else {
inspector_1.default.open(debugInfo.port);
}
};
// Return a user-supplied port otherwise the default Node.js debugging port
const getDebugPort = (port) => port ?? 9229;
const getDebugInfo = (program) => {
if (Object.prototype.hasOwnProperty.call(program, `inspect`)) {
return {
port: getDebugPort(program.inspect),
break: false,
};
}
else if (Object.prototype.hasOwnProperty.call(program, `inspectBrk`)) {
return {
port: getDebugPort(program.inspectBrk),
break: true,
};
}
else {
return null;
}
};
const contentEngine = ({ verbose, openTracingConfigFile, host, port, directory, ledger, engineConfig, frameworkHooks: frameworkHooksPath, pluginDirectories, } = { directory: `` }) => {
if (frameworkHooksPath) {
require(frameworkHooksPath);
}
framework_hooks_1.frameworkHooks.startup.before();
directory ||= process.cwd();
// provide global Gatsby object
global.__GATSBY = process.env.GATSBY_NODE_GLOBALS
? JSON.parse(process.env.GATSBY_NODE_GLOBALS)
: {};
const hostname = `0.0.0.0`;
const isLedgerWriter = ledger?.type === `WRITER`;
const isLedgerReader = ledger?.type === `READER`;
if ((!ledger?.api && ledger?.resourceId) ||
(!ledger?.resourceId && ledger?.api)) {
throw new Error(`A ledger.api and ledger.resourceId must both be provided if one is. resourceId: ${ledger.resourceId}, api: ${ledger.api}`);
}
const ledgerWriter = isLedgerWriter
? new ledger_writer_1.LedgerWriter({
ledgerId: ledger.resourceId,
serverUrl: ledger.api,
actionsToWatch: [
`CREATE_NODE`,
`DELETE_MISSING_NODE`,
`DELETE_NODE`,
`ADD_FIELD_TO_NODE`,
// Possibly needed for Gatsby plugin support:
// `BUILD_TYPE_METADATA`,
// `API_FINISHED`,
// `TOUCH_NODE`,
// `ENABLE_STATEFUL_SOURCE_PLUGIN`,
],
})
: undefined;
const schemaHashCache = new cache_lmdb_1.default({
name: `schema-hash-cache`,
encoding: `json`,
directory,
}).init();
const ledgerDependencyManager = new ledger_dependency_manager_1.LedgerDependencyManager({ directory });
if (ledgerWriter && engineConfig) {
ledgerDependencyManager.checkEngineConfigForPluginInstalls(engineConfig);
}
const resourceId = ledger?.resourceId;
const ledgerReader = isLedgerReader
? new ledger_reader_1.LedgerReader({
handleAction: (action) => {
if (action.type !== `SYNCHRONIZER_CONTROL_SEQUENCE`) {
ledgerDependencyManager.checkLedgerActionForPluginInstall(action);
if (
// don't emit internal ledger actions
action.type !==
`LEDGER_DEPENDENCY_MANAGER_SHOULD_INSTALL_DYNAMIC_CONNECTOR`) {
redux_1.store.dispatch(action);
}
}
},
serverUrl: ledger.api,
directory,
resourceId: ledger.resourceId,
})
: undefined;
const program = {
directory,
sitePackageJson: require(`${directory}/package.json`),
port: (typeof port === `string` ? parseInt(port, 10) : port) || 8000,
hostname,
host: host || `localhost`,
store: redux_1.store,
reporter: reporter_1.default,
openTracingConfigFile: ``,
debugInfo: null,
};
program.debugInfo = getDebugInfo(program);
console.info(`[content-engine] starting "${program.sitePackageJson.name || `unknown`}" engine`);
program.verbose = verbose || false;
reporter_1.default.setVerbose(program.verbose);
if (program.debugInfo) {
openDebuggerPort(program.debugInfo);
}
if (openTracingConfigFile) {
(0, tracer_1.initTracer)(openTracingConfigFile);
}
async function loadData({ webhookBody, parentSpan, shouldBuildSchema = true, connector, } = {
shouldBuildSchema: true,
}) {
const activity = reporter_1.default.activityTimer(`update extension data`);
activity.start();
try {
await framework_hooks_1.frameworkHooks.customizeSchema.before();
await (0, _1.customizeSchema)({
parentSpan,
// can't refresh as a reader because it will emit CLEAR_SCHEMA_CUSTOMIZATION and we'll lose our types
// we can only do refresh: true if dependency manager has installed plugins or we're not a reader
refresh: !isLedgerReader || ledgerDependencyManager.didInstallPlugins,
});
await framework_hooks_1.frameworkHooks.customizeSchema.after();
await framework_hooks_1.frameworkHooks.sourceNodes.before();
await (0, _1.sourceNodes)({
parentSpan,
webhookBody,
store: redux_1.store,
webhookSourcePluginName: connector,
});
await framework_hooks_1.frameworkHooks.sourceNodes.after();
await framework_hooks_1.frameworkHooks.buildSchema.before();
if (shouldBuildSchema) {
await (0, _1.buildSchema)({
parentSpan,
});
}
await framework_hooks_1.frameworkHooks.buildSchema.after();
(0, redux_1.saveState)();
}
finally {
activity.end();
}
}
let app;
let server;
let serverShouldBeRunning = false;
let initialized = false;
async function createExpressApp() {
const appExists = Boolean(app);
if (!appExists && serverShouldBeRunning) {
try {
program.port = await (0, detect_port_in_use_and_prompt_1.detectPortInUseAndPrompt)(program.port, hostname);
}
catch (e) {
if (e.message === `USER_REJECTED`) {
process.exit(0);
}
throw e;
}
app = (0, express_1.default)();
app.use((req, res, next) => {
try {
decodeURIComponent(req.path);
}
catch (e) {
return res.status(500).send(`URI malformatted`);
}
return next();
});
}
}
const publicInitialize = async ({ parentSpan }) => {
await (0, datastore_1.getDataStore)().ready();
await createExpressApp();
if (ledgerReader) {
const { previouslyInstalled, totalInstalled } = await ledgerDependencyManager.installPendingPlugins();
if (totalInstalled > 0 || (!initialized && previouslyInstalled > 0)) {
await (0, _1.initialize)({
parentSpan,
program,
reporter: reporter_1.default,
app,
isLedgerReader: Boolean(ledgerReader),
engineConfig: {
...engineConfig,
plugins: ledgerDependencyManager.getEnginePluginsConfig(),
},
pluginDirectories,
});
initialized = true;
await (0, _1.customizeSchema)({
parentSpan,
// can't refresh as a reader because it will emit CLEAR_SCHEMA_CUSTOMIZATION and we'll lose our types
// we can only do refresh: true if dependency manager has installed plugins or we're not a reader
refresh: !isLedgerReader || ledgerDependencyManager.didInstallPlugins,
});
}
await (0, _1.buildSchema)({
parentSpan,
});
console.info(`Installed ${totalInstalled} plugins from ledger, initialized, then built the GraphQL schema.`);
}
if (!initialized) {
console.info(`[content-engine] initializing`);
if (ledgerWriter) {
await ledgerDependencyManager.installPendingPluginsForWriter();
}
await (0, _1.initialize)({
parentSpan,
program,
reporter: reporter_1.default,
app,
engineConfig: ledgerWriter
? {
...engineConfig,
plugins: engineConfig?.plugins?.map((plugin) => ({
...plugin,
// Need to reset this as it has side effects
parentDir: undefined,
})),
}
: engineConfig,
isLedgerReader: Boolean(ledgerReader),
pluginDirectories,
});
initialized = true;
}
};
framework_hooks_1.frameworkHooks.startup.after();
async function startGraphQlServerWithArgs() {
serverShouldBeRunning = true;
await createExpressApp();
return (0, _1.startGraphQLServer)({
parentSpan: undefined,
program,
app,
store: redux_1.store,
reporter: reporter_1.default,
loadData,
});
}
return {
config: async () => {
throw new Error(`contentEngine().config() can only be called when contentEngine({ runInSubProcess: true }) is set.`);
},
onStdOut: () => {
throw new Error(`contentEngine().onStdOut() can only be called when contentEngine({ runInSubProcess: true }) is set.`);
},
onStdErr: () => {
throw new Error(`contentEngine().onStdErr() can only be called when contentEngine({ runInSubProcess: true }) is set.`);
},
onMessage: () => {
throw new Error(`contentEngine().onMessage() can only be called when contentEngine({ runInSubProcess: true }) is set.`);
},
sendMessage: () => {
throw new Error(`contentEngine().sendMessage() can only be called when contentEngine({ runInSubProcess: true }) is set.`);
},
clearListeners: () => {
throw new Error(`contentEngine().clearListeners() can only be called when contentEngine({ runInSubProcess: true }) is set.`);
},
initialize: async (args) => {
const parentSpan = tracer.startSpan(`initialize`);
await publicInitialize({ ...args, parentSpan });
const sourcingConfigurationId = await (0, configuration_hash_1.computeSourcingConfigurationId)(directory);
parentSpan.finish();
return sourcingConfigurationId;
},
startGraphQLServer: async () => {
serverShouldBeRunning = true;
await publicInitialize({ parentSpan: undefined });
await startGraphQlServerWithArgs();
},
clearIndexes: () => {
(0, datastore_1.getDataStore)().clearIndexes();
},
sync: async ({ runServer, webhookBody, buildSchema: shouldBuildSchema = true, connector, ledger, } = {
buildSchema: true,
}) => {
if (ledger && !isLedgerReader && !isLedgerWriter) {
throw new Error(`Cannot call engine.sync({ ledger: {...} }) without setting const engine = contentEngine({ ledger: { type: "READER" | "WRITER" }})`);
}
if (ledger && ledger.type === `WRITER` && isLedgerReader) {
throw new Error(`Cannot call engine.sync({ ledger: { type: "WRITER" } }) on an engine with ledger.type: "READER"`);
}
if (ledger && ledger.type === `READER` && isLedgerWriter) {
throw new Error(`Cannot call engine.sync({ ledger: { type: "READER" } }) on an engine with ledger.type: "WRITER"`);
}
const release = await syncDataMutex.acquire();
try {
console.info(`[content-engine] sync started`);
await framework_hooks_1.frameworkHooks.sync.before();
if (typeof runServer !== `undefined`) {
serverShouldBeRunning = runServer;
}
const parentSpan = tracer.startSpan(`sync-data`);
let totalActions = 0;
let returnedConfigurationId = ``;
let invalidations = [];
let schemaHash;
let schemaHashChanged = undefined;
if (ledgerReader && ledger?.type === `READER`) {
if (!ledger.configurationId) {
throw new Error(`engine.sync({ ledger: { type: "READER" } }) requires a configurationId to be provided.`);
}
const endBlockVersionId = ledger.untilBlockId;
totalActions = await ledgerReader.read({
endBlockVersionId,
cacheId: ledger.cacheId,
configurationId: ledger.configurationId,
onAction: ledger.onAction,
headers: ledger.headers,
});
await publicInitialize({ parentSpan });
}
else {
// need to initialize before we can write the ledger so that the configuration hash can be computed properly
await publicInitialize({
parentSpan,
});
if (ledgerWriter && ledger?.type === `WRITER`) {
await ledgerWriter.openStream({
blockVersionId: ledger.writeBlockId,
cacheId: ledger.cacheId,
directory,
headers: ledger.headers,
// If we're building the schema, we're doing a full sync, so we can skip invalidations
skipInvalidations: !!shouldBuildSchema,
});
}
try {
await loadData({
parentSpan,
webhookBody,
shouldBuildSchema,
connector,
});
}
catch (e) {
console.error(`[content-engine] error loading data`, e);
throw e;
}
(0, redux_1.savePartialStateToDisk)([`inferenceMetadata`]);
if (resourceId && ledgerWriter && ledger?.type === `WRITER`) {
const result = await ledgerWriter.finalizeStream();
totalActions = result.actionCount;
if (result.configurationId)
returnedConfigurationId = result.configurationId;
if (result.schemaHash) {
const previousSchemaHash = await schemaHashCache.get(resourceId);
schemaHash = result.schemaHash;
await schemaHashCache.set(resourceId, result.schemaHash);
schemaHashChanged = previousSchemaHash !== result.schemaHash;
}
if (result.invalidations) {
invalidations = result.invalidations;
}
}
}
if (serverShouldBeRunning && !server) {
server = await startGraphQlServerWithArgs();
}
if (server && !serverShouldBeRunning) {
server.close();
}
await (0, datastore_1.getDataStore)().ready();
parentSpan.finish();
if (serverShouldBeRunning && server) {
(0, print_instructions_1.printInstructions)(program);
}
await framework_hooks_1.frameworkHooks.sync.after();
return {
totalLedgerActions: totalActions,
configurationId: returnedConfigurationId,
schemaHash,
invalidations,
schemaHashChanged,
};
}
finally {
release();
}
},
stop: () => {
throw new Error(`contentEngine().stop() is only available when running in a subprocess with contentEngine({ runInSubProcess: true }) set.`);
},
restart: () => {
throw new Error(`contentEngine().restart() is only available when running in a subprocess with contentEngine({ runInSubProcess: true }) set.`);
},
test: (0, __1.throwOutsideTestEnv)({
getNodes: index_1.getNodes,
getNodesByType: index_1.getNodesByType,
getNode: index_1.getNode,
query: async (query, variables) => {
if (!initialized) {
throw new Error(`contentEngine().query() called before first contentEngine().sync() completed.\nSchema customization has not run yet so the GraphQL API cannot be queried.`);
}
const { schema } = redux_1.store.getState();
const result = await (0, graphql_1.graphql)({
schema,
source: query,
rootValue: {},
contextValue: (0, context_1.default)({
schema: redux_1.store.getState().schema,
schemaComposer: redux_1.store.getState().schemaCustomization.composer,
context: {},
customContext: redux_1.store.getState().schemaCustomization.context,
}),
variableValues: variables,
});
return result;
},
}),
query: async (query, variables, querySettings) => {
if (!initialized) {
throw new Error(`contentEngine().query() called before first contentEngine().sync() completed.\nSchema customization has not run yet so the GraphQL API cannot be queried.`);
}
const { schema } = redux_1.store.getState();
const cacheTags = new Set();
const schemaComposer = redux_1.store.getState().schemaCustomization.composer;
const context = querySettings?.context
? { ...querySettings.context, cacheTags, schemaComposer }
: { cacheTags, schemaComposer };
const result = await (0, graphql_1.graphql)({
schema,
source: query,
rootValue: {},
contextValue: (0, context_1.default)({
schema: redux_1.store.getState().schema,
schemaComposer,
context,
customContext: redux_1.store.getState().schemaCustomization.context,
}),
variableValues: variables,
});
result.extensions ||= {};
result.extensions.cacheTags = Array.from(cacheTags);
return result;
},
getProcess() {
throw new Error(`contentEngine().process is only available when running in a subprocess with contentEngine({ runInSubProcess: true }) set.`);
},
async buildSchema() {
await framework_hooks_1.frameworkHooks.buildSchema.before();
await (0, _1.buildSchema)({
parentSpan: undefined,
});
await framework_hooks_1.frameworkHooks.buildSchema.after();
},
};
};
exports.contentEngine = contentEngine;
//# sourceMappingURL=content-engine.js.map