@netlify/content-engine
Version:
341 lines • 15.1 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.LedgerWriter = void 0;
const zlib = __importStar(require("minizlib"));
const Stringer_1 = __importDefault(require("stream-json/jsonl/Stringer"));
const got_1 = __importDefault(require("got"));
const fastq_1 = __importDefault(require("fastq"));
const util_1 = __importDefault(require("util"));
const stream_1 = __importDefault(require("stream"));
const redux_1 = require("../../redux");
const configuration_hash_1 = require("./configuration-hash/configuration-hash");
const crypto_1 = require("crypto");
const path_1 = __importDefault(require("path"));
const pipeline = util_1.default.promisify(stream_1.default.pipeline);
// Used to write a stream of ledger actions to a ledger server.
//
// Example:
// const ledgerWriter = new LedgerWriter({
// blockVersionId, // the ledger entry ID. Used to identify the current ledger entry that's being written
// ledgerId, // there may be many ledgers stored. this identifies the ledger for a specific data layer
// configurationId, // the ID of the configuration for this data layer. if the configuration changes we need to write to a different ledger, this ID is a hash of the configuration values and package versions
// cacheId, // when a user manually clears the cache we need to make a new ledger. the ledger is built up from actions in a CMS, and if that CMS completely changes something we aren't aware of, the ledger is no longer accurate for the ledgerId and configurationId so it needs to be recreated. This ID allows recreating a ledger that's already been created but _right now_
// serverUrl: "http://localhost:8039/create-ledger-entries/" // endpoint to stream ledger entries to see ./ledger-server.ts for an example server
// })
//
class LedgerWriter {
actionCount = 0;
serverUrl;
ledgerId;
blockVersionId;
configurationId;
actionsToWatch;
actionsBeingWatched = new Set();
cacheId;
headers;
streamStarted = false;
streamEnded = null;
writeQueue = (0, fastq_1.default)(this, this.handleQueuedAction, 1);
stream;
streamPipeline;
createTypeActions = [];
invalidations = new Map();
skipInvalidations = false;
constructor(input) {
this.writeQueue.pause();
this.serverUrl = input.serverUrl;
this.ledgerId = input.ledgerId;
this.actionsToWatch = input.actionsToWatch;
}
setSkipInvalidations(skipInvalidations) {
this.skipInvalidations = skipInvalidations;
}
addInvalidation({ key, id, typename }) {
if (this.skipInvalidations) {
return;
}
this.invalidations.set(key, { id, typename });
}
addCreateTypeAction(action) {
this.createTypeActions.push(action);
}
clearCreateTypeActions() {
this.createTypeActions = [];
}
clearInvalidations() {
this.invalidations.clear();
}
generateSchemaHash() {
const md5 = (0, crypto_1.createHash)(`md5`);
md5.update(JSON.stringify(this.createTypeActions));
return md5.digest().toString(`hex`);
}
watchActions() {
// tell the reader to install plugins since there's a dynamic connector
if (!this.actionsBeingWatched.has(`ADD_THIRD_PARTY_SCHEMA`)) {
this.actionsBeingWatched.add(`ADD_THIRD_PARTY_SCHEMA`);
redux_1.emitter.on(`ADD_THIRD_PARTY_SCHEMA`, (action) => {
if (!action?.plugin)
return;
const ledgerAction = {
type: `LEDGER_DEPENDENCY_MANAGER_SHOULD_INSTALL_DYNAMIC_CONNECTOR`,
payload: {
plugin: action.plugin,
},
};
this.writeAction(ledgerAction);
this.addCreateTypeAction(ledgerAction);
});
}
if (!this.actionsBeingWatched.has(`CREATE_TYPES`)) {
this.actionsBeingWatched.add(`CREATE_TYPES`);
redux_1.emitter.on(`CREATE_TYPES`, (action) => {
const ledgerAction = { type: action.type, payload: action.payload };
this.writeAction(ledgerAction);
this.addCreateTypeAction(ledgerAction);
});
}
if (!this.actionsBeingWatched.has(`LEDGER_DEPENDENCY_MANAGER_SHOULD_INSTALL_DYNAMIC_CONNECTOR`)) {
this.actionsBeingWatched.add(`LEDGER_DEPENDENCY_MANAGER_SHOULD_INSTALL_DYNAMIC_CONNECTOR`);
redux_1.emitter.on(`LEDGER_DEPENDENCY_MANAGER_SHOULD_INSTALL_DYNAMIC_CONNECTOR`, (action) => {
if (!action?.plugin)
return;
const ledgerAction = {
type: `LEDGER_DEPENDENCY_MANAGER_SHOULD_INSTALL_DYNAMIC_CONNECTOR`,
payload: {
plugin: action.plugin,
},
};
this.writeAction(ledgerAction);
this.addCreateTypeAction(ledgerAction);
});
}
if (!this.actionsBeingWatched.has(`CREATE_FIELD_EXTENSION`)) {
this.actionsBeingWatched.add(`CREATE_FIELD_EXTENSION`);
// This only gets called once, so on consecutive syncs it won't emit, so we should not add it to create the SchemaHash
redux_1.emitter.on(`CREATE_FIELD_EXTENSION`, (action) => {
this.writeAction({ type: action.type, payload: action.payload });
});
}
for (const actionName of this.actionsToWatch) {
if (this.actionsBeingWatched.has(actionName))
continue;
this.actionsBeingWatched.add(actionName);
redux_1.emitter.on(actionName, (action) => {
this.writeAction({
type: action.type,
payload: action.payload,
// I don't think we actually need plugin for these actions but we have it in sourcerer, so I'm going to leave it the same for now
plugin: action.plugin,
});
const typename = action.payload?.internal?.type || action.payload?.type;
// typename invalidation
this.addInvalidation({
key: typename,
id: null,
typename,
});
// node invalidation
this.addInvalidation({
key: action.payload?.id,
id: action.payload?.id,
typename,
});
});
}
}
async openStream(input) {
if (input.skipInvalidations) {
this.setSkipInvalidations(input.skipInvalidations);
}
if (this.streamStarted && !this.streamEnded) {
throw new Error(`Attempted to open a new ledger block stream before ending a previous stream.`);
}
if (!(`blockVersionId` in input))
throw new Error(`blockVersionId is required`);
if (!(`cacheId` in input))
throw new Error(`cacheId is required`);
this.blockVersionId = input.blockVersionId;
this.cacheId = input.cacheId;
if (typeof input.headers === `object`)
this.headers = input.headers;
this.configurationId = await (0, configuration_hash_1.computeSourcingConfigurationId)(input.directory, input.platformId);
this.streamPipeline = this.makeStreamPipeline();
this.actionCount = 0;
this.watchActions();
this.writeQueue.resume();
this.writeStartSequenceAction();
}
async finalizeStream() {
await this.waitForQueueIdle();
await this.writeEndSequenceAction();
await this.waitForQueueIdle();
this.headers = undefined;
this.writeQueue.pause();
if (this.writeQueue.length() > 0) {
console.warn(`Finalized ledger write stream, but found ${this.writeQueue.length()} actions to be written`);
}
this.stream.end();
await this.streamPipeline;
this.streamPipeline = undefined;
this.stream = undefined;
this.streamEnded = true;
this.streamStarted = false;
const invalidations = this.skipInvalidations
? null
: [...this.invalidations.values()];
this.clearInvalidations();
const schemaHash = this.generateSchemaHash();
this.clearCreateTypeActions();
this.setSkipInvalidations(false);
this.cacheId = undefined;
this.blockVersionId = undefined;
const configurationId = this.configurationId;
this.configurationId = undefined;
return {
actionCount: this.actionCount,
configurationId,
invalidations,
schemaHash,
};
}
sharedActionError(action) {
return `Cannot write action (with type ${action.type}) to ledger.`;
}
writeAction(action) {
if (!action.type)
return;
if (!this.streamPipeline || !this.streamStarted) {
console.warn(`new LedgerWriter().openStream() wasn't called yet but saw emitted action ${action.type} with id ${action.payload?.id} from plugin ${action?.plugin?.name}`);
return;
}
if (this.streamEnded === true) {
throw new Error(`${this.sharedActionError(action)} This LedgerWriter has already ended. Make a new LedgerWriter instance to write additional actions.`);
}
return this.writeQueue.push(action);
}
makeStreamPipeline() {
if (this.streamPipeline) {
throw new Error(`LedgerWriter attempted to make a new http stream before the old one was cleaned up. Call new LedgerWriter().finalizeStream() before attempting to open a new stream.`);
}
this.stream = new Stringer_1.default();
return pipeline(this.stream, new zlib.Gzip({}), this.makeHttpStream(),
// catch errors
new stream_1.default.PassThrough());
}
makeHttpStream() {
let url = this.serverUrl;
if (this.configurationId && this.cacheId && this.blockVersionId) {
url = path_1.default.join(this.serverUrl, this.ledgerId, this.configurationId, this.cacheId, this.blockVersionId);
}
return got_1.default.stream.post(url, {
headers: {
...(this.headers || {}),
Connection: `keep-alive`,
[`x-ledger-id`]: this.ledgerId,
[`x-configuration-id`]: this.configurationId,
[`x-cache-id`]: this.cacheId,
[`x-block-id`]: this.blockVersionId,
},
});
}
waitForQueueIdle() {
const queueIsIdle = this.writeQueue.idle();
if (!queueIsIdle) {
this.writeQueue.resume();
return new Promise((resolve) => {
this.writeQueue.drain = () => {
resolve();
};
});
}
return Promise.resolve();
}
async writeStartSequenceAction() {
if (this.streamStarted) {
throw new Error(`Cannot write start sequence. This ledger has already started.`);
}
this.streamStarted = true;
this.streamEnded = false;
this.stream.write({
type: `SYNCHRONIZER_CONTROL_SEQUENCE`,
payload: { event: `START_SOURCING`, blockVersionId: this.blockVersionId },
});
}
async writeEndSequenceAction() {
if (!this.streamStarted) {
throw new Error(`Attempted to close a ledger block stream before starting a stream.`);
}
if (this.streamEnded || !this.stream) {
throw new Error(`Cannot write end sequence. This ledger has already ended.`);
}
this.stream.write({
type: `SYNCHRONIZER_CONTROL_SEQUENCE`,
payload: { event: `END_SOURCING`, blockVersionId: this.blockVersionId },
});
}
handleQueuedAction(queuedAction, cb) {
if (!this.stream) {
throw new Error(`The ledger block stream is missing. This should never happen so something is wrong. Make sure you've called createLedgerBlockStreamer() before adding Gatsby actions to the ledger block write stream queue.`);
}
if (!this.streamStarted) {
throw new Error(`${this.sharedActionError(queuedAction)} This LedgerWriter has not started yet.`);
}
if (!this.stream) {
throw new Error(`${this.sharedActionError(queuedAction)} This LedgerWriter has no open stream. Call new LedgerWriter().openStream() to open a stream.`);
}
this.logLedgerAction(queuedAction);
const highWaterMarkThresholdExceeded =
// see mention of high water mark in buffering section
// here https://nodejs.org/api/stream.html
this.stream.write(queuedAction);
if (!highWaterMarkThresholdExceeded) {
this.stream.once(`drain`, cb);
}
else {
// if we exceeded high water, wait til the next tick so we go below again
process.nextTick(cb);
}
}
logLedgerAction({ type, payload }) {
this.actionCount++;
if (process.env.DEBUG_LEDGER_WRITES &&
process.env.DEBUG_LEDGER_WRITES !== `false`) {
const typesToDebug = process.env.DEBUG_LEDGER_WRITES !== `true`
? process.env.DEBUG_LEDGER_WRITES.split(`,`)
: `all`;
if (typesToDebug === `all` || typesToDebug.includes(type)) {
const stringifiedAction = JSON.stringify({ type, payload }, null, 2);
console.info(`Ledger action:`, stringifiedAction.substring(0, 500), stringifiedAction.length > 500 ? `...` : ``, `\n`);
}
}
}
}
exports.LedgerWriter = LedgerWriter;
//# sourceMappingURL=ledger-writer.js.map