@netlify/content-engine
Version:
88 lines • 3.66 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.syncLedger = syncLedger;
const got_1 = __importDefault(require("got"));
const fastq_1 = __importDefault(require("fastq"));
const stream_1 = __importDefault(require("stream"));
const util_1 = __importDefault(require("util"));
const Parser_1 = require("stream-json/jsonl/Parser");
const redux_1 = require("../redux");
const cache_lmdb_1 = __importDefault(require("./cache-lmdb"));
const pipeline = util_1.default.promisify(stream_1.default.pipeline);
async function syncLedger({ dataLayerId, sourcingConfigurationId, cacheSourceVersionId, blockVersionId, directory, onAction, }) {
let synchronizerCache;
synchronizerCache =
synchronizerCache ||
new cache_lmdb_1.default({
name: `ledger-cache`,
encoding: `json`,
directory,
}).init();
const ledgerApiUrl = `${process.env.CONTENT_ENGINE_SYNC_URL}/${dataLayerId}/${sourcingConfigurationId}/${cacheSourceVersionId}`;
const lastSourcingUlid = await synchronizerCache.get(`lastSourcingUlid`);
console.log(`[content-engine] synchronizing data from ${ledgerApiUrl} from version ${lastSourcingUlid}`);
const httpStream = got_1.default.stream(ledgerApiUrl, {
headers: {
"x-consumer-offset": lastSourcingUlid || "0",
"x-end-offset": blockVersionId,
"x-resource-auth-jwt": process.env.RESOURCE_AUTH_JWT,
"x-site-auth-jwt": process.env.RESOURCE_AUTH_JWT,
"x-resource-type": process.env.RESOURCE_TYPE,
},
});
httpStream.on(`response`, (res) => {
if (process.env.DEBUG_MERLIN_SYNCHRONIZER) {
console.info(`Synchronizer ledger response status code`, res.statusCode);
}
const newLastSourcingUlid = res.headers[`x-consumer-offset`];
if (newLastSourcingUlid) {
console.log(`[content-engine] synchronized data up until version ${newLastSourcingUlid}`);
synchronizerCache.set(`lastSourcingUlid`, newLastSourcingUlid);
}
});
let dispatchCount = 0;
let lastEmittedAction;
const handleActionQueue = (0, fastq_1.default)((entry, cb) => {
lastEmittedAction = entry.value;
if (onAction) {
onAction(entry.value);
}
redux_1.store.dispatch(entry.value);
// Do not block task queue of the event loop for too long:
if (dispatchCount++ % 100 === 0) {
setImmediate(() => {
cb(null);
});
}
else {
cb(null);
}
}, 1);
const decode = (0, Parser_1.parser)();
decode.on(`data`, (data) => handleActionQueue.push(data));
let decodeError = null;
decode.on(`error`, (e) => {
handleActionQueue.kill();
decodeError = e;
});
await pipeline(httpStream, decode);
if (decodeError) {
throw decodeError;
}
if (!handleActionQueue.idle()) {
await new Promise((res) => {
handleActionQueue.drain = () => res(null);
});
}
handleActionQueue.kill();
if (lastEmittedAction &&
lastEmittedAction?.payload?.event !== `END_SOURCING`) {
throw Error(`ledger stream closed but didn't receive an END_SOURCING action`);
}
console.info(`[content-engine] Total actions synced from ledger: ${dispatchCount > 0 ? dispatchCount - 2 : dispatchCount}`);
return dispatchCount;
}
//# sourceMappingURL=synchronize.js.map