@bitblit/epsilon
Version:
Tiny adapter to simplify building API gateway Lambda APIS
346 lines (345 loc) • 16.5 kB
JavaScript
"use strict";
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.BackgroundHandler = void 0;
const epsilon_constants_1 = require("../epsilon-constants");
const background_validator_1 = require("./background-validator");
const background_execution_event_type_1 = require("./background-execution-event-type");
const context_util_1 = require("../util/context-util");
const abstract_background_manager_1 = require("./manager/abstract-background-manager");
const common_1 = require("@bitblit/ratchet/common");
const aws_1 = require("@bitblit/ratchet/aws");
/**
* We use a FIFO queue so that 2 different Lambdas don't both work on the same
* thing at the same time.
*/
class BackgroundHandler {
constructor(cfg, mgr, modelValidator) {
this.cfg = cfg;
this.mgr = mgr;
this.modelValidator = modelValidator;
const cfgErrors = background_validator_1.BackgroundValidator.validateConfig(cfg);
if (cfgErrors.length > 0) {
common_1.ErrorRatchet.throwFormattedErr('Invalid background config : %j', cfgErrors);
}
common_1.Logger.silly('Starting Background processor, %d processors', cfg.processors.length);
this.validator = new background_validator_1.BackgroundValidator(cfg, modelValidator);
this.processors = background_validator_1.BackgroundValidator.validateAndMapProcessors(cfg.processors, modelValidator);
// If there is an immediate processing queue, wire me up to use it
if ((mgr === null || mgr === void 0 ? void 0 : mgr.immediateProcessQueue) && mgr.immediateProcessQueue()) {
common_1.Logger.info('Attaching to immediate processing queue');
mgr.immediateProcessQueue().subscribe((evt) => __awaiter(this, void 0, void 0, function* () {
common_1.Logger.debug('Processing local background entry : %j', evt);
const rval = yield this.processSingleBackgroundEntry(evt);
common_1.Logger.info('Processor returned %s', rval);
}));
}
}
extractLabel(evt, context) {
let rval = null;
if (this.isBackgroundStartSnsEvent(evt)) {
rval = 'BG:START-EVT';
}
else if (this.isBackgroundImmediateFireEvent(evt)) {
const pEvt = this.parseImmediateFireBackgroundEntry(evt);
rval = 'BG:' + pEvt.type + ':' + pEvt.guid;
}
else {
rval = 'BG:UNKNOWN';
}
return rval;
}
handlesEvent(evt) {
return aws_1.LambdaEventDetector.isValidSnsEvent(evt) && this.isBackgroundSNSEvent(evt);
}
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
isBackgroundSNSEvent(event) {
return this.isBackgroundStartSnsEvent(event) || this.isBackgroundImmediateFireEvent(event);
}
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
isBackgroundStartSnsEvent(event) {
let rval = false;
if (event) {
if (aws_1.LambdaEventDetector.isSingleSnsEvent(event)) {
const cast = event;
rval = cast.Records[0].Sns.Message === epsilon_constants_1.EpsilonConstants.BACKGROUND_SNS_START_MARKER;
}
}
return rval;
}
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
isBackgroundImmediateFireEvent(event) {
let rval = false;
if (!!event) {
if (aws_1.LambdaEventDetector.isSingleSnsEvent(event)) {
const cast = event;
const msg = cast.Records[0].Sns.Message;
if (!!common_1.StringRatchet.trimToNull(msg)) {
const parsed = JSON.parse(msg);
rval = !!parsed && parsed['type'] === epsilon_constants_1.EpsilonConstants.BACKGROUND_SNS_IMMEDIATE_RUN_FLAG;
}
}
}
return rval;
}
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
parseImmediateFireBackgroundEntry(event) {
let rval = null;
try {
if (!!event) {
if (aws_1.LambdaEventDetector.isSingleSnsEvent(event)) {
const cast = event;
const msg = cast.Records[0].Sns.Message;
if (!!common_1.StringRatchet.trimToNull(msg)) {
const parsed = JSON.parse(msg);
if (!!parsed && parsed['type'] === epsilon_constants_1.EpsilonConstants.BACKGROUND_SNS_IMMEDIATE_RUN_FLAG) {
rval = parsed['backgroundEntry'];
}
}
}
}
}
catch (err) {
common_1.Logger.error('Could not parse %j as an immediate run event : %s', event, err, err);
}
return rval;
}
/*
public isBackgroundSqsMessage(message: AWS.SQS.Types.ReceiveMessageResult): boolean {
let rval: boolean = false;
if (message && message.Messages && message.Messages.length > 0) {
const missingFlagField: any = message.Messages.find((se) => {
try {
const parsed: any = JSON.parse(se.Body);
return !parsed[EpsilonConstants.BACKGROUND_SQS_TYPE_FIELD];
} catch (err) {
Logger.warn('Failed to parse message : %j %s', se, err);
return true;
}
});
if (missingFlagField) {
Logger.silly('Found at least one message missing a type field');
} else {
rval = true;
}
}
return rval;
}
*/
// Either trigger a pull of the SQS queue, or process immediately
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
processEvent(event, context) {
return __awaiter(this, void 0, void 0, function* () {
let procd = null;
if (!this.isBackgroundStartSnsEvent(event)) {
const backgroundEntry = this.parseImmediateFireBackgroundEntry(event);
if (!!backgroundEntry) {
common_1.Logger.silly('Processing immediate fire event : %j', backgroundEntry);
const result = yield this.processSingleBackgroundEntry(backgroundEntry);
common_1.Logger.silly('Result was : %s', result);
procd = 1; // Process a single entry
}
else {
common_1.Logger.warn('Tried to process non-background start / immediate event : %j returning false', event);
}
}
else {
common_1.Logger.info('Reading task from background queue');
procd = yield this.takeAndProcessSingleBackgroundQueueEntry();
if (procd > 0) {
common_1.Logger.info('Processed %d elements from background queue, refiring', procd);
const refire = yield this.mgr.fireStartProcessingRequest();
common_1.Logger.info('Refire returned %s', refire);
}
else {
common_1.Logger.info('No items processed - stopping');
}
}
const rval = {
statusCode: 200,
body: common_1.StringRatchet.safeString(procd),
isBase64Encoded: false,
};
return rval;
});
}
takeAndProcessSingleBackgroundQueueEntry() {
return __awaiter(this, void 0, void 0, function* () {
let rval = null;
const entries = yield this.mgr.takeEntryFromBackgroundQueue();
// Do them one at a time since Background is meant to throttle. Also, it should really
// only be one per pull anyway
common_1.Logger.info('Found %d entries - processing', entries.length);
for (let i = 0; i < entries.length; i++) {
const e = entries[i];
const result = yield this.processSingleBackgroundEntry(e);
rval += result ? 1 : 0;
}
common_1.Logger.debug('Returning %d', rval);
return rval;
});
}
safeWriteToLogger(entry) {
return __awaiter(this, void 0, void 0, function* () {
if (this.cfg.transactionLogger) {
try {
yield this.cfg.transactionLogger.logTransaction(entry);
}
catch (err) {
common_1.Logger.error('Failed to write to transaction logger : %j : %s', entry, err, err);
}
}
else {
common_1.Logger.silly('Skipping - no logger defined');
}
});
}
conditionallyStartTransactionLog(e) {
return __awaiter(this, void 0, void 0, function* () {
if (!common_1.StringRatchet.trimToNull(e.guid)) {
common_1.Logger.warn('No guid found - creating');
e.guid = abstract_background_manager_1.AbstractBackgroundManager.generateBackgroundGuid();
const log = {
request: e,
running: true,
};
yield this.safeWriteToLogger(log);
}
common_1.Logger.debug('Starting transaction log');
});
}
conditionallyCompleteTransactionLog(e, result, error, runtimeMS) {
return __awaiter(this, void 0, void 0, function* () {
common_1.Logger.debug('Completing transaction log');
const log = {
request: e,
result: result,
error: error ? common_1.ErrorRatchet.safeStringifyErr(error) : null,
running: false,
runtimeMS: runtimeMS,
};
yield this.safeWriteToLogger(log);
});
}
conditionallyRunErrorProcessor(e, error) {
return __awaiter(this, void 0, void 0, function* () {
try {
if (this.cfg.errorProcessor) {
common_1.Logger.info('Running error processor');
yield this.cfg.errorProcessor.handleError(e, error);
}
}
catch (err) {
common_1.Logger.error('Background : BAD - Failed to run error processor : %s', err, err);
}
});
}
fireListenerEvent(event) {
return __awaiter(this, void 0, void 0, function* () {
const listeners = this.cfg.executionListeners || [];
for (const listener of listeners) {
try {
yield listener.onEvent(event);
}
catch (err) {
common_1.Logger.error('Failure triggering handler %s : %s', common_1.StringRatchet.trimToNull(listener === null || listener === void 0 ? void 0 : listener.label) || 'No-name', err);
}
}
});
}
// CAW 2020-08-08 : I am making processSingle public because there are times (such as when
// using AWS batch) that you want to be able to run a background command directly, eg, from
// the command line without needing an AWS-compliant event wrapping it. Thus, this.
processSingleBackgroundEntry(inE) {
var _a;
return __awaiter(this, void 0, void 0, function* () {
let e = inE; // Just to make it clear the input parameter is invariant
// Set the trace ids appropriately
context_util_1.ContextUtil.setOverrideTraceFromInternalBackgroundEntry(e);
common_1.Logger.info('Background Process Start: %j', e);
const sw = new common_1.StopWatch();
yield this.conditionallyStartTransactionLog(e);
if ((_a = this === null || this === void 0 ? void 0 : this.mgr) === null || _a === void 0 ? void 0 : _a.modifyPayloadPreProcess) {
common_1.Logger.silly('Firing payload pre-process');
e = yield this.mgr.modifyPayloadPreProcess(e);
}
let rval = false;
try {
yield this.fireListenerEvent({
type: background_execution_event_type_1.BackgroundExecutionEventType.ProcessStarting,
processorType: e.type,
data: e.data,
guid: e.guid,
});
const processorInput = this.processors.get(e.type);
if (!processorInput) {
common_1.ErrorRatchet.throwFormattedErr('Found no processor for background entry : %j (returning false)', e);
yield this.fireListenerEvent({
type: background_execution_event_type_1.BackgroundExecutionEventType.NoMatchProcessorName,
processorType: e.type,
data: e.data,
guid: e.guid,
});
}
let dataValidationErrors = [];
if (common_1.StringRatchet.trimToNull(processorInput.dataSchemaName)) {
// If it was submitted through HTTP this was checked on the API side, but if they used the
// background manager directly (or direct-posted to SQS/SNS) that would have been bypassed. We'll double
// check here
dataValidationErrors = this.modelValidator.validate(processorInput.dataSchemaName, e.data, false, false);
}
if (dataValidationErrors.length > 0) {
yield this.fireListenerEvent({
type: background_execution_event_type_1.BackgroundExecutionEventType.DataValidationError,
processorType: e.type,
data: e.data,
errors: dataValidationErrors,
guid: e.guid,
});
common_1.ErrorRatchet.throwFormattedErr('Not processing, data failed validation; entry was %j : errors : %j', e, dataValidationErrors);
}
else {
let result = yield processorInput.handleEvent(e.data, this.mgr);
result = result || 'SUCCESSFUL COMPLETION : NO RESULT RETURNED';
yield this.conditionallyCompleteTransactionLog(e, result, null, sw.elapsedMS());
yield this.fireListenerEvent({
type: background_execution_event_type_1.BackgroundExecutionEventType.ExecutionSuccessfullyComplete,
processorType: e.type,
data: result,
guid: e.guid,
});
rval = true;
}
}
catch (err) {
common_1.Logger.error('Background Process Error: %j : %s', e, err, err);
yield this.conditionallyRunErrorProcessor(e, err);
yield this.conditionallyCompleteTransactionLog(e, null, err, sw.elapsedMS());
yield this.fireListenerEvent({
type: background_execution_event_type_1.BackgroundExecutionEventType.ExecutionFailedError,
processorType: e.type,
data: e.data,
errors: [common_1.ErrorRatchet.safeStringifyErr(err)],
guid: e.guid,
});
}
common_1.Logger.info('Background Process Stop: %j : %s', e, sw.dump());
return rval;
});
}
// Returns a copy so you cannot modify the internal one here
getConfig() {
const rval = Object.assign({}, this.cfg);
return rval;
}
}
exports.BackgroundHandler = BackgroundHandler;
//# sourceMappingURL=background-handler.js.map