@message-queue-toolkit/core
Version:
Useful utilities, interfaces and base classes for message queue handling. Supports AMQP and SQS with a common abstraction on top currently
408 lines • 18.9 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.AbstractQueueService = void 0;
const node_util_1 = require("node:util");
const node_core_1 = require("@lokalise/node-core");
const schemas_1 = require("@message-queue-toolkit/schemas");
const messageDeduplicationTypes_1 = require("../message-deduplication/messageDeduplicationTypes");
const JsonStreamStringifySerializer_1 = require("../payload-store/JsonStreamStringifySerializer");
const offloadedPayloadMessageSchemas_1 = require("../payload-store/offloadedPayloadMessageSchemas");
const payloadStoreTypes_1 = require("../payload-store/payloadStoreTypes");
const dateUtils_1 = require("../utils/dateUtils");
const streamUtils_1 = require("../utils/streamUtils");
const toDateProcessor_1 = require("../utils/toDateProcessor");
const HandlerSpy_1 = require("./HandlerSpy");
const MessageSchemaContainer_1 = require("./MessageSchemaContainer");
class AbstractQueueService {
/**
* Used to keep track of the number of `retryLater` results received for a message to be able to
* calculate the delay for the next retry
*/
messageRetryLaterCountField = '_internalRetryLaterCount';
/**
* Used to know when the message was sent initially so we can have a max retry date and avoid
* a infinite `retryLater` loop
*/
messageTimestampField;
/**
* Used to know the message deduplication id
*/
messageDeduplicationIdField;
/**
* Used to know the store-based message deduplication options
*/
messageDeduplicationOptionsField;
errorReporter;
logger;
messageIdField;
messageTypeField;
logMessages;
creationConfig;
locatorConfig;
deletionConfig;
payloadStoreConfig;
messageDeduplicationConfig;
messageMetricsManager;
_handlerSpy;
isInitted;
get handlerSpy() {
if (!this._handlerSpy) {
throw new Error('HandlerSpy was not instantiated, please pass `handlerSpy` parameter during queue service creation.');
}
return this._handlerSpy;
}
constructor({ errorReporter, logger, messageMetricsManager }, options) {
this.errorReporter = errorReporter;
this.logger = logger;
this.messageMetricsManager = messageMetricsManager;
this.messageIdField = options.messageIdField ?? 'id';
this.messageTypeField = options.messageTypeField;
this.messageTimestampField = options.messageTimestampField ?? 'timestamp';
this.messageDeduplicationIdField = options.messageDeduplicationIdField ?? 'deduplicationId';
this.messageDeduplicationOptionsField =
options.messageDeduplicationOptionsField ?? 'deduplicationOptions';
this.creationConfig = options.creationConfig;
this.locatorConfig = options.locatorConfig;
this.deletionConfig = options.deletionConfig;
this.payloadStoreConfig = options.payloadStoreConfig
? {
serializer: JsonStreamStringifySerializer_1.jsonStreamStringifySerializer,
...options.payloadStoreConfig,
}
: undefined;
this.messageDeduplicationConfig = options.messageDeduplicationConfig;
this.logMessages = options.logMessages ?? false;
this._handlerSpy = (0, HandlerSpy_1.resolveHandlerSpy)(options);
this.isInitted = false;
}
resolveConsumerMessageSchemaContainer(options) {
const messageSchemas = options.handlers.map((entry) => entry.schema);
const messageDefinitions = options.handlers
.map((entry) => entry.definition)
.filter((entry) => entry !== undefined);
return new MessageSchemaContainer_1.MessageSchemaContainer({
messageSchemas,
messageDefinitions,
messageTypeField: options.messageTypeField,
});
}
resolvePublisherMessageSchemaContainer(options) {
const messageSchemas = options.messageSchemas;
const messageDefinitions = [];
return new MessageSchemaContainer_1.MessageSchemaContainer({
messageSchemas,
messageDefinitions,
messageTypeField: options.messageTypeField,
});
}
/**
* Format message for logging
*/
resolveMessageLog(message, _messageType) {
return message;
}
/**
* Log preformatted and potentially presanitized message payload
*/
logMessage(messageLogEntry) {
this.logger.debug(messageLogEntry);
}
handleError(err, context) {
const logObject = (0, node_core_1.resolveGlobalErrorLogObject)(err);
this.logger.error({
...logObject,
...context,
});
if (node_util_1.types.isNativeError(err)) {
this.errorReporter.report({ error: err, context });
}
}
handleMessageProcessed(params) {
const { message, processingResult, messageId } = params;
const messageProcessingEndTimestamp = Date.now();
this._handlerSpy?.addProcessedMessage({
message,
processingResult,
}, messageId);
const debugLoggingEnabled = this.logMessages && this.logger.isLevelEnabled('debug');
if (!debugLoggingEnabled && !this.messageMetricsManager)
return;
const processedMessageMetadata = this.resolveProcessedMessageMetadata(message, processingResult, params.messageProcessingStartTimestamp, messageProcessingEndTimestamp, params.queueName, messageId);
if (debugLoggingEnabled) {
this.logger.debug(processedMessageMetadata, `Finished processing message ${processedMessageMetadata.messageId}`);
}
if (this.messageMetricsManager) {
this.messageMetricsManager.registerProcessedMessage(processedMessageMetadata);
}
}
resolveProcessedMessageMetadata(message, processingResult, messageProcessingStartTimestamp, messageProcessingEndTimestamp, queueName, messageId) {
// @ts-ignore
const resolvedMessageId = message?.[this.messageIdField] ?? messageId;
const messageTimestamp = message ? this.tryToExtractTimestamp(message)?.getTime() : undefined;
const messageType = message && this.messageTypeField in message
? // @ts-ignore
message[this.messageTypeField]
: undefined;
const messageDeduplicationId = message && this.messageDeduplicationIdField in message
? // @ts-ignore
message[this.messageDeduplicationId]
: undefined;
return {
processingResult,
messageId: resolvedMessageId ?? '(unknown id)',
messageType,
queueName,
message,
messageTimestamp,
messageDeduplicationId,
messageProcessingStartTimestamp,
messageProcessingEndTimestamp,
};
}
processPrehandlersInternal(preHandlers, message) {
if (preHandlers.length === 0) {
return Promise.resolve({});
}
return new Promise((resolve, reject) => {
try {
const preHandlerOutput = {};
const next = this.resolveNextFunction(preHandlers, message, 0, preHandlerOutput, resolve, reject);
next({ result: 'success' });
}
catch (err) {
reject(err);
}
});
}
async preHandlerBarrierInternal(barrier, message, executionContext, preHandlerOutput) {
if (!barrier) {
// @ts-ignore
return {
isPassing: true,
output: undefined,
};
}
// @ts-ignore
return await barrier(message, executionContext, preHandlerOutput);
}
shouldBeRetried(message, maxRetryDuration) {
const timestamp = this.tryToExtractTimestamp(message) ?? new Date();
return !(0, dateUtils_1.isRetryDateExceeded)(timestamp, maxRetryDuration);
}
getMessageRetryDelayInSeconds(message) {
// if not defined, this is the first attempt
const retries = this.tryToExtractNumberOfRetries(message) ?? 0;
// exponential backoff -> (2 ^ (attempts)) * delay
// delay = 1 second
return Math.pow(2, retries);
}
updateInternalProperties(message) {
const messageCopy = { ...message }; // clone the message to avoid mutation
/**
* If the message doesn't have a timestamp field -> add it
* will be used to prevent infinite retries on the same message
*/
if (!this.tryToExtractTimestamp(message)) {
// @ts-ignore
messageCopy[this.messageTimestampField] = new Date().toISOString();
this.logger.warn(`${this.messageTimestampField} not defined, adding it automatically`);
}
/**
* add/increment the number of retries performed to exponential message delay
*/
const numberOfRetries = this.tryToExtractNumberOfRetries(message);
// @ts-ignore
messageCopy[this.messageRetryLaterCountField] =
numberOfRetries !== undefined ? numberOfRetries + 1 : 0;
return messageCopy;
}
tryToExtractTimestamp(message) {
// @ts-ignore
if (this.messageTimestampField in message) {
// @ts-ignore
const res = (0, toDateProcessor_1.toDatePreprocessor)(message[this.messageTimestampField]);
if (!(res instanceof Date)) {
throw new Error(`${this.messageTimestampField} invalid type`);
}
return res;
}
return undefined;
}
tryToExtractNumberOfRetries(message) {
if (this.messageRetryLaterCountField in message &&
typeof message[this.messageRetryLaterCountField] === 'number') {
// @ts-ignore
return message[this.messageRetryLaterCountField];
}
return undefined;
}
// eslint-disable-next-line max-params
resolveNextPreHandlerFunctionInternal(preHandlers, executionContext, message, index, preHandlerOutput, resolve, reject) {
return (preHandlerResult) => {
if (preHandlerResult.error) {
reject(preHandlerResult.error);
}
if (preHandlers.length < index + 1) {
resolve(preHandlerOutput);
}
else {
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
preHandlers[index](message, executionContext,
// @ts-ignore
preHandlerOutput, this.resolveNextPreHandlerFunctionInternal(preHandlers, executionContext, message, index + 1, preHandlerOutput, resolve, reject));
}
};
}
/**
* Offload message payload to an external store if it exceeds the threshold.
* Returns a special type that contains a pointer to the offloaded payload or the original payload if it was not offloaded.
* Requires message size as only the implementation knows how to calculate it.
*/
async offloadMessagePayloadIfNeeded(message, messageSizeFn) {
if (!this.payloadStoreConfig ||
messageSizeFn() <= this.payloadStoreConfig.messageSizeThreshold) {
return message;
}
let offloadedPayloadPointer;
const serializedPayload = await this.payloadStoreConfig.serializer.serialize(message);
try {
offloadedPayloadPointer = await this.payloadStoreConfig.store.storePayload(serializedPayload);
}
finally {
if ((0, payloadStoreTypes_1.isDestroyable)(serializedPayload)) {
await serializedPayload.destroy();
}
}
return {
offloadedPayloadPointer,
offloadedPayloadSize: serializedPayload.size,
// @ts-ignore
[this.messageIdField]: message[this.messageIdField],
// @ts-ignore
[this.messageTypeField]: message[this.messageTypeField],
// @ts-ignore
[this.messageTimestampField]: message[this.messageTimestampField],
// @ts-ignore
[this.messageDeduplicationIdField]: message[this.messageDeduplicationIdField],
// @ts-ignore
[this.messageDeduplicationOptionsField]: message[this.messageDeduplicationOptionsField],
};
}
/**
* Retrieve previously offloaded message payload using provided pointer payload.
* Returns the original payload or an error if the payload was not found or could not be parsed.
*/
async retrieveOffloadedMessagePayload(maybeOffloadedPayloadPointerPayload) {
if (!this.payloadStoreConfig) {
return {
error: new Error('Payload store is not configured, cannot retrieve offloaded message payload'),
};
}
const pointerPayloadParseResult = offloadedPayloadMessageSchemas_1.OFFLOADED_PAYLOAD_POINTER_PAYLOAD_SCHEMA.safeParse(maybeOffloadedPayloadPointerPayload);
if (!pointerPayloadParseResult.success) {
return {
error: new Error('Given payload is not a valid offloaded payload pointer payload', {
cause: pointerPayloadParseResult.error,
}),
};
}
const serializedOffloadedPayloadReadable = await this.payloadStoreConfig.store.retrievePayload(pointerPayloadParseResult.data.offloadedPayloadPointer);
if (serializedOffloadedPayloadReadable === null) {
return {
error: new Error(`Payload with key ${pointerPayloadParseResult.data.offloadedPayloadPointer} was not found in the store`),
};
}
const serializedOffloadedPayloadString = await (0, streamUtils_1.streamWithKnownSizeToString)(serializedOffloadedPayloadReadable, pointerPayloadParseResult.data.offloadedPayloadSize);
try {
return { result: JSON.parse(serializedOffloadedPayloadString) };
}
catch (e) {
return { error: new Error('Failed to parse serialized offloaded payload', { cause: e }) };
}
}
/**
* Checks if the message is duplicated against the deduplication store.
* Returns true if the message is duplicated.
* Returns false if message is not duplicated or deduplication config is missing.
*/
async isMessageDuplicated(message, requester) {
if (!this.isDeduplicationEnabledForMessage(message)) {
return false;
}
const deduplicationId = this.getMessageDeduplicationId(message);
const deduplicationConfig = this.messageDeduplicationConfig;
try {
return await deduplicationConfig.deduplicationStore.keyExists(`${requester.toString()}:${deduplicationId}`);
}
catch (err) {
this.handleError(err);
// In case of errors, we treat the message as not duplicated to enable further processing
return false;
}
}
/**
* Checks if the message is duplicated.
* If it's not, stores the deduplication key in the deduplication store and returns false.
* If it is, returns true.
* If deduplication config is not provided, always returns false to allow further processing of the message.
*/
async deduplicateMessage(message, requester) {
if (!this.isDeduplicationEnabledForMessage(message)) {
return { isDuplicated: false };
}
const deduplicationId = this.getMessageDeduplicationId(message);
const { deduplicationWindowSeconds } = this.getParsedMessageDeduplicationOptions(message);
const deduplicationConfig = this.messageDeduplicationConfig;
try {
const wasDeduplicationKeyStored = await deduplicationConfig.deduplicationStore.setIfNotExists(`${requester.toString()}:${deduplicationId}`, new Date().toISOString(), deduplicationWindowSeconds);
return { isDuplicated: !wasDeduplicationKeyStored };
}
catch (err) {
this.handleError(err);
// In case of errors, we treat the message as not duplicated to enable further processing
return { isDuplicated: false };
}
}
/**
* Acquires exclusive lock for the message to prevent concurrent processing.
* If lock was acquired successfully, returns a lock object that should be released after processing.
* If lock couldn't be acquired due to timeout (meaning another process acquired it earlier), returns AcquireLockTimeoutError
* If lock couldn't be acquired for any other reasons or if deduplication config is not provided, always returns a lock object that does nothing, so message processing can continue.
*/
async acquireLockForMessage(message) {
if (!this.isDeduplicationEnabledForMessage(message)) {
return { result: messageDeduplicationTypes_1.noopReleasableLock };
}
const deduplicationId = this.getMessageDeduplicationId(message);
const deduplicationOptions = this.getParsedMessageDeduplicationOptions(message);
const deduplicationConfig = this.messageDeduplicationConfig;
const acquireLockResult = await deduplicationConfig.deduplicationStore.acquireLock(`${messageDeduplicationTypes_1.DeduplicationRequester.Consumer.toString()}:${deduplicationId}`, deduplicationOptions);
if (acquireLockResult.error && !(acquireLockResult.error instanceof messageDeduplicationTypes_1.AcquireLockTimeoutError)) {
this.handleError(acquireLockResult.error);
return { result: messageDeduplicationTypes_1.noopReleasableLock };
}
return acquireLockResult;
}
isDeduplicationEnabledForMessage(message) {
return !!this.messageDeduplicationConfig && !!this.getMessageDeduplicationId(message);
}
getMessageDeduplicationId(message) {
// @ts-ignore
return message[this.messageDeduplicationIdField];
}
getParsedMessageDeduplicationOptions(message) {
const parsedOptions = schemas_1.MESSAGE_DEDUPLICATION_OPTIONS_SCHEMA.safeParse(
// @ts-expect-error
message[this.messageDeduplicationOptionsField] ?? {});
if (parsedOptions.error) {
this.logger.warn({ error: parsedOptions.error.message }, `${this.messageDeduplicationOptionsField} contains one or more invalid values, falling back to default options`);
return messageDeduplicationTypes_1.DEFAULT_MESSAGE_DEDUPLICATION_OPTIONS;
}
return {
...messageDeduplicationTypes_1.DEFAULT_MESSAGE_DEDUPLICATION_OPTIONS,
...Object.fromEntries(Object.entries(parsedOptions.data).filter(([_, value]) => value !== undefined)),
};
}
}
exports.AbstractQueueService = AbstractQueueService;
//# sourceMappingURL=AbstractQueueService.js.map