@message-queue-toolkit/core
Version:
Useful utilities, interfaces and base classes for message queue handling. Supports AMQP and SQS with a common abstraction on top currently
404 lines • 18.6 kB
JavaScript
import { types } from 'node:util';
import { stringValueSerializer, } from '@lokalise/node-core';
import { resolveGlobalErrorLogObject } from '@lokalise/node-core';
import { MESSAGE_DEDUPLICATION_OPTIONS_SCHEMA, } from '@message-queue-toolkit/schemas';
import { isAcquireLockTimeoutError, } from '../message-deduplication/AcquireLockTimeoutError.js';
import { DEFAULT_MESSAGE_DEDUPLICATION_OPTIONS, DeduplicationRequesterEnum, noopReleasableLock, } from "../message-deduplication/messageDeduplicationTypes.js";
import { jsonStreamStringifySerializer } from "../payload-store/JsonStreamStringifySerializer.js";
import { OFFLOADED_PAYLOAD_POINTER_PAYLOAD_SCHEMA, } from "../payload-store/offloadedPayloadMessageSchemas.js";
import { isDestroyable } from "../payload-store/payloadStoreTypes.js";
import { isRetryDateExceeded } from "../utils/dateUtils.js";
import { streamWithKnownSizeToString } from "../utils/streamUtils.js";
import { toDatePreprocessor } from "../utils/toDateProcessor.js";
import { resolveHandlerSpy } from "./HandlerSpy.js";
import { MessageSchemaContainer } from "./MessageSchemaContainer.js";
export class AbstractQueueService {
/**
* Used to keep track of the number of `retryLater` results received for a message to be able to
* calculate the delay for the next retry
*/
messageRetryLaterCountField = '_internalRetryLaterCount';
/**
* Used to know when the message was sent initially so we can have a max retry date and avoid
* a infinite `retryLater` loop
*/
messageTimestampField;
/**
* Used to know the message deduplication id
*/
messageDeduplicationIdField;
/**
* Used to know the store-based message deduplication options
*/
messageDeduplicationOptionsField;
errorReporter;
logger;
messageIdField;
messageTypeField;
logMessages;
creationConfig;
locatorConfig;
deletionConfig;
payloadStoreConfig;
messageDeduplicationConfig;
messageMetricsManager;
_handlerSpy;
isInitted;
get handlerSpy() {
if (!this._handlerSpy) {
throw new Error('HandlerSpy was not instantiated, please pass `handlerSpy` parameter during queue service creation.');
}
return this._handlerSpy;
}
constructor({ errorReporter, logger, messageMetricsManager }, options) {
this.errorReporter = errorReporter;
this.logger = logger;
this.messageMetricsManager = messageMetricsManager;
this.messageIdField = options.messageIdField ?? 'id';
this.messageTypeField = options.messageTypeField;
this.messageTimestampField = options.messageTimestampField ?? 'timestamp';
this.messageDeduplicationIdField = options.messageDeduplicationIdField ?? 'deduplicationId';
this.messageDeduplicationOptionsField =
options.messageDeduplicationOptionsField ?? 'deduplicationOptions';
this.creationConfig = options.creationConfig;
this.locatorConfig = options.locatorConfig;
this.deletionConfig = options.deletionConfig;
this.payloadStoreConfig = options.payloadStoreConfig
? {
serializer: jsonStreamStringifySerializer,
...options.payloadStoreConfig,
}
: undefined;
this.messageDeduplicationConfig = options.messageDeduplicationConfig;
this.logMessages = options.logMessages ?? false;
this._handlerSpy = resolveHandlerSpy(options);
this.isInitted = false;
}
resolveConsumerMessageSchemaContainer(options) {
const messageSchemas = options.handlers.map((entry) => entry.schema);
const messageDefinitions = options.handlers
.map((entry) => entry.definition)
.filter((entry) => entry !== undefined);
return new MessageSchemaContainer({
messageTypeField: options.messageTypeField,
messageSchemas,
messageDefinitions,
});
}
resolvePublisherMessageSchemaContainer(options) {
const messageSchemas = options.messageSchemas;
return new MessageSchemaContainer({
messageTypeField: options.messageTypeField,
messageSchemas,
messageDefinitions: [],
});
}
/**
* Format message for logging
*/
resolveMessageLog(message, _messageType) {
return message;
}
/**
* Log preformatted and potentially presanitized message payload
*/
logMessage(messageLogEntry) {
this.logger.debug(messageLogEntry);
}
handleError(err, context) {
const logObject = resolveGlobalErrorLogObject(err);
this.logger.error({
...logObject,
...context,
});
if (types.isNativeError(err)) {
this.errorReporter.report({ error: err, context });
}
}
handleMessageProcessed(params) {
const { message, processingResult, messageId } = params;
const messageProcessingEndTimestamp = Date.now();
this._handlerSpy?.addProcessedMessage({
message,
processingResult,
}, messageId);
const debugLoggingEnabled = this.logMessages && this.logger.isLevelEnabled('debug');
if (!debugLoggingEnabled && !this.messageMetricsManager)
return;
const processedMessageMetadata = this.resolveProcessedMessageMetadata(message, processingResult, params.messageProcessingStartTimestamp, messageProcessingEndTimestamp, params.queueName, messageId);
if (debugLoggingEnabled) {
this.logger.debug({ processedMessageMetadata: stringValueSerializer(processedMessageMetadata) }, `Finished processing message ${processedMessageMetadata.messageId}`);
}
if (this.messageMetricsManager) {
this.messageMetricsManager.registerProcessedMessage(processedMessageMetadata);
}
}
resolveProcessedMessageMetadata(message, processingResult, messageProcessingStartTimestamp, messageProcessingEndTimestamp, queueName, messageId) {
// @ts-ignore
const resolvedMessageId = message?.[this.messageIdField] ?? messageId;
const messageTimestamp = message ? this.tryToExtractTimestamp(message)?.getTime() : undefined;
const messageType = message && this.messageTypeField in message
? // @ts-ignore
message[this.messageTypeField]
: undefined;
const messageDeduplicationId = message && this.messageDeduplicationIdField in message
? // @ts-ignore
message[this.messageDeduplicationId]
: undefined;
return {
processingResult,
messageId: resolvedMessageId ?? '(unknown id)',
messageType,
queueName,
message,
messageTimestamp,
messageDeduplicationId,
messageProcessingStartTimestamp,
messageProcessingEndTimestamp,
};
}
processPrehandlersInternal(preHandlers, message) {
if (preHandlers.length === 0) {
return Promise.resolve({});
}
return new Promise((resolve, reject) => {
try {
const preHandlerOutput = {};
const next = this.resolveNextFunction(preHandlers, message, 0, preHandlerOutput, resolve, reject);
next({ result: 'success' });
}
catch (err) {
reject(err);
}
});
}
async preHandlerBarrierInternal(barrier, message, executionContext, preHandlerOutput) {
if (!barrier) {
// @ts-ignore
return {
isPassing: true,
output: undefined,
};
}
// @ts-ignore
return await barrier(message, executionContext, preHandlerOutput);
}
shouldBeRetried(message, maxRetryDuration) {
const timestamp = this.tryToExtractTimestamp(message) ?? new Date();
return !isRetryDateExceeded(timestamp, maxRetryDuration);
}
getMessageRetryDelayInSeconds(message) {
// if not defined, this is the first attempt
const retries = this.tryToExtractNumberOfRetries(message) ?? 0;
// exponential backoff -> (2 ^ (attempts)) * delay
// delay = 1 second
return Math.pow(2, retries);
}
updateInternalProperties(message) {
const messageCopy = { ...message }; // clone the message to avoid mutation
/**
* If the message doesn't have a timestamp field -> add it
* will be used to prevent infinite retries on the same message
*/
if (!this.tryToExtractTimestamp(message)) {
// @ts-ignore
messageCopy[this.messageTimestampField] = new Date().toISOString();
this.logger.warn(`${this.messageTimestampField} not defined, adding it automatically`);
}
/**
* add/increment the number of retries performed to exponential message delay
*/
const numberOfRetries = this.tryToExtractNumberOfRetries(message);
// @ts-ignore
messageCopy[this.messageRetryLaterCountField] =
numberOfRetries !== undefined ? numberOfRetries + 1 : 0;
return messageCopy;
}
tryToExtractTimestamp(message) {
// @ts-ignore
if (this.messageTimestampField in message) {
// @ts-ignore
const res = toDatePreprocessor(message[this.messageTimestampField]);
if (!(res instanceof Date)) {
throw new Error(`${this.messageTimestampField} invalid type`);
}
return res;
}
return undefined;
}
tryToExtractNumberOfRetries(message) {
if (this.messageRetryLaterCountField in message &&
typeof message[this.messageRetryLaterCountField] === 'number') {
// @ts-ignore
return message[this.messageRetryLaterCountField];
}
return undefined;
}
resolveNextPreHandlerFunctionInternal(preHandlers, executionContext, message, index, preHandlerOutput, resolve, reject) {
return (preHandlerResult) => {
if (preHandlerResult.error) {
reject(preHandlerResult.error);
}
if (preHandlers.length < index + 1) {
resolve(preHandlerOutput);
}
else {
// biome-ignore lint/style/noNonNullAssertion: <explanation>
preHandlers[index](message, executionContext,
// @ts-ignore
preHandlerOutput, this.resolveNextPreHandlerFunctionInternal(preHandlers, executionContext, message, index + 1, preHandlerOutput, resolve, reject));
}
};
}
/**
* Offload message payload to an external store if it exceeds the threshold.
* Returns a special type that contains a pointer to the offloaded payload or the original payload if it was not offloaded.
* Requires message size as only the implementation knows how to calculate it.
*/
async offloadMessagePayloadIfNeeded(message, messageSizeFn) {
if (!this.payloadStoreConfig ||
messageSizeFn() <= this.payloadStoreConfig.messageSizeThreshold) {
return message;
}
let offloadedPayloadPointer;
const serializedPayload = await this.payloadStoreConfig.serializer.serialize(message);
try {
offloadedPayloadPointer = await this.payloadStoreConfig.store.storePayload(serializedPayload);
}
finally {
if (isDestroyable(serializedPayload)) {
await serializedPayload.destroy();
}
}
return {
offloadedPayloadPointer,
offloadedPayloadSize: serializedPayload.size,
// @ts-ignore
[this.messageIdField]: message[this.messageIdField],
// @ts-ignore
[this.messageTypeField]: message[this.messageTypeField],
// @ts-ignore
[this.messageTimestampField]: message[this.messageTimestampField],
// @ts-ignore
[this.messageDeduplicationIdField]: message[this.messageDeduplicationIdField],
// @ts-ignore
[this.messageDeduplicationOptionsField]: message[this.messageDeduplicationOptionsField],
};
}
/**
* Retrieve previously offloaded message payload using provided pointer payload.
* Returns the original payload or an error if the payload was not found or could not be parsed.
*/
async retrieveOffloadedMessagePayload(maybeOffloadedPayloadPointerPayload) {
if (!this.payloadStoreConfig) {
return {
error: new Error('Payload store is not configured, cannot retrieve offloaded message payload'),
};
}
const pointerPayloadParseResult = OFFLOADED_PAYLOAD_POINTER_PAYLOAD_SCHEMA.safeParse(maybeOffloadedPayloadPointerPayload);
if (!pointerPayloadParseResult.success) {
return {
error: new Error('Given payload is not a valid offloaded payload pointer payload', {
cause: pointerPayloadParseResult.error,
}),
};
}
const serializedOffloadedPayloadReadable = await this.payloadStoreConfig.store.retrievePayload(pointerPayloadParseResult.data.offloadedPayloadPointer);
if (serializedOffloadedPayloadReadable === null) {
return {
error: new Error(`Payload with key ${pointerPayloadParseResult.data.offloadedPayloadPointer} was not found in the store`),
};
}
const serializedOffloadedPayloadString = await streamWithKnownSizeToString(serializedOffloadedPayloadReadable, pointerPayloadParseResult.data.offloadedPayloadSize);
try {
return { result: JSON.parse(serializedOffloadedPayloadString) };
}
catch (e) {
return { error: new Error('Failed to parse serialized offloaded payload', { cause: e }) };
}
}
/**
* Checks if the message is duplicated against the deduplication store.
* Returns true if the message is duplicated.
* Returns false if message is not duplicated or deduplication config is missing.
*/
async isMessageDuplicated(message, requester) {
if (!this.isDeduplicationEnabledForMessage(message)) {
return false;
}
const deduplicationId = this.getMessageDeduplicationId(message);
const deduplicationConfig = this.messageDeduplicationConfig;
try {
return await deduplicationConfig.deduplicationStore.keyExists(`${requester.toString()}:${deduplicationId}`);
}
catch (err) {
this.handleError(err);
// In case of errors, we treat the message as not duplicated to enable further processing
return false;
}
}
/**
* Checks if the message is duplicated.
* If it's not, stores the deduplication key in the deduplication store and returns false.
* If it is, returns true.
* If deduplication config is not provided, always returns false to allow further processing of the message.
*/
async deduplicateMessage(message, requester) {
if (!this.isDeduplicationEnabledForMessage(message)) {
return { isDuplicated: false };
}
const deduplicationId = this.getMessageDeduplicationId(message);
const { deduplicationWindowSeconds } = this.getParsedMessageDeduplicationOptions(message);
const deduplicationConfig = this.messageDeduplicationConfig;
try {
const wasDeduplicationKeyStored = await deduplicationConfig.deduplicationStore.setIfNotExists(`${requester.toString()}:${deduplicationId}`, new Date().toISOString(), deduplicationWindowSeconds);
return { isDuplicated: !wasDeduplicationKeyStored };
}
catch (err) {
this.handleError(err);
// In case of errors, we treat the message as not duplicated to enable further processing
return { isDuplicated: false };
}
}
/**
* Acquires exclusive lock for the message to prevent concurrent processing.
* If lock was acquired successfully, returns a lock object that should be released after processing.
* If lock couldn't be acquired due to timeout (meaning another process acquired it earlier), returns AcquireLockTimeoutError
* If lock couldn't be acquired for any other reasons or if deduplication config is not provided, always returns a lock object that does nothing, so message processing can continue.
*/
async acquireLockForMessage(message) {
if (!this.isDeduplicationEnabledForMessage(message)) {
return { result: noopReleasableLock };
}
const deduplicationId = this.getMessageDeduplicationId(message);
const deduplicationOptions = this.getParsedMessageDeduplicationOptions(message);
const deduplicationConfig = this.messageDeduplicationConfig;
const acquireLockResult = await deduplicationConfig.deduplicationStore.acquireLock(`${DeduplicationRequesterEnum.Consumer.toString()}:${deduplicationId}`, deduplicationOptions);
if (acquireLockResult.error && !isAcquireLockTimeoutError(acquireLockResult.error)) {
this.handleError(acquireLockResult.error);
return { result: noopReleasableLock };
}
return acquireLockResult;
}
isDeduplicationEnabledForMessage(message) {
return !!this.messageDeduplicationConfig && !!this.getMessageDeduplicationId(message);
}
getMessageDeduplicationId(message) {
// @ts-ignore
return message[this.messageDeduplicationIdField];
}
getParsedMessageDeduplicationOptions(message) {
const parsedOptions = MESSAGE_DEDUPLICATION_OPTIONS_SCHEMA.safeParse(
// @ts-expect-error
message[this.messageDeduplicationOptionsField] ?? {});
if (parsedOptions.error) {
this.logger.warn({ error: parsedOptions.error.message }, `${this.messageDeduplicationOptionsField} contains one or more invalid values, falling back to default options`);
return DEFAULT_MESSAGE_DEDUPLICATION_OPTIONS;
}
return {
...DEFAULT_MESSAGE_DEDUPLICATION_OPTIONS,
...Object.fromEntries(Object.entries(parsedOptions.data).filter(([_, value]) => value !== undefined)),
};
}
}
//# sourceMappingURL=AbstractQueueService.js.map