@message-queue-toolkit/core
Version:
Useful utilities, interfaces and base classes for message queue handling. Supports AMQP and SQS with a common abstraction on top currently
569 lines • 27.1 kB
JavaScript
import { types } from 'node:util';
import { resolveGlobalErrorLogObject, stringValueSerializer, } from '@lokalise/node-core';
import { MESSAGE_DEDUPLICATION_OPTIONS_SCHEMA, } from '@message-queue-toolkit/schemas';
import { getProperty, setProperty } from 'dot-prop';
import { isAcquireLockTimeoutError, } from "../message-deduplication/AcquireLockTimeoutError.js";
import { DEFAULT_MESSAGE_DEDUPLICATION_OPTIONS, DeduplicationRequesterEnum, noopReleasableLock, } from "../message-deduplication/messageDeduplicationTypes.js";
import { jsonStreamStringifySerializer } from "../payload-store/JsonStreamStringifySerializer.js";
import { OFFLOADED_PAYLOAD_POINTER_PAYLOAD_SCHEMA, } from "../payload-store/offloadedPayloadMessageSchemas.js";
import { isDestroyable, isMultiPayloadStoreConfig } from "../payload-store/payloadStoreTypes.js";
import { isRetryDateExceeded } from "../utils/dateUtils.js";
import { streamWithKnownSizeToString } from "../utils/streamUtils.js";
import { toDatePreprocessor } from "../utils/toDateProcessor.js";
import { resolveHandlerSpy, TYPE_NOT_RESOLVED } from "./HandlerSpy.js";
import { MessageSchemaContainer } from "./MessageSchemaContainer.js";
import { isMessageTypePathConfig, resolveMessageType, } from "./MessageTypeResolver.js";
export class AbstractQueueService {
/**
* Used to keep track of the number of `retryLater` results received for a message to be able to
* calculate the delay for the next retry
*/
messageRetryLaterCountField = '_internalRetryLaterCount';
/**
* Used to know when the message was sent initially so we can have a max retry date and avoid
* a infinite `retryLater` loop
*/
messageTimestampField;
/**
* Used to know the message deduplication id
*/
messageDeduplicationIdField;
/**
* Used to know the store-based message deduplication options
*/
messageDeduplicationOptionsField;
/**
* Used to know where metadata is stored - for debug logging purposes only
*/
messageMetadataField;
errorReporter;
logger;
messageIdField;
/**
* Configuration for resolving message types.
*/
messageTypeResolver;
logMessages;
creationConfig;
locatorConfig;
deletionConfig;
payloadStoreConfig;
messageDeduplicationConfig;
messageMetricsManager;
_handlerSpy;
isInitted;
get handlerSpy() {
if (!this._handlerSpy) {
throw new Error('HandlerSpy was not instantiated, please pass `handlerSpy` parameter during queue service creation.');
}
return this._handlerSpy;
}
constructor({ errorReporter, logger, messageMetricsManager }, options) {
this.errorReporter = errorReporter;
this.logger = logger;
this.messageMetricsManager = messageMetricsManager;
this.messageIdField = options.messageIdField ?? 'id';
this.messageTypeResolver = options.messageTypeResolver;
this.messageTimestampField = options.messageTimestampField ?? 'timestamp';
this.messageDeduplicationIdField = options.messageDeduplicationIdField ?? 'deduplicationId';
this.messageDeduplicationOptionsField =
options.messageDeduplicationOptionsField ?? 'deduplicationOptions';
this.messageMetadataField = options.messageMetadataField ?? 'metadata';
this.creationConfig = options.creationConfig;
this.locatorConfig = options.locatorConfig;
this.deletionConfig = options.deletionConfig;
this.payloadStoreConfig = options.payloadStoreConfig
? {
serializer: jsonStreamStringifySerializer,
...options.payloadStoreConfig,
}
: undefined;
this.messageDeduplicationConfig = options.messageDeduplicationConfig;
this.logMessages = options.logMessages ?? false;
this._handlerSpy = resolveHandlerSpy(options);
this.isInitted = false;
}
resolveConsumerMessageSchemaContainer(options) {
const messageSchemas = options.handlers.map((entry) => ({
schema: entry.schema,
messageType: entry.messageType,
}));
const messageDefinitions = options.handlers
.filter((entry) => entry.definition !== undefined)
.map((entry) => ({
// biome-ignore lint/style/noNonNullAssertion: filtered above
definition: entry.definition,
messageType: entry.messageType,
}));
return new MessageSchemaContainer({
messageTypeResolver: options.messageTypeResolver,
messageSchemas,
messageDefinitions,
});
}
resolvePublisherMessageSchemaContainer(options) {
const messageSchemas = options.messageSchemas.map((schema) => ({ schema }));
return new MessageSchemaContainer({
messageTypeResolver: options.messageTypeResolver,
messageSchemas,
messageDefinitions: [],
});
}
/**
* Resolves message type from message data and optional attributes using messageTypeResolver.
*
* @param messageData - The parsed message data
* @param messageAttributes - Optional message-level attributes (e.g., PubSub attributes)
* @returns The resolved message type, or undefined if not configured
*/
resolveMessageTypeFromMessage(messageData, messageAttributes) {
if (this.messageTypeResolver) {
const context = { messageData, messageAttributes };
return resolveMessageType(this.messageTypeResolver, context);
}
return undefined;
}
/**
* Format message for logging
*/
resolveMessageLog(_processedMessageMetadata) {
return null;
}
logMessageProcessed(processedMessageMetadata) {
const processedMessageMetadataLog = {
processingResult: processedMessageMetadata.processingResult,
messageId: processedMessageMetadata.messageId,
messageType: processedMessageMetadata.messageType,
queueName: processedMessageMetadata.queueName,
messageTimestamp: processedMessageMetadata.messageTimestamp,
messageDeduplicationId: processedMessageMetadata.messageDeduplicationId,
messageProcessingStartTimestamp: processedMessageMetadata.messageProcessingStartTimestamp,
messageProcessingEndTimestamp: processedMessageMetadata.messageProcessingEndTimestamp,
messageMetadata: stringValueSerializer(processedMessageMetadata.messageMetadata),
};
const resolvedMessageLog = this.resolveMessageLog(processedMessageMetadata);
this.logger.debug({
processedMessageMetadata: processedMessageMetadataLog,
...(resolvedMessageLog ? { message: resolvedMessageLog } : {}),
}, `Finished processing message ${processedMessageMetadata.messageId}`);
}
handleError(err, context) {
const logObject = resolveGlobalErrorLogObject(err);
this.logger.error({
...logObject,
...context,
});
if (types.isNativeError(err)) {
this.errorReporter.report({ error: err, context });
}
}
handleMessageProcessed(params) {
const { message, processingResult, messageId } = params;
const messageProcessingEndTimestamp = Date.now();
// Resolve message type once and pass to spy for consistent type resolution
const messageType = message
? (this.resolveMessageTypeFromMessage(message) ?? TYPE_NOT_RESOLVED)
: TYPE_NOT_RESOLVED;
this._handlerSpy?.addProcessedMessage({
message,
processingResult,
}, messageId, messageType);
const debugMessageLoggingEnabled = this.logMessages && this.logger.isLevelEnabled('debug');
if (!debugMessageLoggingEnabled && !this.messageMetricsManager)
return;
const processedMessageMetadata = this.resolveProcessedMessageMetadata(message, processingResult, params.messageProcessingStartTimestamp, messageProcessingEndTimestamp, params.queueName, messageId);
if (debugMessageLoggingEnabled) {
this.logMessageProcessed(processedMessageMetadata);
}
if (this.messageMetricsManager) {
this.messageMetricsManager.registerProcessedMessage(processedMessageMetadata);
}
}
resolveProcessedMessageMetadata(message, processingResult, messageProcessingStartTimestamp, messageProcessingEndTimestamp, queueName, messageId) {
// @ts-expect-error
const resolvedMessageId = message?.[this.messageIdField] ?? messageId;
const messageTimestamp = message ? this.tryToExtractTimestamp(message)?.getTime() : undefined;
const messageType = message ? this.resolveMessageTypeFromMessage(message) : undefined;
const messageDeduplicationId = message && this.messageDeduplicationIdField in message
? // @ts-expect-error
message[this.messageDeduplicationIdField]
: undefined;
const messageMetadata = message && this.messageMetadataField in message
? // @ts-expect-error
message[this.messageMetadataField]
: undefined;
return {
processingResult,
messageId: resolvedMessageId ?? '(unknown id)',
messageType,
queueName,
message,
messageTimestamp,
messageDeduplicationId,
messageProcessingStartTimestamp,
messageProcessingEndTimestamp,
messageMetadata,
};
}
processPrehandlersInternal(preHandlers, message) {
if (preHandlers.length === 0) {
return Promise.resolve({});
}
return new Promise((resolve, reject) => {
try {
const preHandlerOutput = {};
const next = this.resolveNextFunction(preHandlers, message, 0, preHandlerOutput, resolve, reject);
next({ result: 'success' });
}
catch (err) {
reject(err);
}
});
}
async preHandlerBarrierInternal(barrier, message, executionContext, preHandlerOutput) {
if (!barrier) {
// @ts-expect-error
return {
isPassing: true,
output: undefined,
};
}
return await barrier(message, executionContext, preHandlerOutput);
}
shouldBeRetried(message, maxRetryDuration) {
const timestamp = this.tryToExtractTimestamp(message) ?? new Date();
return !isRetryDateExceeded(timestamp, maxRetryDuration);
}
getMessageRetryDelayInSeconds(message) {
// if not defined, this is the first attempt
const retries = this.tryToExtractNumberOfRetries(message) ?? 0;
// exponential backoff -> (2 ^ (attempts)) * delay
// delay = 1 second
return Math.pow(2, retries);
}
updateInternalProperties(message) {
const messageCopy = { ...message }; // clone the message to avoid mutation
/**
* If the message doesn't have a timestamp field -> add it
* will be used to prevent infinite retries on the same message
*/
if (!this.tryToExtractTimestamp(message)) {
// @ts-expect-error
messageCopy[this.messageTimestampField] = new Date().toISOString();
this.logger.warn(`${this.messageTimestampField} not defined, adding it automatically`);
}
/**
* add/increment the number of retries performed to exponential message delay
*/
const numberOfRetries = this.tryToExtractNumberOfRetries(message);
// @ts-expect-error
messageCopy[this.messageRetryLaterCountField] =
numberOfRetries !== undefined ? numberOfRetries + 1 : 0;
return messageCopy;
}
tryToExtractTimestamp(message) {
if (this.messageTimestampField in message) {
// @ts-expect-error
const res = toDatePreprocessor(message[this.messageTimestampField]);
if (!(res instanceof Date)) {
throw new Error(`${this.messageTimestampField} invalid type`);
}
return res;
}
return undefined;
}
tryToExtractNumberOfRetries(message) {
if (this.messageRetryLaterCountField in message &&
typeof message[this.messageRetryLaterCountField] === 'number') {
// @ts-expect-error
return message[this.messageRetryLaterCountField];
}
return undefined;
}
resolveNextPreHandlerFunctionInternal(preHandlers, executionContext, message, index, preHandlerOutput, resolve, reject) {
return (preHandlerResult) => {
if (preHandlerResult.error) {
reject(preHandlerResult.error);
}
if (preHandlers.length < index + 1) {
resolve(preHandlerOutput);
}
else {
// biome-ignore lint/style/noNonNullAssertion: It's ok
preHandlers[index](message, executionContext, preHandlerOutput, this.resolveNextPreHandlerFunctionInternal(preHandlers, executionContext, message, index + 1, preHandlerOutput, resolve, reject));
}
};
}
/**
* Resolves the store and store name for outgoing (publishing) messages.
* For multi-store: uses outgoingStore from config.
* For single-store: uses the configured store and storeName.
* @throws Error if payloadStoreConfig is not configured or the named store is not found.
*/
resolveOutgoingStore() {
if (!this.payloadStoreConfig) {
throw new Error('Payload store is not configured');
}
if (isMultiPayloadStoreConfig(this.payloadStoreConfig)) {
const storeName = this.payloadStoreConfig.outgoingStore;
const store = this.payloadStoreConfig.stores[storeName];
if (!store) {
throw new Error(`Outgoing store "${storeName}" not found in stores configuration. Available stores: ${Object.keys(this.payloadStoreConfig.stores).join(', ')}`);
}
return { store, storeName };
}
// Single-store configuration
return { store: this.payloadStoreConfig.store, storeName: this.payloadStoreConfig.storeName };
}
/**
* Resolves store from payloadRef (new format).
*/
resolveStoreFromPayloadRef(config, payloadRef) {
if (isMultiPayloadStoreConfig(config)) {
const store = config.stores[payloadRef.store];
if (!store) {
return {
error: new Error(`Store "${payloadRef.store}" specified in payloadRef not found in stores configuration. Available stores: ${Object.keys(config.stores).join(', ')}`),
};
}
return { result: { store, payloadId: payloadRef.id } };
}
// Single-store config - validate that payloadRef.store matches configured store name
if (payloadRef.store !== config.storeName) {
return {
error: new Error(`Store "${payloadRef.store}" specified in payloadRef does not match configured store name "${config.storeName}". ` +
'This may indicate a misconfiguration or that the message was published by a different system. ' +
'If you need to consume messages from multiple stores, consider using MultiPayloadStoreConfig.'),
};
}
return { result: { store: config.store, payloadId: payloadRef.id } };
}
/**
* Resolves store from legacy pointer (old format).
*/
resolveStoreFromLegacyPointer(config, legacyPointer) {
if (isMultiPayloadStoreConfig(config)) {
if (!config.defaultIncomingStore) {
return {
error: new Error('Message contains legacy offloadedPayloadPointer format, but no defaultIncomingStore is configured in multi-store setup. Please configure defaultIncomingStore or migrate messages to use payloadRef format.'),
};
}
const store = config.stores[config.defaultIncomingStore];
if (!store) {
return {
error: new Error(`Default incoming store "${config.defaultIncomingStore}" not found in stores configuration. Available stores: ${Object.keys(config.stores).join(', ')}`),
};
}
return { result: { store, payloadId: legacyPointer } };
}
return { result: { store: config.store, payloadId: legacyPointer } };
}
/**
* Resolves the store for incoming (consuming) messages based on payload reference.
* For multi-store with payloadRef: uses the store specified in payloadRef.
* For multi-store with legacy format: uses defaultIncomingStore.
* For single-store: always uses the configured store.
*/
resolveIncomingStore(payloadRef, legacyPointer) {
if (!this.payloadStoreConfig) {
return { error: new Error('Payload store is not configured') };
}
if (payloadRef) {
return this.resolveStoreFromPayloadRef(this.payloadStoreConfig, payloadRef);
}
if (legacyPointer) {
return this.resolveStoreFromLegacyPointer(this.payloadStoreConfig, legacyPointer);
}
return {
error: new Error('Invalid offloaded payload: neither payloadRef nor offloadedPayloadPointer is present'),
};
}
/**
* Offload message payload to an external store if it exceeds the threshold.
* Returns a special type that contains a pointer to the offloaded payload or the original payload if it was not offloaded.
* Requires message size as only the implementation knows how to calculate it.
*
* For multi-store configuration, uses the configured outgoingStore.
* For single-store configuration, uses the single store.
*
* The returned payload includes both the new payloadRef format and legacy fields for backward compatibility.
*/
async offloadMessagePayloadIfNeeded(message, messageSizeFn) {
if (!this.payloadStoreConfig ||
messageSizeFn() <= this.payloadStoreConfig.messageSizeThreshold) {
return message;
}
const { store, storeName } = this.resolveOutgoingStore();
const serializedPayload = await this.payloadStoreConfig.serializer.serialize(message);
let payloadId;
try {
payloadId = await store.storePayload(serializedPayload);
}
finally {
if (isDestroyable(serializedPayload)) {
await serializedPayload.destroy();
}
}
// Return message with both new and legacy formats for backward compatibility
const result = {
// Extended payload reference format
payloadRef: {
id: payloadId,
store: storeName,
size: serializedPayload.size,
},
// Legacy format for backward compatibility
offloadedPayloadPointer: payloadId,
offloadedPayloadSize: serializedPayload.size,
// @ts-expect-error
[this.messageIdField]: message[this.messageIdField],
// @ts-expect-error
[this.messageTimestampField]: message[this.messageTimestampField],
// @ts-expect-error
[this.messageDeduplicationIdField]: message[this.messageDeduplicationIdField],
// @ts-expect-error
[this.messageDeduplicationOptionsField]: message[this.messageDeduplicationOptionsField],
};
// Preserve message type field if using messageTypePath resolver (supports nested paths)
if (this.messageTypeResolver && isMessageTypePathConfig(this.messageTypeResolver)) {
const messageTypePath = this.messageTypeResolver.messageTypePath;
const typeValue = getProperty(message, messageTypePath);
if (typeValue !== undefined) {
setProperty(result, messageTypePath, typeValue);
}
}
return result;
}
/**
* Retrieve previously offloaded message payload using provided pointer payload.
* Returns the original payload or an error if the payload was not found or could not be parsed.
*
* Supports both new multi-store format (payloadRef) and legacy format (offloadedPayloadPointer).
*/
async retrieveOffloadedMessagePayload(maybeOffloadedPayloadPointerPayload) {
if (!this.payloadStoreConfig) {
return {
error: new Error('Payload store is not configured, cannot retrieve offloaded message payload'),
};
}
const pointerPayloadParseResult = OFFLOADED_PAYLOAD_POINTER_PAYLOAD_SCHEMA.safeParse(maybeOffloadedPayloadPointerPayload);
if (!pointerPayloadParseResult.success) {
return {
error: new Error('Given payload is not a valid offloaded payload pointer payload', {
cause: pointerPayloadParseResult.error,
}),
};
}
const parsedPayload = pointerPayloadParseResult.data;
// Resolve which store to use
const storeResult = this.resolveIncomingStore(parsedPayload.payloadRef, parsedPayload.offloadedPayloadPointer);
if (storeResult.error) {
return storeResult;
}
const { store, payloadId } = storeResult.result;
const payloadSize = parsedPayload.payloadRef?.size ?? parsedPayload.offloadedPayloadSize;
if (payloadSize === undefined) {
return {
error: new Error('Invalid offloaded payload: payload size is missing from both payloadRef and offloadedPayloadSize'),
};
}
// Retrieve the payload from the resolved store
const serializedOffloadedPayloadReadable = await store.retrievePayload(payloadId);
if (serializedOffloadedPayloadReadable === null) {
return {
error: new Error(`Payload with key ${payloadId} was not found in the store`),
};
}
const serializedOffloadedPayloadString = await streamWithKnownSizeToString(serializedOffloadedPayloadReadable, payloadSize);
try {
return { result: JSON.parse(serializedOffloadedPayloadString) };
}
catch (e) {
return { error: new Error('Failed to parse serialized offloaded payload', { cause: e }) };
}
}
/**
* Checks if the message is duplicated against the deduplication store.
* Returns true if the message is duplicated.
* Returns false if message is not duplicated or deduplication config is missing.
*/
async isMessageDuplicated(message, requester) {
if (!this.isDeduplicationEnabledForMessage(message)) {
return false;
}
const deduplicationId = this.getMessageDeduplicationId(message);
const deduplicationConfig = this.messageDeduplicationConfig;
try {
return await deduplicationConfig.deduplicationStore.keyExists(`${requester.toString()}:${deduplicationId}`);
}
catch (err) {
this.handleError(err);
// In case of errors, we treat the message as not duplicated to enable further processing
return false;
}
}
/**
* Checks if the message is duplicated.
* If it's not, stores the deduplication key in the deduplication store and returns false.
* If it is, returns true.
* If deduplication config is not provided, always returns false to allow further processing of the message.
*/
async deduplicateMessage(message, requester) {
if (!this.isDeduplicationEnabledForMessage(message)) {
return { isDuplicated: false };
}
const deduplicationId = this.getMessageDeduplicationId(message);
const { deduplicationWindowSeconds } = this.getParsedMessageDeduplicationOptions(message);
const deduplicationConfig = this.messageDeduplicationConfig;
try {
const wasDeduplicationKeyStored = await deduplicationConfig.deduplicationStore.setIfNotExists(`${requester.toString()}:${deduplicationId}`, new Date().toISOString(), deduplicationWindowSeconds);
return { isDuplicated: !wasDeduplicationKeyStored };
}
catch (err) {
this.handleError(err);
// In case of errors, we treat the message as not duplicated to enable further processing
return { isDuplicated: false };
}
}
/**
* Acquires exclusive lock for the message to prevent concurrent processing.
* If lock was acquired successfully, returns a lock object that should be released after processing.
* If lock couldn't be acquired due to timeout (meaning another process acquired it earlier), returns AcquireLockTimeoutError
* If lock couldn't be acquired for any other reasons or if deduplication config is not provided, always returns a lock object that does nothing, so message processing can continue.
*/
async acquireLockForMessage(message) {
if (!this.isDeduplicationEnabledForMessage(message)) {
return { result: noopReleasableLock };
}
const deduplicationId = this.getMessageDeduplicationId(message);
const deduplicationOptions = this.getParsedMessageDeduplicationOptions(message);
const deduplicationConfig = this.messageDeduplicationConfig;
const acquireLockResult = await deduplicationConfig.deduplicationStore.acquireLock(`${DeduplicationRequesterEnum.Consumer.toString()}:${deduplicationId}`, deduplicationOptions);
if (acquireLockResult.error && !isAcquireLockTimeoutError(acquireLockResult.error)) {
this.handleError(acquireLockResult.error);
return { result: noopReleasableLock };
}
return acquireLockResult;
}
isDeduplicationEnabledForMessage(message) {
return !!this.messageDeduplicationConfig && !!this.getMessageDeduplicationId(message);
}
getMessageDeduplicationId(message) {
// @ts-expect-error
return message[this.messageDeduplicationIdField];
}
getParsedMessageDeduplicationOptions(message) {
const parsedOptions = MESSAGE_DEDUPLICATION_OPTIONS_SCHEMA.safeParse(
// @ts-expect-error
message[this.messageDeduplicationOptionsField] ?? {});
if (parsedOptions.error) {
this.logger.warn({ error: parsedOptions.error.message }, `${this.messageDeduplicationOptionsField} contains one or more invalid values, falling back to default options`);
return DEFAULT_MESSAGE_DEDUPLICATION_OPTIONS;
}
return {
...DEFAULT_MESSAGE_DEDUPLICATION_OPTIONS,
...Object.fromEntries(Object.entries(parsedOptions.data).filter(([_, value]) => value !== undefined)),
};
}
}
//# sourceMappingURL=AbstractQueueService.js.map