@hotmeshio/hotmesh
Version:
Permanent-Memory Workflows & AI Agents
306 lines (305 loc) • 11 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.RedisStreamService = void 0;
const index_1 = require("../../index");
const utils_1 = require("../../../../modules/utils");
const key_1 = require("../../../../modules/key");
const enums_1 = require("../../../../modules/enums");
class RedisStreamService extends index_1.StreamService {
constructor(streamClient, storeClient, config = {}) {
super(streamClient, storeClient, config);
}
async init(namespace, appId, logger) {
this.namespace = namespace;
this.logger = logger;
this.appId = appId;
}
mintKey(type, params) {
if (!this.namespace)
throw new Error('namespace not set');
return key_1.KeyService.mintKey(this.namespace, type, {
...params,
appId: this.appId,
});
}
transact() {
return this.streamClient.multi();
}
// Core streaming operations
async createStream(streamName) {
try {
// streams are created when you add messages.
// To create an empty stream, we can add and delete a dummy message.
const dummyId = await this.streamClient.XADD(streamName, '*', {
field: 'value',
});
await this.streamClient.XDEL(streamName, dummyId);
return true;
}
catch (error) {
this.logger.error(`Error creating stream ${streamName}`, { error });
throw error;
}
}
async deleteStream(streamName) {
try {
const result = await this.streamClient.DEL(streamName);
return result > 0;
}
catch (error) {
this.logger.error(`Error deleting stream ${streamName}`, { error });
throw error;
}
}
// Consumer group operations
async createConsumerGroup(streamName, groupName) {
try {
const result = (await this.storeClient.sendCommand([
'XGROUP',
'CREATE',
streamName,
groupName,
'$',
'MKSTREAM',
]));
return result === 'OK';
}
catch (error) {
const streamType = 'with MKSTREAM';
this.logger.debug(`x-group-error ${streamType} for key: ${streamName} and group: ${groupName}`, { error });
throw error;
}
}
async deleteConsumerGroup(streamName, groupName) {
try {
const result = await this.streamClient.xGroupDestroy(streamName, groupName);
return result;
}
catch (error) {
this.logger.error(`Error deleting consumer group ${groupName} for stream ${streamName}`, { error });
throw error;
}
}
async publishMessages(streamName, messages, options) {
try {
const multi = options?.transaction ||
(messages.length > 1 && this.storeClient.multi());
let response;
for (const message of messages) {
response = await (multi || this.storeClient).XADD(streamName, '*', {
message: message,
});
}
if (multi && !options?.transaction) {
//only exec if we created the multi;
//otherwise caller is responsible
return (await multi.exec()).map((result) => result[1]);
}
else {
return [response];
}
}
catch (error) {
this.logger.error(`ioredis-xadd-error key: ${streamName}`, { error });
throw error;
}
}
async consumeMessages(streamName, groupName, consumerName, options) {
const command = 'GROUP';
const blockOption = 'BLOCK';
const streamsOption = 'STREAMS';
const id = '>';
try {
const result = await this.streamClient.sendCommand([
'XREADGROUP',
command,
groupName,
consumerName,
blockOption,
options?.blockTimeout?.toString() ?? enums_1.HMSH_BLOCK_TIME_MS.toString(),
streamsOption,
streamName,
id,
]);
const response = [];
if ((0, utils_1.isStreamMessage)(result)) {
const [[, messages]] = result;
for (const [id, message] of messages) {
response.push({
id,
data: (0, utils_1.parseStreamMessage)(message[1]),
});
}
}
else {
return [];
}
return response;
}
catch (error) {
this.logger.error(`Error consuming messages from ${streamName}`, {
error,
});
throw error;
}
}
async ackAndDelete(stream, group, ids) {
const multi = this.storeClient.multi();
await this.acknowledgeMessages(stream, group, ids, { multi });
await this.deleteMessages(stream, group, ids, { multi });
await multi.exec();
return ids.length;
}
async acknowledgeMessages(stream, group, ids, options) {
try {
if (options?.multi) {
options.multi.XACK(stream, group, ids);
return options.multi;
}
else {
return await this.streamClient.XACK(stream, group, ...ids);
}
}
catch (error) {
this.logger.error(`Error in acknowledging messages in group: ${group} for key: ${stream}`, { error });
throw error;
}
}
async deleteMessages(stream, group, ids, options) {
try {
if (options?.multi) {
options.multi.xDel(stream, ids);
return options.multi;
}
else {
return await this.streamClient.XDEL(stream, ...ids);
}
}
catch (error) {
this.logger.error(`Error in deleting messages with ids: ${ids.join(',')} for key: ${stream}`, { error });
throw error;
}
}
async getPendingMessages(stream, group, count, consumer) {
try {
const start = '-';
const end = '+';
const args = [stream, group];
if (start)
args.push(start);
if (end)
args.push(end);
if (count !== undefined)
args.push(count.toString());
if (consumer)
args.push(consumer);
try {
return await this.streamClient.sendCommand(['XPENDING', ...args]);
}
catch (error) {
this.logger.error('error, args', { error }, args);
}
}
catch (error) {
this.logger.error(`Error retrieving pending messages for group: ${group} in key: ${stream}`, { error });
throw error;
}
}
// Retry Method
async retryMessages(streamName, groupName, options) {
let pendingMessages = [];
const pendingMessagesInfo = await this.getPendingMessages(streamName, groupName, options?.limit); //[[ '1688768134881-0', 'testConsumer1', 1017, 1 ]]
for (const pendingMessageInfo of pendingMessagesInfo) {
if (Array.isArray(pendingMessageInfo)) {
const [id, , elapsedTimeMs, deliveryCount] = pendingMessageInfo;
if (elapsedTimeMs > options?.minIdleTime) {
const reclaimedMessage = await this.claimMessage(streamName, groupName, options?.consumerName, options?.minIdleTime, id);
pendingMessages = pendingMessages.concat(reclaimedMessage);
}
}
}
return pendingMessages;
}
async claimMessage(stream, group, consumer, minIdleTime, id, ...args) {
try {
const message = (await this.streamClient.sendCommand([
'XCLAIM',
stream,
group,
consumer,
minIdleTime.toString(),
id,
...args,
]));
return {
id: message[0][0],
data: (0, utils_1.parseStreamMessage)(message[0][1][1]),
};
}
catch (error) {
this.logger.error(`Error in claiming message with id: ${id} in group: ${group} for key: ${stream}`, { error });
throw error;
}
}
async getStreamStats(streamName) {
return {
messageCount: await this.getStreamDepth(streamName),
};
}
async getStreamDepth(streamName, options) {
try {
if (options?.multi) {
options.multi.XLEN(streamName);
return 0;
}
const length = await this.streamClient.XLEN(streamName);
return length;
}
catch (error) {
this.logger.error(`Error getting depth for ${streamName}`, { error });
throw error;
}
}
async getStreamDepths(streamNames) {
const multi = this.storeClient.multi();
const uniqueStreams = new Map(); // Use a Map to track unique streams
// Add unique streams to the multi command
streamNames.forEach((profile) => {
if (!uniqueStreams.has(profile.stream)) {
uniqueStreams.set(profile.stream, -1); // Initialize depth to -1 as a placeholder
this.getStreamDepth(profile.stream, { multi: multi });
}
});
// Execute all commands in multi (ioredis strips null fields)
const streamDepthResults = (await multi.exec());
// Update the uniqueStreams map with actual depths from multi.exec() results
Array.from(uniqueStreams.keys()).forEach((stream, idx) => {
uniqueStreams.set(stream, streamDepthResults[idx]);
});
// Map back to the original `streamNames` array with correct depths
return streamNames.map((profile) => {
return {
stream: profile.stream,
depth: uniqueStreams.get(profile.stream) || 0,
};
});
}
async trimStream(streamName, options) {
//no-op for now
return 0;
}
// Provider-specific helpers
getProviderSpecificFeatures() {
return {
supportsBatching: true,
supportsDeadLetterQueue: false,
supportsOrdering: true,
supportsTrimming: true,
supportsRetry: true,
supportsNotifications: false,
maxMessageSize: 512 * 1024 * 1024,
maxBatchSize: 1000,
};
}
}
exports.RedisStreamService = RedisStreamService;