@hotmeshio/hotmesh
Version:
Permanent-Memory Workflows & AI Agents
273 lines (272 loc) • 10.2 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.IORedisStreamService = void 0;
const index_1 = require("../../index");
const utils_1 = require("../../../../modules/utils");
const key_1 = require("../../../../modules/key");
const enums_1 = require("../../../../modules/enums");
class IORedisStreamService extends index_1.StreamService {
constructor(streamClient, storeClient, config = {}) {
super(streamClient, storeClient, config);
}
async init(namespace, appId, logger) {
this.namespace = namespace;
this.logger = logger;
this.appId = appId;
}
mintKey(type, params) {
if (!this.namespace)
throw new Error('namespace not set');
return key_1.KeyService.mintKey(this.namespace, type, {
...params,
appId: this.appId,
});
}
transact() {
return this.streamClient.multi();
}
// Core streaming operations
async createStream(streamName) {
try {
// streams are created when you add messages.
// To create an empty stream, we can add and delete a dummy message.
const dummyId = await this.streamClient.xadd(streamName, '*', 'field', 'value');
await this.streamClient.xdel(streamName, dummyId);
return true;
}
catch (error) {
this.logger.error(`Error creating stream ${streamName}`, { error });
throw error;
}
}
async deleteStream(streamName) {
try {
const result = await this.streamClient.del(streamName);
return result > 0;
}
catch (error) {
this.logger.error(`Error deleting stream ${streamName}`, { error });
throw error;
}
}
// Consumer group operations
async createConsumerGroup(key, groupName) {
try {
return ((await this.storeClient.xgroup('CREATE', key, groupName, '$', 'MKSTREAM')) === 'OK');
}
catch (err) {
this.logger.debug('stream-mkstream-caught', { key, group: groupName });
throw err;
}
}
async deleteConsumerGroup(streamName, groupName) {
try {
const result = await this.streamClient.xgroup('DESTROY', streamName, groupName);
return result === 1;
}
catch (error) {
this.logger.error(`Error deleting consumer group ${groupName} for stream ${streamName}`, { error });
throw error;
}
}
// Message operations
async publishMessages(streamName, messages, options) {
try {
const multi = options?.transaction ||
(messages.length > 1 && this.storeClient.multi());
let response;
for (const message of messages) {
response = await (multi || this.storeClient).xadd(streamName, '*', 'message', message);
}
if (multi && !options?.transaction) {
//only exec if we created the multi;
//otherwise caller is responsible
return (await multi.exec()).map((result) => result[1]);
}
else {
return [response];
}
}
catch (error) {
this.logger.error(`ioredis-xadd-error key: ${streamName}`, { error });
throw error;
}
}
async consumeMessages(streamName, groupName, consumerName, options) {
try {
const result = await this.streamClient.xreadgroup('GROUP', groupName, consumerName, 'BLOCK', options?.blockTimeout ?? enums_1.HMSH_BLOCK_TIME_MS, 'STREAMS', streamName, '>');
const response = [];
if ((0, utils_1.isStreamMessage)(result)) {
const [[, messages]] = result;
for (const [id, message] of messages) {
response.push({
id,
data: (0, utils_1.parseStreamMessage)(message[1]),
});
}
}
else {
return [];
}
return response;
}
catch (error) {
this.logger.error(`Error consuming messages from ${streamName}`, {
error,
});
throw error;
}
}
async ackAndDelete(stream, group, ids) {
const multi = this.storeClient.multi();
this.acknowledgeMessages(stream, group, ids, { multi });
this.deleteMessages(stream, group, ids, { multi });
await multi.exec();
return ids.length;
}
async acknowledgeMessages(stream, group, ids, options) {
try {
if (options?.multi) {
options.multi.xack(stream, group, ...ids);
return options.multi;
}
else {
return await this.streamClient.xack(stream, group, ...ids);
}
}
catch (error) {
this.logger.error(`Error in acknowledging messages: [${ids}] in group: ${group} for key: ${stream}`, { error });
throw error;
}
}
async deleteMessages(stream, group, ids, options) {
try {
if (options?.multi) {
options.multi.xdel(stream, ...ids);
return options.multi;
}
else {
return await this.streamClient.xdel(stream, ...ids);
}
}
catch (error) {
this.logger.error(`Error in deleting messages: ${ids} for key: ${stream}`, { error });
throw error;
}
}
async getPendingMessages(stream, group, count, consumer) {
const start = '-';
const end = '+';
try {
const args = [stream, group];
if (start)
args.push(start);
if (end)
args.push(end);
if (count !== undefined)
args.push(count.toString());
if (consumer)
args.push(consumer);
try {
return (await this.streamClient.call('XPENDING', ...args));
}
catch (error) {
this.logger.error('err, args', { error }, args);
}
}
catch (error) {
this.logger.error(`Error in retrieving pending messages for [stream ${stream}], [group ${group}]`, { error });
throw error;
}
}
// Retry Method
async retryMessages(streamName, groupName, options) {
let pendingMessages = [];
const pendingMessagesInfo = await this.getPendingMessages(streamName, groupName, options?.limit); //[[ '1688768134881-0', 'testConsumer1', 1017, 1 ]]
for (const pendingMessageInfo of pendingMessagesInfo) {
if (Array.isArray(pendingMessageInfo)) {
const [id, , elapsedTimeMs, deliveryCount] = pendingMessageInfo;
if (elapsedTimeMs > options?.minIdleTime) {
const reclaimedMessage = await this.claimMessage(streamName, groupName, options?.consumerName, options?.minIdleTime, id);
pendingMessages = pendingMessages.concat(reclaimedMessage);
}
}
}
return pendingMessages;
}
async claimMessage(streamName, groupName, consumerName, minIdleTime, messageId, ...args) {
try {
const message = (await this.streamClient.xclaim(streamName, groupName, consumerName, minIdleTime, messageId, ...args));
return {
id: message[0][0],
data: (0, utils_1.parseStreamMessage)(message[0][1][1]),
};
}
catch (error) {
this.logger.error(`Error in claiming message with id: ${messageId} in group: ${groupName} for key: ${streamName}`, { error });
throw error;
}
}
async getStreamStats(streamName) {
return {
messageCount: await this.getStreamDepth(streamName),
};
}
async getStreamDepth(streamName, options) {
try {
if (options?.multi) {
options.multi.xlen(streamName);
return 0;
}
const length = await this.streamClient.xlen(streamName);
return length;
}
catch (error) {
this.logger.error(`Error getting depth for ${streamName}`, { error });
throw error;
}
}
async getStreamDepths(streamNames) {
const multi = this.storeClient.multi();
const uniqueStreams = new Map(); // to store unique streams
// Add unique streams to the multi command
streamNames.forEach((profile) => {
if (!uniqueStreams.has(profile.stream)) {
uniqueStreams.set(profile.stream, -1); // initialize depth to -1 as a placeholder
this.getStreamDepth(profile.stream, { multi });
}
});
// Execute all commands
const streamDepthResults = (await multi.exec());
// Update the uniqueStreams map with the actual depths from multi.exec() results
Array.from(uniqueStreams.keys()).forEach((stream, idx) => {
uniqueStreams.set(stream, streamDepthResults[idx][1]);
});
// Map back to the original `streamNames` array with correct depths
const updatedNames = streamNames.map((profile) => {
return {
stream: profile.stream,
depth: uniqueStreams.get(profile.stream) || 0,
};
});
return updatedNames;
}
async trimStream(streamName, options) {
//no-op for now
return 0;
}
// Provider-specific helpers
getProviderSpecificFeatures() {
return {
supportsBatching: true,
supportsDeadLetterQueue: false,
supportsOrdering: true,
supportsTrimming: true,
supportsRetry: true,
supportsNotifications: false,
maxMessageSize: 512 * 1024 * 1024,
maxBatchSize: 1000,
};
}
}
exports.IORedisStreamService = IORedisStreamService;