firestore-queue
Version:
A powerful, scalable queue system built on Google Firestore with time-based indexing, auto-configuration, and connection reuse
969 lines • 43.1 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __exportStar = (this && this.__exportStar) || function(m, exports) {
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.FireQueue = void 0;
const admin = __importStar(require("firebase-admin"));
const events_1 = require("events");
const types_1 = require("../types");
/**
* Fire Queue - A powerful Firestore-based queue system
*
* Key features:
* - Time-based indexing for optimal performance
* - Updatable messages while in queue
* - Consumer progress tracking with timestamp-based resume
* - TTL support with automatic cleanup
* - Batch processing capabilities
* - Built-in monitoring and metrics
*/
class FireQueue extends events_1.EventEmitter {
constructor(config) {
super();
this.isInitialized = false;
this.consumers = new Map();
this.indexesCreated = new Set();
this.indexRetryCount = new Map();
// Hooks for extensibility
this.beforeEnqueueHooks = [];
this.afterEnqueueHooks = [];
this.beforeProcessHooks = [];
this.afterProcessHooks = [];
this.config = types_1.QueueConfigSchema.parse(config);
this.initializeFirestore();
}
/**
* Initialize Firestore connection
*/
initializeFirestore() {
try {
// Use pre-configured Firestore instance if provided
if (this.config.firestoreInstance) {
this.db = this.config.firestoreInstance;
return;
}
// Check if Firebase Admin is already initialized
if (admin.apps.length === 0) {
if (this.config.serviceAccountPath) {
const serviceAccount = require(this.config.serviceAccountPath);
admin.initializeApp({
credential: admin.credential.cert(serviceAccount),
projectId: this.config.projectId,
});
}
else {
// Use default credentials (works in GCP environment)
admin.initializeApp({
projectId: this.config.projectId,
});
}
}
// Create Firestore instance with specified database
if (this.config.databaseId !== '(default)') {
this.db = new admin.firestore.Firestore({
projectId: this.config.projectId,
databaseId: this.config.databaseId,
});
}
else {
this.db = admin.firestore();
}
}
catch (error) {
throw new Error(`Failed to initialize Firestore: ${error}`);
}
}
/**
* Initialize the queue system
*/
async initialize() {
if (this.isInitialized)
return;
try {
// Skip index creation during initialization
// Indexes will be created on-demand when needed
if (this.config.enableAutoCleanup) {
this.startCleanupTimer();
}
this.isInitialized = true;
console.log('🔥 Fire Queue initialized successfully');
}
catch (error) {
throw new Error(`Failed to initialize Fire Queue: ${error}`);
}
}
/**
* Create necessary Firestore indexes for optimal performance
*/
async createIndexes() {
// Check if index creation is enabled
if (!this.config.autoCreateIndexes) {
console.log('📋 Auto-index creation disabled. Required Firestore indexes:');
console.log(`Collection: ${this.config.queueCollection}`);
console.log('- Composite index: status ASC, priority ASC, updatedAt ASC');
console.log('- Composite index: status ASC, type ASC, updatedAt ASC');
console.log('- Composite index: status ASC, tags ARRAY, updatedAt ASC');
console.log('- Single field index: updatedAt ASC');
console.log('- Single field index: expiresAt ASC');
console.log('- TTL index: expiresAt (for automatic cleanup)');
console.log(`Collection: ${this.config.consumerCollection}`);
console.log('- Composite index: queueName ASC, isActive ASC, heartbeatAt ASC');
console.log('- Single field index: lastProcessedTimestamp ASC');
return;
}
console.log('🔨 Attempting to create Firestore indexes automatically...');
try {
// Import the Admin SDK for Firestore Admin operations
const { exec } = require('child_process');
const { promisify } = require('util');
const execAsync = promisify(exec);
// Create index definitions for the queue collection
const queueIndexes = [
{
fields: [
{ fieldPath: 'status', order: 'ASCENDING' },
{ fieldPath: 'priority', order: 'ASCENDING' },
{ fieldPath: 'updatedAt', order: 'ASCENDING' }
]
},
{
fields: [
{ fieldPath: 'status', order: 'ASCENDING' },
{ fieldPath: 'type', order: 'ASCENDING' },
{ fieldPath: 'updatedAt', order: 'ASCENDING' }
]
},
{
fields: [
{ fieldPath: 'status', order: 'ASCENDING' },
{ fieldPath: 'completedAt', order: 'ASCENDING' },
{ fieldPath: '__name__', order: 'ASCENDING' }
]
}
];
// Create index definitions for the consumer collection
const consumerIndexes = [
{
fields: [
{ fieldPath: 'queueName', order: 'ASCENDING' },
{ fieldPath: 'isActive', order: 'ASCENDING' },
{ fieldPath: 'heartbeatAt', order: 'ASCENDING' }
]
}
];
// Try to create indexes using gcloud CLI if available
const databaseFlag = this.config.databaseId !== '(default)' ? `--database=${this.config.databaseId}` : '';
for (const index of queueIndexes) {
const fieldConfigs = index.fields.map(f => `--field-config field-path=${f.fieldPath},order=${f.order}`).join(' ');
const cmd = `gcloud firestore indexes composite create --collection-group=${this.config.queueCollection} --query-scope=COLLECTION ${fieldConfigs} --project=${this.config.projectId} ${databaseFlag} --quiet 2>/dev/null || true`;
try {
await execAsync(cmd);
console.log(`✅ Created index for ${this.config.queueCollection}: ${index.fields.map(f => f.fieldPath).join(', ')}`);
}
catch (error) {
// Index might already exist or gcloud not available
}
}
for (const index of consumerIndexes) {
const fieldConfigs = index.fields.map(f => `--field-config field-path=${f.fieldPath},order=${f.order}`).join(' ');
const cmd = `gcloud firestore indexes composite create --collection-group=${this.config.consumerCollection} --query-scope=COLLECTION ${fieldConfigs} --project=${this.config.projectId} ${databaseFlag} --quiet 2>/dev/null || true`;
try {
await execAsync(cmd);
console.log(`✅ Created index for ${this.config.consumerCollection}: ${index.fields.map(f => f.fieldPath).join(', ')}`);
}
catch (error) {
// Index might already exist or gcloud not available
}
}
console.log('📋 Index creation attempted. Some indexes may take time to build.');
}
catch (error) {
console.log('⚠️ Could not auto-create indexes. Please create them manually:');
console.log(' Run: gcloud firestore indexes composite list --project=' + this.config.projectId);
console.log(' Or visit: https://console.firebase.google.com/project/' + this.config.projectId + '/firestore/indexes');
}
}
/**
* Create indexes for a specific topic/collection
* Useful when using topic-based queues
*/
async createTopicIndexes(topicName) {
if (!this.config.autoCreateIndexes) {
console.log(`📋 Auto-index creation disabled for topic: ${topicName}`);
return;
}
console.log(`🔨 Creating indexes for topic: ${topicName}...`);
try {
const { exec } = require('child_process');
const { promisify } = require('util');
const execAsync = promisify(exec);
const databaseFlag = this.config.databaseId !== '(default)' ? `--database=${this.config.databaseId}` : '';
// Essential indexes for queue processing and consumer operations
const topicIndexes = [
{
description: 'Cleanup index',
fields: [
{ fieldPath: 'status', order: 'ASCENDING' },
{ fieldPath: 'completedAt', order: 'ASCENDING' },
{ fieldPath: '__name__', order: 'ASCENDING' }
]
},
{
description: 'Consumer processing index',
fields: [
{ fieldPath: 'status', order: 'ASCENDING' },
{ fieldPath: 'updatedAt', order: 'ASCENDING' },
{ fieldPath: 'scheduledFor', order: 'ASCENDING' }
]
},
{
description: 'Complex consumer query index',
fields: [
{ fieldPath: 'status', order: 'ASCENDING' },
{ fieldPath: 'updatedAt', order: 'ASCENDING' },
{ fieldPath: 'priority', order: 'ASCENDING' },
{ fieldPath: 'scheduledFor', order: 'ASCENDING' },
{ fieldPath: '__name__', order: 'ASCENDING' }
]
}
];
for (const index of topicIndexes) {
const fieldConfigs = index.fields.map(f => `--field-config field-path=${f.fieldPath},order=${f.order}`).join(' ');
const cmd = `gcloud firestore indexes composite create --collection-group=${topicName} --query-scope=COLLECTION_GROUP ${fieldConfigs} --project=${this.config.projectId} ${databaseFlag} --quiet 2>/dev/null || true`;
try {
await execAsync(cmd);
console.log(`✅ Created index for topic ${topicName}: ${index.fields.map(f => f.fieldPath).join(', ')}`);
}
catch (error) {
// Index might already exist or gcloud not available, which is fine
}
}
}
catch (error) {
console.log(`⚠️ Failed to create indexes for topic ${topicName}. Create manually if needed.`);
}
}
/**
* Create indexes for topic consumers collection
* This ensures consumer tracking works efficiently
*/
async createConsumerIndexes(topicName) {
if (!this.config.autoCreateIndexes) {
return;
}
console.log(`🔨 Creating consumer indexes for topic: ${topicName}...`);
try {
const { exec } = require('child_process');
const { promisify } = require('util');
const execAsync = promisify(exec);
const databaseFlag = this.config.databaseId !== '(default)' ? `--database=${this.config.databaseId}` : '';
const consumerCollection = `${topicName}_consumers`;
// Consumer tracking indexes
const consumerIndexes = [
{
description: 'Active consumer lookup',
fields: [
{ fieldPath: 'isActive', order: 'ASCENDING' },
{ fieldPath: 'lastProcessedTimestamp', order: 'ASCENDING' }
]
},
{
description: 'Consumer health monitoring',
fields: [
{ fieldPath: 'isActive', order: 'ASCENDING' },
{ fieldPath: 'heartbeatAt', order: 'ASCENDING' }
]
}
];
for (const index of consumerIndexes) {
const fieldConfigs = index.fields.map(f => `--field-config field-path=${f.fieldPath},order=${f.order}`).join(' ');
const cmd = `gcloud firestore indexes composite create --collection-group=${consumerCollection} --query-scope=COLLECTION ${fieldConfigs} --project=${this.config.projectId} ${databaseFlag} --quiet 2>/dev/null || true`;
try {
await execAsync(cmd);
console.log(`✅ Created consumer index for ${consumerCollection}: ${index.fields.map(f => f.fieldPath).join(', ')}`);
}
catch (error) {
// Index might already exist, which is fine
}
}
}
catch (error) {
console.log(`⚠️ Failed to create consumer indexes for topic ${topicName}.`);
}
}
/**
* Enqueue a message to the queue
*/
async enqueue(message) {
if (!this.isInitialized) {
await this.initialize();
}
// Validate and transform message
let validatedMessage = types_1.QueueMessageSchema.parse(message);
// Apply before-enqueue hooks
for (const hook of this.beforeEnqueueHooks) {
validatedMessage = await hook(validatedMessage);
}
const now = admin.firestore.Timestamp.now();
const messageId = validatedMessage.id || this.generateMessageId();
// Calculate expiration time
const ttlSeconds = validatedMessage.ttlSeconds || this.config.defaultTtlSeconds;
const expiresAt = new admin.firestore.Timestamp(now.seconds + ttlSeconds, now.nanoseconds);
const queueDocument = {
...validatedMessage,
id: messageId,
status: 'pending',
createdAt: now,
updatedAt: now,
expiresAt,
scheduledFor: validatedMessage.scheduledFor
? validatedMessage.scheduledFor
: now.toDate(),
};
try {
await this.db
.collection(this.config.queueCollection)
.doc(messageId)
.set(queueDocument);
// Apply after-enqueue hooks
for (const hook of this.afterEnqueueHooks) {
await hook(queueDocument);
}
this.emit('message.enqueued', queueDocument);
// Trigger configured actions after successful enqueue
if (this.triggerConfig?.onEnqueue) {
try {
await this.triggerConfig.onEnqueue(messageId, queueDocument);
}
catch (triggerError) {
console.error('Failed to execute enqueue trigger:', triggerError);
// Don't fail the enqueue operation due to trigger failure
}
}
return messageId;
}
catch (error) {
throw new Error(`Failed to enqueue message: ${error}`);
}
}
/**
* Update an existing message in the queue
*/
async updateMessage(messageId, updates) {
if (!this.isInitialized) {
await this.initialize();
}
const docRef = this.db.collection(this.config.queueCollection).doc(messageId);
try {
await this.db.runTransaction(async (transaction) => {
const doc = await transaction.get(docRef);
if (!doc.exists) {
throw new Error(`Message ${messageId} not found`);
}
const currentData = doc.data();
if (currentData.status !== 'pending') {
throw new Error(`Cannot update message ${messageId} with status ${currentData.status}`);
}
const updatedData = {
...updates,
version: (currentData.version || 1) + 1,
updatedAt: admin.firestore.Timestamp.now(),
};
transaction.update(docRef, updatedData);
this.emit('message.updated', currentData, { ...currentData, ...updatedData });
});
}
catch (error) {
throw new Error(`Failed to update message ${messageId}: ${error}`);
}
}
/**
* Start consuming messages with a given consumer ID
*/
async consume(consumerId, handler) {
if (!this.isInitialized) {
await this.initialize();
}
if (this.consumers.has(consumerId)) {
throw new Error(`Consumer ${consumerId} is already active`);
}
this.consumers.set(consumerId, { active: true, handler });
// Initialize consumer state
await this.initializeConsumerState(consumerId);
this.emit('consumer.started', consumerId);
// Start processing loop
this.startConsumerLoop(consumerId);
}
/**
* Stop consuming messages for a given consumer
*/
async stopConsumer(consumerId) {
const consumer = this.consumers.get(consumerId);
if (consumer) {
consumer.active = false;
this.consumers.delete(consumerId);
// Update consumer state
await this.updateConsumerState(consumerId, { isActive: false });
this.emit('consumer.stopped', consumerId);
}
}
/**
* Initialize consumer state tracking
*/
async initializeConsumerState(consumerId) {
const consumerDoc = this.db
.collection(this.config.consumerCollection)
.doc(consumerId);
const doc = await consumerDoc.get();
const now = admin.firestore.Timestamp.now();
if (!doc.exists) {
// Create new consumer state
const consumerConfig = this.config.consumers[consumerId] || {};
const initialState = {
consumerId,
queueName: this.config.queueCollection,
lastProcessedTimestamp: new admin.firestore.Timestamp(0, 0), // Start from epoch to process existing messages
isActive: true,
heartbeatAt: now,
processedCount: 0,
errorCount: 0,
configuration: {
batchSize: consumerConfig.batchSize || this.config.defaultBatchSize,
pollIntervalMs: consumerConfig.pollIntervalMs || this.config.defaultPollIntervalMs,
lockTimeoutMs: 300000, // 5 minutes
maxConcurrency: consumerConfig.maxConcurrency || 1,
},
};
await consumerDoc.set(initialState);
}
else {
// Update existing consumer state
await consumerDoc.update({
isActive: true,
heartbeatAt: now,
});
}
}
/**
* Update consumer state
*/
async updateConsumerState(consumerId, updates) {
await this.db
.collection(this.config.consumerCollection)
.doc(consumerId)
.update({
...updates,
heartbeatAt: admin.firestore.Timestamp.now(),
});
}
/**
* Main consumer processing loop
*/
async startConsumerLoop(consumerId) {
const consumer = this.consumers.get(consumerId);
if (!consumer)
return;
const consumerConfig = this.config.consumers[consumerId] || {};
const pollInterval = consumerConfig.pollIntervalMs || this.config.defaultPollIntervalMs;
while (consumer.active) {
try {
await this.processBatch(consumerId);
await this.sleep(pollInterval);
}
catch (error) {
console.error(`Consumer ${consumerId} error:`, error);
const errorMessage = error instanceof Error ? error.message : String(error);
this.emit('consumer.error', consumerId, errorMessage);
// Update error count
await this.updateConsumerState(consumerId, {
errorCount: admin.firestore.FieldValue.increment(1),
lastError: errorMessage,
});
// Back off on error
await this.sleep(Math.min(pollInterval * 2, 30000));
}
}
}
/**
* Process a batch of messages for a consumer
*/
async processBatch(consumerId) {
const consumer = this.consumers.get(consumerId);
if (!consumer || !consumer.active)
return;
// Get consumer state
const consumerDoc = await this.db
.collection(this.config.consumerCollection)
.doc(consumerId)
.get();
if (!consumerDoc.exists)
return;
const consumerState = consumerDoc.data();
const config = consumerState.configuration;
const consumerConfigOverride = this.config.consumers[consumerId] || {};
// Get messages to process
const messages = await this.getMessagesForProcessing(consumerId, consumerState.lastProcessedTimestamp, config.batchSize, consumerConfigOverride);
if (messages.length === 0)
return;
// Apply before-process hooks
let processableMessages = messages;
for (const hook of this.beforeProcessHooks) {
processableMessages = await hook(processableMessages);
}
// READ-ONLY: No locking needed for read-only consumers
// Multiple consumers can process the same messages simultaneously
const result = {
processed: messages.length,
successful: 0,
failed: 0,
errors: [],
};
try {
// Prepare message objects for handler (READ-ONLY)
const handlerMessages = messages.map(msg => ({
id: msg.id,
type: msg.type,
payload: msg.payload,
metadata: msg.metadata,
// READ-ONLY: ack/nack methods are no-ops since consumers only read
ack: () => Promise.resolve(), // No-op for read-only
nack: (error) => Promise.resolve(), // No-op for read-only
updatePayload: (newPayload) => Promise.resolve(), // No-op for read-only
}));
// Call user handler
await consumer.handler(handlerMessages);
result.successful = messages.length;
// Update consumer progress (READ-ONLY: track last read position)
const lastMessage = messages[messages.length - 1];
await this.updateConsumerState(consumerId, {
lastProcessedTimestamp: lastMessage.updatedAt,
processedCount: admin.firestore.FieldValue.increment(messages.length),
});
console.log(`📖 Consumer ${consumerId} read ${messages.length} messages`);
}
catch (error) {
console.error(`Read-only processing failed for consumer ${consumerId}:`, error);
// READ-ONLY: Just log errors, don't modify message state
const errorMessage = error instanceof Error ? error.message : String(error);
result.errors.push({
messageId: 'batch-error',
error: errorMessage,
});
result.failed = messages.length;
}
finally {
// READ-ONLY: No locks to release
console.log(`🔓 Read-only processing completed for consumer ${consumerId}`);
}
// Apply after-process hooks
for (const hook of this.afterProcessHooks) {
await hook(result);
}
this.emit('batch.processed', consumerId, result);
}
/**
* Get messages ready for processing by a consumer (Kafka-style: multiple consumers can process same messages)
*/
async getMessagesForProcessing(consumerId, lastProcessedTimestamp, batchSize, consumerConfig) {
const now = admin.firestore.Timestamp.now();
// KAFKA-STYLE: Remove status filter - allow all messages regardless of processing state
// Each consumer tracks its own progress independently
let query = this.db
.collection(this.config.queueCollection)
.where('updatedAt', '>', lastProcessedTimestamp)
.where('scheduledFor', '<=', now)
.orderBy('updatedAt')
.orderBy('priority')
.limit(batchSize);
// Apply consumer-specific filters
if (consumerConfig.messageTypes && consumerConfig.messageTypes.length > 0) {
query = query.where('type', 'in', consumerConfig.messageTypes);
}
if (consumerConfig.tags && consumerConfig.tags.length > 0) {
query = query.where('tags', 'array-contains-any', consumerConfig.tags);
}
try {
const snapshot = await query.get();
return snapshot.docs.map(doc => doc.data());
}
catch (error) {
// Check if error is due to missing index
if (error?.code === 9 && error?.message?.includes('index')) {
const retryKey = `queue-${this.config.queueCollection}`;
const currentRetries = this.indexRetryCount.get(retryKey) || 0;
console.log(`⚠️ Missing index detected for queue: ${this.config.queueCollection} (attempt ${currentRetries + 1}/3)`);
// Attempt to create indexes if not exceeded retry limit
if (currentRetries < 3 && !this.indexesCreated.has(this.config.queueCollection)) {
console.log('🔨 Attempting to create indexes...');
this.indexRetryCount.set(retryKey, currentRetries + 1);
try {
await this.createIndexes();
this.indexesCreated.add(this.config.queueCollection);
// Retry the query
const snapshot = await query.get();
return snapshot.docs.map(doc => doc.data());
}
catch (retryError) {
console.error(`❌ Query failed after index creation attempt ${currentRetries + 1}:`, retryError);
// If this was the last retry, throw the error
if (currentRetries >= 2) {
console.error(`❌ Maximum retries (3) reached for queue ${this.config.queueCollection}`);
throw retryError;
}
// For index-related errors, continue retrying; for other errors, throw immediately
if (retryError?.code === 9 && retryError?.message?.includes('index')) {
throw new Error(`Index creation failed after ${currentRetries + 1} attempts. Will retry on next operation.`);
}
else {
throw retryError;
}
}
}
else if (currentRetries >= 3) {
console.error(`❌ Maximum retries (3) exceeded for queue ${this.config.queueCollection}`);
throw new Error(`Index creation failed after 3 attempts for queue ${this.config.queueCollection}`);
}
throw error;
}
throw error;
}
}
/**
* Lock messages for processing (Kafka-style: per-consumer locking without global status change)
*/
async lockMessages(messages, consumerId, lockTimeoutMs) {
const batch = this.db.batch();
const lockUntil = new admin.firestore.Timestamp(Math.floor((Date.now() + lockTimeoutMs) / 1000), 0);
for (const message of messages) {
// Validate message ID to prevent invalid document path errors
if (!message.id || message.id.trim() === '') {
console.error('FireQueue: Invalid message ID detected:', message.id, 'Message:', message);
continue; // Skip this message
}
const docRef = this.db.collection(this.config.queueCollection).doc(message.id);
// KAFKA-STYLE: Don't change global status, use per-consumer locking
// Multiple consumers can lock the same message simultaneously
const consumerLockField = `consumerLocks.${consumerId}`;
batch.update(docRef, {
[consumerLockField]: {
lockedAt: admin.firestore.Timestamp.now(),
lockedUntil: lockUntil,
processingStartedAt: admin.firestore.Timestamp.now(),
}
});
this.emit('message.processing', message, consumerId);
}
await batch.commit();
}
/**
* Unlock messages (remove per-consumer processing lock)
*/
async unlockMessages(messages, consumerId) {
const batch = this.db.batch();
for (const message of messages) {
// Validate message ID to prevent invalid document path errors
if (!message.id || message.id.trim() === '') {
console.error('FireQueue: Invalid message ID detected in unlock:', message.id, 'Message:', message);
continue; // Skip this message
}
const docRef = this.db.collection(this.config.queueCollection).doc(message.id);
// KAFKA-STYLE: Remove only this consumer's lock, leave others intact
const consumerLockField = `consumerLocks.${consumerId}`;
batch.update(docRef, {
[consumerLockField]: admin.firestore.FieldValue.delete(),
});
}
await batch.commit();
}
/**
* Acknowledge a message (mark as completed by specific consumer)
*/
async ackMessage(messageId, consumerId) {
// KAFKA-STYLE: Track per-consumer completion, don't change global status
const consumerCompletionField = `consumerCompletions.${consumerId}`;
const consumerLockField = `consumerLocks.${consumerId}`;
await this.db
.collection(this.config.queueCollection)
.doc(messageId)
.update({
[consumerCompletionField]: {
completedAt: admin.firestore.Timestamp.now(),
processingTimeMs: Date.now() - (Date.now() - 1000), // Approximate
},
[consumerLockField]: admin.firestore.FieldValue.delete(), // Remove lock
});
// Emit event (we'll need to get the message data for the event)
const doc = await this.db.collection(this.config.queueCollection).doc(messageId).get();
if (doc.exists) {
this.emit('message.completed', doc.data(), consumerId);
}
}
/**
* Negative acknowledge a message (mark as failed or retry for specific consumer)
*/
async nackMessage(messageId, error, consumerId) {
const docRef = this.db.collection(this.config.queueCollection).doc(messageId);
await this.db.runTransaction(async (transaction) => {
const doc = await transaction.get(docRef);
if (!doc.exists)
return;
const data = doc.data();
if (consumerId) {
// KAFKA-STYLE: Track per-consumer failures
const consumerFailureField = `consumerFailures.${consumerId}`;
const consumerLockField = `consumerLocks.${consumerId}`;
transaction.update(docRef, {
[consumerFailureField]: {
failedAt: admin.firestore.Timestamp.now(),
error: error || 'Processing failed',
retryCount: (data.retryCount || 0) + 1,
},
[consumerLockField]: admin.firestore.FieldValue.delete(), // Remove lock
});
this.emit('message.failed', data, error || 'Processing failed', consumerId);
}
else {
// Fallback: Legacy behavior for global retries
const newRetryCount = data.retryCount + 1;
if (newRetryCount <= data.maxRetries && data.retryable) {
// Retry with exponential backoff
const backoffSeconds = Math.pow(2, newRetryCount) * 60; // 2min, 4min, 8min...
const scheduledFor = new admin.firestore.Timestamp(Math.floor(Date.now() / 1000 + backoffSeconds), 0);
transaction.update(docRef, {
retryCount: newRetryCount,
scheduledFor,
error,
updatedAt: admin.firestore.Timestamp.now(),
});
}
else {
// Mark as globally failed
transaction.update(docRef, {
status: 'failed',
error,
completedAt: admin.firestore.Timestamp.now(),
});
this.emit('message.failed', data, error || 'Max retries exceeded', '');
}
}
});
}
/**
* Get queue metrics and statistics
*/
async getMetrics() {
const [totalSnapshot, pendingSnapshot, processingSnapshot, completedSnapshot, failedSnapshot, expiredSnapshot, consumersSnapshot] = await Promise.all([
this.db.collection(this.config.queueCollection).count().get(),
this.db.collection(this.config.queueCollection).where('status', '==', 'pending').count().get(),
this.db.collection(this.config.queueCollection).where('status', '==', 'processing').count().get(),
this.db.collection(this.config.queueCollection).where('status', '==', 'completed').count().get(),
this.db.collection(this.config.queueCollection).where('status', '==', 'failed').count().get(),
this.db.collection(this.config.queueCollection).where('status', '==', 'expired').count().get(),
this.db.collection(this.config.consumerCollection).get()
]);
// Get message type and priority distributions
const messagesSnapshot = await this.db
.collection(this.config.queueCollection)
.select('type', 'priority', 'createdAt', 'completedAt')
.limit(1000)
.get();
const messagesByType = {};
const messagesByPriority = {};
let totalProcessingTime = 0;
let processedCount = 0;
messagesSnapshot.docs.forEach(doc => {
const data = doc.data();
// Count by type
messagesByType[data.type] = (messagesByType[data.type] || 0) + 1;
// Count by priority
messagesByPriority[data.priority] = (messagesByPriority[data.priority] || 0) + 1;
// Calculate processing time
if (data.completedAt && data.createdAt) {
totalProcessingTime += data.completedAt.toMillis() - data.createdAt.toMillis();
processedCount++;
}
});
const consumerHealth = consumersSnapshot.docs.map(doc => {
const data = doc.data();
return {
consumerId: data.consumerId,
isActive: data.isActive,
lastHeartbeat: data.heartbeatAt.toDate(),
processedCount: data.processedCount,
errorCount: data.errorCount,
};
});
return {
totalMessages: totalSnapshot.data().count,
pendingMessages: pendingSnapshot.data().count,
processingMessages: processingSnapshot.data().count,
completedMessages: completedSnapshot.data().count,
failedMessages: failedSnapshot.data().count,
expiredMessages: expiredSnapshot.data().count,
averageProcessingTimeMs: processedCount > 0 ? totalProcessingTime / processedCount : 0,
messagesByType,
messagesByPriority,
consumerHealth,
};
}
/**
* Clean up expired and old completed messages
*/
async cleanup() {
const now = admin.firestore.Timestamp.now();
const retainUntil = new admin.firestore.Timestamp(now.seconds - this.config.retainCompletedSeconds, now.nanoseconds);
try {
// Clean up expired messages
const expiredQuery = this.db
.collection(this.config.queueCollection)
.where('expiresAt', '<=', now)
.limit(500);
const expiredSnapshot = await expiredQuery.get();
const expiredBatch = this.db.batch();
expiredSnapshot.docs.forEach(doc => {
expiredBatch.update(doc.ref, { status: 'expired' });
this.emit('message.expired', doc.data());
});
if (!expiredSnapshot.empty) {
await expiredBatch.commit();
}
// Clean up old completed messages
const completedQuery = this.db
.collection(this.config.queueCollection)
.where('status', '==', 'completed')
.where('completedAt', '<=', retainUntil)
.limit(500);
const completedSnapshot = await completedQuery.get();
const completedBatch = this.db.batch();
completedSnapshot.docs.forEach(doc => {
completedBatch.delete(doc.ref);
});
if (!completedSnapshot.empty) {
await completedBatch.commit();
}
return {
expired: expiredSnapshot.size,
completed: completedSnapshot.size,
};
}
catch (error) {
// Check if error is due to missing index
if (error?.code === 9 && error?.message?.includes('index')) {
console.log('⚠️ Missing index detected during cleanup for:', this.config.queueCollection);
// Create cleanup index if not already attempted
if (!this.indexesCreated.has(`${this.config.queueCollection}_cleanup`)) {
console.log('🔨 Creating cleanup indexes...');
await this.createIndexes();
this.indexesCreated.add(`${this.config.queueCollection}_cleanup`);
// Retry cleanup
return this.cleanup();
}
}
throw error;
}
}
/**
* Start automatic cleanup timer
*/
startCleanupTimer() {
this.cleanupTimer = setInterval(async () => {
try {
const result = await this.cleanup();
if (result.expired > 0 || result.completed > 0) {
console.log(`🧹 Cleaned up ${result.expired} expired and ${result.completed} completed messages`);
}
}
catch (error) {
console.error('Cleanup error:', error);
}
}, this.config.cleanupIntervalMs);
}
/**
* Set trigger configuration for the queue
*/
setTriggerConfig(config) {
this.triggerConfig = config;
}
/**
* Add hooks for extensibility
*/
addBeforeEnqueueHook(hook) {
this.beforeEnqueueHooks.push(hook);
}
addAfterEnqueueHook(hook) {
this.afterEnqueueHooks.push(hook);
}
addBeforeProcessHook(hook) {
this.beforeProcessHooks.push(hook);
}
addAfterProcessHook(hook) {
this.afterProcessHooks.push(hook);
}
/**
* Shutdown the queue system gracefully
*/
async shutdown() {
console.log('🔥 Shutting down Fire Queue...');
// Stop all consumers
const consumerIds = Array.from(this.consumers.keys());
await Promise.all(consumerIds.map(id => this.stopConsumer(id)));
// Clear cleanup timer
if (this.cleanupTimer) {
clearInterval(this.cleanupTimer);
this.cleanupTimer = undefined;
}
// Final cleanup
if (this.config.enableAutoCleanup) {
await this.cleanup();
}
this.isInitialized = false;
console.log('🔥 Fire Queue shutdown complete');
}
/**
* Utility methods
*/
generateMessageId() {
return `msg_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
}
exports.FireQueue = FireQueue;
// Re-export types for convenience
__exportStar(require("../types"), exports);
//# sourceMappingURL=FireQueue.js.map