firestore-queue
Version:
A powerful, scalable queue system built on Google Firestore with time-based indexing, auto-configuration, and connection reuse
314 lines • 13.6 kB
JavaScript
;
/**
* Advanced usage example for Fire Queue
* Demonstrates configuration-driven setup, multiple consumers, and hooks
*/
Object.defineProperty(exports, "__esModule", { value: true });
exports.advancedExample = advancedExample;
const index_1 = require("../index");
async function advancedExample() {
console.log('🚀 Fire Queue Advanced Usage Example');
console.log('====================================\n');
// 1. Advanced configuration
const config = {
projectId: 'your-project-id',
databaseId: 'firequeue',
queueCollection: 'advanced_queue',
consumerCollection: 'advanced_queue_consumers',
// Global settings
defaultTtlSeconds: 7200, // 2 hours
defaultBatchSize: 25,
defaultPollIntervalMs: 2000, // 2 seconds
// Cleanup settings
enableAutoCleanup: true,
cleanupIntervalMs: 180000, // 3 minutes
retainCompletedSeconds: 3600, // 1 hour
// Consumer configurations
consumers: {
'image-processor': {
enabled: true,
batchSize: 5, // Smaller batches for heavy processing
maxConcurrency: 3,
messageTypes: ['image_resize', 'image_filter'],
pollIntervalMs: 5000, // 5 seconds
},
'notification-sender': {
enabled: true,
batchSize: 50, // Large batches for notifications
maxConcurrency: 2,
messageTypes: ['email', 'sms', 'push'],
tags: ['user-facing'],
},
'data-analytics': {
enabled: true,
batchSize: 100, // Very large batches for analytics
maxConcurrency: 1,
messageTypes: ['analytics'],
pollIntervalMs: 10000, // 10 seconds
},
},
// Required properties
enableTimeBasedSharding: false,
shardIntervalHours: 24,
};
const queue = new index_1.FireQueue(config);
try {
// 2. Add hooks for logging and validation
queue.addBeforeEnqueueHook(async (message) => {
console.log(`🔍 Before enqueue: ${message.type} (priority: ${message.priority})`);
// Add timestamp to all messages
message.metadata = {
...message.metadata,
enqueuedAt: new Date().toISOString(),
source: 'advanced-example',
};
return message;
});
queue.addAfterEnqueueHook(async (document) => {
console.log(`✅ After enqueue: ${document.id} scheduled for ${document.scheduledFor}`);
});
queue.addBeforeProcessHook(async (messages) => {
console.log(`⚡ Processing batch of ${messages.length} messages`);
return messages;
});
queue.addAfterProcessHook(async (result) => {
console.log(`📊 Batch complete: ${result.successful}/${result.processed} successful`);
if (result.failed > 0) {
console.log(` Errors: ${result.errors.map(e => e.error).join(', ')}`);
}
});
// 3. Initialize and set up event monitoring
await queue.initialize();
// Monitor queue events
queue.on('message.enqueued', (message) => {
console.log(`📥 Enqueued: ${message.id} (${message.type})`);
});
queue.on('message.failed', (message, error, consumerId) => {
console.log(`💥 Failed: ${message.id} by ${consumerId} - ${error}`);
});
queue.on('consumer.started', (consumerId) => {
console.log(`🟢 Consumer started: ${consumerId}`);
});
// 4. Enqueue various types of messages
console.log('📝 Enqueuing diverse message types...\n');
// Image processing messages
await Promise.all([
queue.enqueue({
type: 'image_resize',
payload: {
imageId: 'img_001',
originalUrl: 'https://example.com/image1.jpg',
targetSizes: [100, 200, 400],
},
priority: 2,
retryable: true,
maxRetries: 3,
retryCount: 0,
version: 1,
tags: ['batch-process'],
metadata: {},
ttlSeconds: 3600, // 1 hour for processing
}),
queue.enqueue({
type: 'image_filter',
payload: {
imageId: 'img_002',
originalUrl: 'https://example.com/image2.jpg',
filters: ['sepia', 'blur'],
},
priority: 3,
tags: ['batch-process'],
}),
]);
// Notification messages
await Promise.all([
queue.enqueue({
type: 'email',
payload: {
to: 'user1@example.com',
template: 'welcome',
variables: { name: 'John', discount: '20%' },
},
priority: 1,
tags: ['user-facing', 'welcome-flow'],
}),
queue.enqueue({
type: 'push',
payload: {
userId: 'user_123',
title: 'New message',
body: 'You have a new message from a friend',
data: { messageId: 'msg_456' },
},
priority: 1,
tags: ['user-facing', 'realtime'],
}),
]);
// Analytics messages (delayed processing)
await Promise.all([
queue.enqueue({
type: 'analytics',
payload: {
event: 'page_view',
userId: 'user_789',
page: '/dashboard',
timestamp: Date.now(),
},
priority: 5,
scheduledFor: new Date(Date.now() + 30000), // Process in 30 seconds
tags: ['analytics'],
}),
queue.enqueue({
type: 'analytics',
payload: {
event: 'button_click',
userId: 'user_789',
element: 'signup-button',
timestamp: Date.now(),
},
priority: 5,
scheduledFor: new Date(Date.now() + 60000), // Process in 1 minute
tags: ['analytics'],
}),
]);
// 5. Start specialized consumers
console.log('👥 Starting specialized consumers...\n');
// Image processor - handles heavy computational tasks
await queue.consume('image-processor', async (messages) => {
console.log(`🖼️ Image processor handling ${messages.length} messages`);
for (const message of messages) {
try {
const { imageId, targetSizes, filters } = message.payload;
if (message.type === 'image_resize') {
console.log(` Resizing image ${imageId} to sizes: ${targetSizes?.join(', ')}`);
// Simulate heavy processing
await new Promise(resolve => setTimeout(resolve, 1000));
}
else if (message.type === 'image_filter') {
console.log(` Applying filters to image ${imageId}: ${filters?.join(', ')}`);
await new Promise(resolve => setTimeout(resolve, 800));
}
await message.ack();
console.log(` ✅ Image processing completed: ${imageId}`);
}
catch (error) {
console.log(` ❌ Image processing failed: ${error}`);
await message.nack(error instanceof Error ? error.message : String(error));
}
}
});
// Notification sender - handles user-facing communications
await queue.consume('notification-sender', async (messages) => {
console.log(`📢 Notification sender handling ${messages.length} messages`);
for (const message of messages) {
try {
if (message.type === 'email') {
const { to, template } = message.payload;
console.log(` Sending email to ${to} using template ${template}`);
await new Promise(resolve => setTimeout(resolve, 100));
}
else if (message.type === 'push') {
const { userId, title } = message.payload;
console.log(` Sending push notification to ${userId}: ${title}`);
await new Promise(resolve => setTimeout(resolve, 50));
}
await message.ack();
console.log(` ✅ Notification sent: ${message.type}`);
}
catch (error) {
console.log(` ❌ Notification failed: ${error}`);
await message.nack(error instanceof Error ? error.message : String(error));
}
}
});
// Analytics processor - handles batch analytics
await queue.consume('data-analytics', async (messages) => {
console.log(`📊 Analytics processor handling ${messages.length} messages`);
// Group by event type for batch processing
const eventGroups = messages.reduce((groups, message) => {
const eventType = message.payload.event;
if (!groups[eventType])
groups[eventType] = [];
groups[eventType].push(message);
return groups;
}, {});
for (const [eventType, eventMessages] of Object.entries(eventGroups)) {
try {
console.log(` Processing ${eventMessages.length} ${eventType} events`);
// Simulate batch analytics processing
await new Promise(resolve => setTimeout(resolve, 200));
// Acknowledge all messages in the group
await Promise.all(eventMessages.map(msg => msg.ack()));
console.log(` ✅ Analytics batch completed: ${eventType}`);
}
catch (error) {
console.log(` ❌ Analytics batch failed: ${error}`);
await Promise.all(eventMessages.map(msg => msg.nack(error instanceof Error ? error.message : String(error))));
}
}
});
// 6. Demonstrate message updates
console.log('\n🔄 Demonstrating message updates...');
const updateableMessageId = await queue.enqueue({
type: 'data_export',
payload: {
exportId: 'export_001',
status: 'pending',
progress: 0,
},
priority: 3,
});
// Simulate progress updates
for (let i = 25; i <= 100; i += 25) {
await new Promise(resolve => setTimeout(resolve, 1000));
await queue.updateMessage(updateableMessageId, {
payload: {
exportId: 'export_001',
status: i === 100 ? 'completed' : 'processing',
progress: i,
},
});
console.log(` Updated export progress: ${i}%`);
}
// 7. Let the system run
console.log('\n⏳ Running advanced processing for 3 minutes...');
await new Promise(resolve => setTimeout(resolve, 180000)); // 3 minutes
// 8. Show comprehensive metrics
console.log('\n📊 Final Advanced Metrics:');
const metrics = await queue.getMetrics();
console.log(`Total Messages: ${metrics.totalMessages}`);
console.log(`Pending: ${metrics.pendingMessages}`);
console.log(`Processing: ${metrics.processingMessages}`);
console.log(`Completed: ${metrics.completedMessages}`);
console.log(`Failed: ${metrics.failedMessages}`);
console.log(`Average Processing Time: ${metrics.averageProcessingTimeMs.toFixed(0)}ms`);
console.log('\nBy Message Type:');
Object.entries(metrics.messagesByType).forEach(([type, count]) => {
console.log(` ${type}: ${count}`);
});
console.log('\nBy Priority:');
Object.entries(metrics.messagesByPriority).forEach(([priority, count]) => {
console.log(` Priority ${priority}: ${count}`);
});
console.log('\nConsumer Health:');
metrics.consumerHealth.forEach(consumer => {
const status = consumer.isActive ? '🟢' : '🔴';
console.log(` ${status} ${consumer.consumerId}: ${consumer.processedCount} processed, ${consumer.errorCount} errors`);
});
}
catch (error) {
console.error('❌ Advanced example error:', error);
}
finally {
// 9. Graceful shutdown
console.log('\n🧹 Performing graceful shutdown...');
await queue.shutdown();
console.log('👋 Advanced example completed');
}
}
// Run the example
if (require.main === module) {
advancedExample().catch(console.error);
}
//# sourceMappingURL=advanced-usage.js.map