firestore-queue
Version:
A powerful, scalable queue system built on Google Firestore with time-based indexing, auto-configuration, and connection reuse
261 lines • 11.5 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.createSimpleQueue = createSimpleQueue;
exports.quickStart = quickStart;
exports.createReadyQueue = createReadyQueue;
exports.createQueueWithConsumer = createQueueWithConsumer;
const FireQueue_1 = require("../core/FireQueue");
const FirestoreWriter_1 = require("../writers/FirestoreWriter");
const database_setup_1 = require("./database-setup");
const service_account_helper_1 = require("./service-account-helper");
const pubsub_1 = require("@google-cloud/pubsub");
/**
* Trigger processing for a specific topic via Pub/Sub
* This enables automatic processing when new messages are enqueued
* Uses configurable trigger mapping instead of hardcoded topics
*/
async function triggerProcessingForTopic(topicName, reason, triggerMapping) {
if (!triggerMapping || Object.keys(triggerMapping).length === 0) {
console.log(`No trigger mapping provided for topic: ${topicName}`);
return;
}
try {
const triggerTopicName = triggerMapping[topicName];
if (!triggerTopicName) {
console.log(`No trigger configured for topic: ${topicName}`);
return;
}
const pubsub = new pubsub_1.PubSub();
const topic = pubsub.topic(triggerTopicName);
const messageData = {
timestamp: new Date().toISOString(),
reason,
topicName,
source: 'fire-queue-enqueue'
};
const messageBuffer = Buffer.from(JSON.stringify(messageData));
console.log(`📨 Triggering processing for ${topicName}: ${reason}`);
const messageId = await topic.publishMessage({ data: messageBuffer });
console.log(`✅ Processing triggered with message ID: ${messageId}`);
}
catch (error) {
console.warn(`Failed to trigger processing for ${topicName}:`, error);
// Don't throw - we don't want to fail the enqueue operation
}
}
/**
* Create a Fire Queue with minimal configuration
* Perfect for quick setup and testing
*/
function createSimpleQueue(config) {
// Auto-load Firebase configuration if needed
let resolvedConfig = config;
if (!config.firestoreInstance) {
try {
const autoConfig = (0, service_account_helper_1.autoLoadFirebaseConfig)({
projectId: config.projectId,
serviceAccountPath: config.serviceAccountPath,
});
resolvedConfig = {
...config,
projectId: config.projectId || autoConfig.projectId,
serviceAccountPath: config.serviceAccountPath || autoConfig.serviceAccountPath,
};
}
catch (error) {
if (!config.projectId) {
throw new Error('Either provide projectId, serviceAccountPath, or firestoreInstance');
}
// If auto-config fails but we have projectId, continue with what we have
}
}
// Build full configuration from simple config
const fullConfig = {
projectId: resolvedConfig.projectId,
databaseId: resolvedConfig.dbId || 'firequeue', // Default to dedicated "firequeue" database
serviceAccountPath: resolvedConfig.serviceAccountPath,
firestoreInstance: resolvedConfig.firestoreInstance,
queueCollection: resolvedConfig.topic,
consumerCollection: `${resolvedConfig.topic}_consumers`,
// Sensible defaults
defaultTtlSeconds: 3600, // 1 hour
defaultBatchSize: 50,
defaultPollIntervalMs: 5000, // 5 seconds
enableAutoCleanup: true,
cleanupIntervalMs: 300000, // 5 minutes
retainCompletedSeconds: 86400, // 24 hours
enableTimeBasedSharding: false,
shardIntervalHours: 24,
autoCreateIndexes: true, // Enable automatic index creation
// Empty consumers config (can be added later)
consumers: {},
};
// Create queue instance
const queue = new FireQueue_1.FireQueue(fullConfig);
// Create writer instance
const writer = new FirestoreWriter_1.FirestoreWriter({
projectId: resolvedConfig.projectId,
queueName: resolvedConfig.topic,
databaseId: resolvedConfig.dbId,
serviceAccountPath: resolvedConfig.serviceAccountPath,
firestoreInstance: resolvedConfig.firestoreInstance,
validatePayload: true,
enableBatching: true,
batchSize: 50,
batchTimeoutMs: 5000,
enableMetrics: true,
});
return { queue, writer, config: fullConfig };
}
/**
* One-liner setup for the simplest use case
*/
function quickStart(projectId, serviceAccountPath, topic, dbId) {
const { queue, writer } = createSimpleQueue({
projectId,
serviceAccountPath,
topic,
dbId,
});
return { queue, writer };
}
/**
* Setup with automatic initialization
*/
async function createReadyQueue(config) {
// Auto-resolve configuration first
const autoConfig = !config.firestoreInstance ?
(0, service_account_helper_1.autoLoadFirebaseConfig)({
projectId: config.projectId,
serviceAccountPath: config.serviceAccountPath,
}) : null;
const resolvedProjectId = config.projectId || autoConfig?.projectId;
const resolvedServiceAccountPath = config.serviceAccountPath || autoConfig?.serviceAccountPath;
// Check and create database if needed (only if using service account path)
if (!config.firestoreInstance && resolvedServiceAccountPath) {
const databaseId = config.dbId || 'firequeue';
await (0, database_setup_1.checkAndGuideSetup)({
projectId: resolvedProjectId,
serviceAccountPath: resolvedServiceAccountPath
});
}
const { queue, writer } = createSimpleQueue(config);
// Initialize the queue
await queue.initialize();
// Skip automatic index creation - indexes will be created only when needed
// (when queries fail due to missing indexes)
// Return convenient wrapper functions
return {
queue,
writer,
// Simple enqueue function
enqueue: async (type, payload, options = {}) => {
const result = await writer.write({
type,
payload,
priority: options.priority || 5,
tags: options.tags || [],
ttlSeconds: options.ttlSeconds,
scheduledFor: options.scheduledFor,
metadata: options.metadata || {},
retryable: options.retryable !== false,
maxRetries: options.maxRetries || 3,
retryCount: 0,
version: 1,
});
if (!result.success) {
throw new Error(result.error || 'Failed to enqueue message');
}
// Trigger processing for this topic if enabled
if (options.autoTrigger !== false) {
try {
await triggerProcessingForTopic(config.topic, 'new message enqueued', config.triggerMapping);
}
catch (triggerError) {
console.warn('Failed to trigger processing, message still enqueued:', triggerError);
}
}
return result.messageId || 'unknown';
},
// Simple consume function with smart index creation (max 3 retries)
consume: async (consumerId, handler) => {
const retryKey = `topic-${config.topic}`;
let retryCount = 0;
const maxRetries = 3;
const attemptConsume = async () => {
try {
return await queue.consume(consumerId, handler);
}
catch (error) {
// Check if error is due to missing index
if (error?.code === 9 && error?.message?.includes('index')) {
retryCount++;
console.log(`⚠️ Missing topic index detected for topic: ${config.topic} (attempt ${retryCount}/${maxRetries})`);
// Attempt to create topic indexes if auto-create is enabled and retries available
if (retryCount <= maxRetries && queue['config'].autoCreateIndexes && !queue['indexesCreated']?.has(retryKey)) {
console.log('🔨 Attempting to create topic indexes...');
try {
await queue.createTopicIndexes(config.topic);
await queue.createConsumerIndexes(config.topic);
queue['indexesCreated']?.add(retryKey);
console.log(`✅ Topic indexes created for ${config.topic}, retrying consume operation...`);
// Retry the consume operation
return await queue.consume(consumerId, handler);
}
catch (indexError) {
console.error(`❌ Failed to create topic indexes (attempt ${retryCount}):`, indexError);
// If this was the last retry, throw the original error
if (retryCount >= maxRetries) {
console.error(`❌ Maximum retries (${maxRetries}) reached for topic ${config.topic}`);
throw new Error(`Topic index creation failed after ${maxRetries} attempts for topic ${config.topic}`);
}
// Continue to next retry
return attemptConsume();
}
}
else if (retryCount > maxRetries) {
console.error(`❌ Maximum retries (${maxRetries}) exceeded for topic ${config.topic}`);
throw new Error(`Index creation failed after ${maxRetries} attempts for topic ${config.topic}`);
}
}
throw error;
}
};
return attemptConsume();
},
// Get metrics
getMetrics: () => queue.getMetrics(),
// Shutdown everything
shutdown: async () => {
await writer.close();
await queue.shutdown();
},
};
}
/**
* Create a queue with automatic consumer setup
*/
async function createQueueWithConsumer(config, consumerId, messageHandler) {
const { queue, writer, enqueue, shutdown } = await createReadyQueue(config);
// Start the consumer
await queue.consume(consumerId, async (messages) => {
try {
await messageHandler(messages.map(msg => ({
id: msg.id,
type: msg.type,
payload: msg.payload,
metadata: msg.metadata,
})));
// Acknowledge all messages if handler succeeds
await Promise.all(messages.map(msg => msg.ack()));
}
catch (error) {
console.error(`Consumer ${consumerId} failed:`, error);
// Negative acknowledge all messages
const errorMessage = error instanceof Error ? error.message : String(error);
await Promise.all(messages.map(msg => msg.nack(errorMessage)));
}
});
return { queue, writer, enqueue, shutdown };
}
//# sourceMappingURL=simple-setup.js.map