firestore-queue
Version:
A powerful, scalable queue system built on Google Firestore with time-based indexing, auto-configuration, and connection reuse
351 lines • 12.6 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.FirestoreWriter = void 0;
const admin = __importStar(require("firebase-admin"));
const types_1 = require("../types");
/**
* Direct Firestore writer for Fire Queue
* Allows writing directly to the queue collection from any application
*/
class FirestoreWriter {
constructor(config) {
this.batchBuffer = [];
this.config = {
validatePayload: true,
maxPayloadSize: 1024 * 1024, // 1MB
enableBatching: false,
batchSize: 50,
batchTimeoutMs: 5000,
retryAttempts: 3,
retryDelayMs: 1000,
enableMetrics: true,
...config,
};
this.metrics = {
totalWrites: 0,
successfulWrites: 0,
failedWrites: 0,
averageLatencyMs: 0,
currentBatchSize: 0,
lastWriteTimestamp: new Date(),
};
this.initializeFirestore();
}
/**
* Initialize Firestore connection
*/
initializeFirestore() {
try {
// Use pre-configured Firestore instance if provided
if (this.config.firestoreInstance) {
this.db = this.config.firestoreInstance;
return;
}
// Initialize Firebase Admin if not already done
if (admin.apps.length === 0) {
if (this.config.serviceAccountPath) {
const serviceAccount = require(this.config.serviceAccountPath);
admin.initializeApp({
credential: admin.credential.cert(serviceAccount),
projectId: this.config.projectId,
});
}
else {
admin.initializeApp({
projectId: this.config.projectId,
});
}
}
// Create Firestore instance with specified database
if (this.config.databaseId && this.config.databaseId !== '(default)') {
this.db = new admin.firestore.Firestore({
projectId: this.config.projectId,
databaseId: this.config.databaseId,
});
}
else {
this.db = admin.firestore();
}
}
catch (error) {
throw new Error(`Failed to initialize Firestore writer: ${error}`);
}
}
/**
* Write a single message to the queue
*/
async write(message) {
const startTime = Date.now();
try {
// Validate message
const validatedMessage = this.validateMessage(message);
if (this.config.enableBatching) {
return await this.addToBatch(validatedMessage);
}
else {
return await this.writeImmediate(validatedMessage);
}
}
catch (error) {
this.updateMetrics(false, Date.now() - startTime);
return {
success: false,
error: error instanceof Error ? error.message : String(error),
timestamp: new Date(),
latencyMs: Date.now() - startTime,
};
}
}
/**
* Write multiple messages as a batch
*/
async writeBatch(messages) {
const startTime = Date.now();
const messageIds = [];
const errors = [];
try {
// Validate all messages first
const validatedMessages = messages.map((msg, index) => {
try {
return this.validateMessage(msg);
}
catch (error) {
errors.push({ index, error: error instanceof Error ? error.message : String(error) });
return null;
}
}).filter(msg => msg !== null);
if (validatedMessages.length === 0) {
throw new Error('No valid messages to write');
}
// Write in Firestore batches (max 500 operations per batch)
const batchSize = 500;
const batches = [];
let currentBatch = this.db.batch();
let operationsInBatch = 0;
for (let i = 0; i < validatedMessages.length; i++) {
const message = validatedMessages[i];
const messageId = this.generateMessageId();
const queueDocument = this.createQueueDocument(message, messageId);
const docRef = this.db.collection(this.config.queueName).doc(messageId);
currentBatch.set(docRef, queueDocument);
messageIds.push(messageId);
operationsInBatch++;
if (operationsInBatch >= batchSize || i === validatedMessages.length - 1) {
batches.push(currentBatch);
if (i < validatedMessages.length - 1) {
currentBatch = this.db.batch();
operationsInBatch = 0;
}
}
}
// Execute all batches
await Promise.all(batches.map(batch => batch.commit()));
const result = {
success: true,
totalMessages: messages.length,
successfulWrites: validatedMessages.length,
failedWrites: errors.length,
messageIds,
errors,
timestamp: new Date(),
latencyMs: Date.now() - startTime,
};
this.updateMetrics(true, Date.now() - startTime, validatedMessages.length);
return result;
}
catch (error) {
this.updateMetrics(false, Date.now() - startTime, messages.length);
return {
success: false,
totalMessages: messages.length,
successfulWrites: 0,
failedWrites: messages.length,
messageIds: [],
errors: [{ index: -1, error: error instanceof Error ? error.message : String(error) }],
timestamp: new Date(),
latencyMs: Date.now() - startTime,
};
}
}
/**
* Add message to batch buffer
*/
async addToBatch(message) {
this.batchBuffer.push(message);
this.metrics.currentBatchSize = this.batchBuffer.length;
// Start batch timer if not already running
if (!this.batchTimer) {
this.batchTimer = setTimeout(() => this.flushBatch(), this.config.batchTimeoutMs);
}
// Flush immediately if batch is full
if (this.batchBuffer.length >= this.config.batchSize) {
await this.flushBatch();
}
return {
success: true,
messageId: 'batched',
timestamp: new Date(),
latencyMs: 0, // Batched writes have minimal latency
};
}
/**
* Flush batch buffer
*/
async flushBatch() {
if (this.batchBuffer.length === 0)
return;
const messages = [...this.batchBuffer];
this.batchBuffer = [];
this.metrics.currentBatchSize = 0;
if (this.batchTimer) {
clearTimeout(this.batchTimer);
this.batchTimer = undefined;
}
try {
await this.writeBatch(messages);
}
catch (error) {
console.error('Batch flush failed:', error);
// Messages are lost - consider implementing dead letter queue
}
}
/**
* Write message immediately
*/
async writeImmediate(message) {
const messageId = this.generateMessageId();
const queueDocument = this.createQueueDocument(message, messageId);
await this.db
.collection(this.config.queueName)
.doc(messageId)
.set(queueDocument);
this.updateMetrics(true, 0);
return {
success: true,
messageId,
timestamp: new Date(),
latencyMs: 0,
};
}
/**
* Validate message against schema and config
*/
validateMessage(message) {
if (this.config.validatePayload) {
// Validate with Zod schema
const validated = types_1.QueueMessageSchema.parse(message);
// Check payload size
const payloadSize = JSON.stringify(validated.payload).length;
if (payloadSize > this.config.maxPayloadSize) {
throw new Error(`Payload too large: ${payloadSize} > ${this.config.maxPayloadSize} bytes`);
}
// Check allowed message types
if (this.config.allowedMessageTypes &&
!this.config.allowedMessageTypes.includes(validated.type)) {
throw new Error(`Message type '${validated.type}' not allowed`);
}
return validated;
}
return types_1.QueueMessageSchema.parse(message);
}
/**
* Create queue document from message
*/
createQueueDocument(message, messageId) {
const now = admin.firestore.Timestamp.now();
const ttlSeconds = message.ttlSeconds || 3600; // Default 1 hour
const expiresAt = new admin.firestore.Timestamp(now.seconds + ttlSeconds, now.nanoseconds);
return {
...message,
id: messageId,
status: 'pending',
createdAt: now,
updatedAt: now,
expiresAt,
scheduledFor: message.scheduledFor
? message.scheduledFor
: now.toDate(),
};
}
/**
* Generate unique message ID
*/
generateMessageId() {
return `msg_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
}
/**
* Update internal metrics
*/
updateMetrics(success, latencyMs, count = 1) {
if (!this.config.enableMetrics)
return;
this.metrics.totalWrites += count;
if (success) {
this.metrics.successfulWrites += count;
}
else {
this.metrics.failedWrites += count;
}
// Update average latency
const totalLatency = this.metrics.averageLatencyMs * (this.metrics.totalWrites - count) + latencyMs;
this.metrics.averageLatencyMs = totalLatency / this.metrics.totalWrites;
this.metrics.lastWriteTimestamp = new Date();
// Call metrics callback if provided
if (this.config.metricsCallback) {
this.config.metricsCallback(this.metrics);
}
}
/**
* Get current metrics
*/
getMetrics() {
return { ...this.metrics };
}
/**
* Close the writer and flush any pending batches
*/
async close() {
if (this.batchBuffer.length > 0) {
await this.flushBatch();
}
if (this.batchTimer) {
clearTimeout(this.batchTimer);
this.batchTimer = undefined;
}
}
}
exports.FirestoreWriter = FirestoreWriter;
//# sourceMappingURL=FirestoreWriter.js.map