firestore-queue
Version:
A powerful, scalable queue system built on Google Firestore with time-based indexing, auto-configuration, and connection reuse
313 lines • 10.6 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.BulkWriter = void 0;
const FirestoreWriter_1 = require("./FirestoreWriter");
/**
* Bulk Writer for Fire Queue
* Optimized for high-volume data ingestion with batching and streaming
*/
class BulkWriter {
constructor(config) {
this.buffer = [];
this.isProcessing = false;
this.config = {
batchSize: 500,
flushIntervalMs: 5000,
maxBufferSize: 10000,
enableDeduplication: false,
compressionThreshold: 1024,
retryFailedBatches: true,
maxRetries: 3,
...config,
};
this.writer = new FirestoreWriter_1.FirestoreWriter({
...config,
enableBatching: false, // We handle batching at this level
});
this.stats = {
totalReceived: 0,
totalQueued: 0,
totalFailed: 0,
batchesProcessed: 0,
averageBatchSize: 0,
averageFlushTimeMs: 0,
lastFlushTime: new Date(),
bufferSize: 0,
duplicatesRejected: 0,
};
this.startFlushTimer();
}
/**
* Add a single message to the bulk buffer
*/
async add(message) {
this.stats.totalReceived++;
this.stats.bufferSize = this.buffer.length;
// Check buffer capacity
if (this.buffer.length >= this.config.maxBufferSize) {
console.warn(`Buffer full (${this.config.maxBufferSize}), dropping message`);
this.stats.totalFailed++;
return;
}
// Deduplication check
if (this.config.enableDeduplication && this.isDuplicate(message)) {
this.stats.duplicatesRejected++;
return;
}
// Compress large payloads
if (this.config.compressionThreshold) {
message = await this.compressIfNeeded(message);
}
this.buffer.push(message);
this.stats.bufferSize = this.buffer.length;
// Flush if batch size reached
if (this.buffer.length >= this.config.batchSize) {
await this.flush();
}
}
/**
* Add multiple messages to the bulk buffer
*/
async addBatch(messages) {
for (const message of messages) {
await this.add(message);
}
}
/**
* Add messages from a readable stream
*/
async addFromStream(stream) {
return new Promise((resolve, reject) => {
const chunks = [];
stream.on('data', (chunk) => {
try {
const message = JSON.parse(chunk.toString());
chunks.push(this.add(message));
}
catch (error) {
console.error('Invalid JSON in stream:', error);
}
});
stream.on('end', async () => {
try {
await Promise.all(chunks);
await this.flush(); // Final flush
resolve();
}
catch (error) {
reject(error);
}
});
stream.on('error', reject);
});
}
/**
* Add messages from a CSV file
*/
async addFromCSV(csvData, columnMapping, messageDefaults = {}) {
const lines = csvData.split('\n').filter(line => line.trim());
const headers = lines[0].split(',').map(h => h.trim());
for (let i = 1; i < lines.length; i++) {
const values = lines[i].split(',').map(v => v.trim());
const row = {};
headers.forEach((header, index) => {
row[header] = values[index];
});
// Map CSV columns to message fields
const message = {
type: messageDefaults.type || 'csv_import',
payload: row,
priority: messageDefaults.priority || 5,
retryable: true,
maxRetries: 3,
retryCount: 0,
version: 1,
tags: [...(messageDefaults.tags || []), 'csv-import'],
metadata: {
...messageDefaults.metadata,
source: 'csv',
row: i,
},
};
// Apply column mapping
Object.entries(columnMapping).forEach(([csvColumn, messageField]) => {
if (row[csvColumn] !== undefined) {
this.setNestedProperty(message, messageField, row[csvColumn]);
}
});
await this.add(message);
}
}
/**
* Add messages from JSON Lines format
*/
async addFromJSONLines(jsonlData) {
const lines = jsonlData.split('\n').filter(line => line.trim());
for (const line of lines) {
try {
const message = JSON.parse(line);
await this.add(message);
}
catch (error) {
console.error('Invalid JSON line:', error);
this.stats.totalFailed++;
}
}
}
/**
* Flush the current buffer
*/
async flush() {
if (this.buffer.length === 0 || this.isProcessing) {
return {
success: true,
totalMessages: 0,
successfulWrites: 0,
failedWrites: 0,
messageIds: [],
errors: [],
timestamp: new Date(),
latencyMs: 0,
};
}
this.isProcessing = true;
const startTime = Date.now();
try {
const messages = [...this.buffer];
this.buffer = [];
this.stats.bufferSize = 0;
const result = await this.writer.writeBatch(messages);
// Update stats
this.stats.totalQueued += result.successfulWrites;
this.stats.totalFailed += result.failedWrites;
this.stats.batchesProcessed++;
this.stats.averageBatchSize =
(this.stats.averageBatchSize * (this.stats.batchesProcessed - 1) + messages.length) /
this.stats.batchesProcessed;
const flushTime = Date.now() - startTime;
this.stats.averageFlushTimeMs =
(this.stats.averageFlushTimeMs * (this.stats.batchesProcessed - 1) + flushTime) /
this.stats.batchesProcessed;
this.stats.lastFlushTime = new Date();
// Retry failed messages if enabled
if (this.config.retryFailedBatches && result.failedWrites > 0) {
await this.retryFailedMessages(messages, result.errors);
}
return result;
}
finally {
this.isProcessing = false;
}
}
/**
* Get current statistics
*/
getStats() {
return { ...this.stats };
}
/**
* Get buffer info
*/
getBufferInfo() {
return {
size: this.buffer.length,
maxSize: this.config.maxBufferSize,
utilizationPercent: (this.buffer.length / this.config.maxBufferSize) * 100,
isProcessing: this.isProcessing,
nextFlushIn: this.flushTimer ? this.config.flushIntervalMs : 0,
};
}
/**
* Start automatic flush timer
*/
startFlushTimer() {
this.flushTimer = setInterval(async () => {
if (this.buffer.length > 0) {
await this.flush();
}
}, this.config.flushIntervalMs);
}
/**
* Check if message is a duplicate
*/
isDuplicate(message) {
// Simple deduplication based on message content hash
const messageHash = this.hashMessage(message);
return this.buffer.some(bufferedMsg => this.hashMessage(bufferedMsg) === messageHash);
}
/**
* Hash message for deduplication
*/
hashMessage(message) {
const content = JSON.stringify({
type: message.type,
payload: message.payload,
});
// Simple hash function (consider using crypto for production)
let hash = 0;
for (let i = 0; i < content.length; i++) {
const char = content.charCodeAt(i);
hash = ((hash << 5) - hash) + char;
hash = hash & hash; // Convert to 32-bit integer
}
return hash.toString();
}
/**
* Compress message payload if it exceeds threshold
*/
async compressIfNeeded(message) {
const payloadSize = JSON.stringify(message.payload).length;
if (payloadSize > this.config.compressionThreshold) {
// In a real implementation, use actual compression library
console.log(`Message payload (${payloadSize} bytes) exceeds compression threshold`);
// message.metadata.compressed = true;
}
return message;
}
/**
* Retry failed messages
*/
async retryFailedMessages(originalMessages, errors) {
const failedMessages = errors.map(error => originalMessages[error.index]);
for (let attempt = 1; attempt <= this.config.maxRetries; attempt++) {
if (failedMessages.length === 0)
break;
console.log(`Retrying ${failedMessages.length} failed messages (attempt ${attempt})`);
const retryResult = await this.writer.writeBatch(failedMessages);
if (retryResult.success) {
console.log(`Retry successful: ${retryResult.successfulWrites} messages recovered`);
break;
}
// Wait before next retry
await new Promise(resolve => setTimeout(resolve, 1000 * attempt));
}
}
/**
* Set nested property on object
*/
setNestedProperty(obj, path, value) {
const keys = path.split('.');
let current = obj;
for (let i = 0; i < keys.length - 1; i++) {
if (!(keys[i] in current)) {
current[keys[i]] = {};
}
current = current[keys[i]];
}
current[keys[keys.length - 1]] = value;
}
/**
* Close the bulk writer
*/
async close() {
if (this.flushTimer) {
clearInterval(this.flushTimer);
this.flushTimer = undefined;
}
// Final flush
await this.flush();
await this.writer.close();
}
}
exports.BulkWriter = BulkWriter;
//# sourceMappingURL=BulkWriter.js.map