firestore-queue/dist/writers/BulkWriter.d.ts

A powerful, scalable queue system built on Google Firestore with time-based indexing, auto-configuration, and connection reuse

import { WriterConfig, BulkWriteResult } from './types';
import { QueueMessage, QueueMessageInput } from '../types';
/**
 * Bulk Writer for Fire Queue
 * Optimized for high-volume data ingestion with batching and streaming
 */
export declare class BulkWriter {
    private writer;
    private config;
    private buffer;
    private flushTimer?;
    private isProcessing;
    private stats;
    constructor(config: BulkWriterConfig);
    /**
     * Add a single message to the bulk buffer
     */
    add(message: QueueMessageInput): Promise<void>;
    /**
     * Add multiple messages to the bulk buffer
     */
    addBatch(messages: QueueMessageInput[]): Promise<void>;
    /**
     * Add messages from a readable stream
     */
    addFromStream(stream: NodeJS.ReadableStream): Promise<void>;
    /**
     * Add messages from a CSV file
     */
    addFromCSV(csvData: string, columnMapping: Record<string, string>, messageDefaults?: Partial<QueueMessage>): Promise<void>;
    /**
     * Add messages from JSON Lines format
     */
    addFromJSONLines(jsonlData: string): Promise<void>;
    /**
     * Flush the current buffer
     */
    flush(): Promise<BulkWriteResult>;
    /**
     * Get current statistics
     */
    getStats(): BulkWriteStats;
    /**
     * Get buffer info
     */
    getBufferInfo(): BufferInfo;
    /**
     * Start automatic flush timer
     */
    private startFlushTimer;
    /**
     * Check if message is a duplicate
     */
    private isDuplicate;
    /**
     * Hash message for deduplication
     */
    private hashMessage;
    /**
     * Compress message payload if it exceeds threshold
     */
    private compressIfNeeded;
    /**
     * Retry failed messages
     */
    private retryFailedMessages;
    /**
     * Set nested property on object
     */
    private setNestedProperty;
    /**
     * Close the bulk writer
     */
    close(): Promise<void>;
}
/**
 * Bulk writer configuration
 */
export interface BulkWriterConfig extends WriterConfig {
    batchSize?: number;
    flushIntervalMs?: number;
    maxBufferSize?: number;
    enableDeduplication?: boolean;
    compressionThreshold?: number;
    retryFailedBatches?: boolean;
    maxRetries?: number;
}
/**
 * Bulk write statistics
 */
export interface BulkWriteStats {
    totalReceived: number;
    totalQueued: number;
    totalFailed: number;
    batchesProcessed: number;
    averageBatchSize: number;
    averageFlushTimeMs: number;
    lastFlushTime: Date;
    bufferSize: number;
    duplicatesRejected: number;
}
/**
 * Buffer information
 */
export interface BufferInfo {
    size: number;
    maxSize: number;
    utilizationPercent: number;
    isProcessing: boolean;
    nextFlushIn: number;
}
//# sourceMappingURL=BulkWriter.d.ts.map