@tmlmobilidade/utils
Version:
A collection of utility functions and helpers for the TML Mobilidade Go monorepo, providing common functionality for batching operations, caching, HTTP requests, object manipulation, permissions, and more.
166 lines (165 loc) • 7.55 kB
JavaScript
/* eslint-disable perfectionist/sort-classes */
/* * */
import { Timer } from '@tmlmobilidade/timer';
/* * */
export class BatchWriter {
//
params;
dataBucketAlwaysAvailable = [];
dataBucketFlushOps = [];
batchTimeoutTimer = null;
idleTimeoutTimer = null;
sessionTimer = new Timer();
constructor(params) {
if (!params.title)
throw new Error('BATCHWRITER: Title is required.');
if (!params.insertFn)
throw new Error('BATCHWRITER: Insert function is required.');
if (!params.batch_size)
throw new Error('BATCHWRITER: Batch size is required.');
this.params = params;
}
/**
* Flushes the current batch of data.
* This method is called internally when the batch size or timeouts are reached,
* but can also be called manually if needed.
* @param callback Optional callback to execute after the flush is complete, receiving the flushed data as a parameter
*/
async flush(callback) {
try {
//
const flushTimer = new Timer();
const sessionTimerResult = this.sessionTimer.get();
//
// Invalidate all timers since a flush operation is being performed
if (this.idleTimeoutTimer) {
clearTimeout(this.idleTimeoutTimer);
this.idleTimeoutTimer = null;
}
if (this.batchTimeoutTimer) {
clearTimeout(this.batchTimeoutTimer);
this.batchTimeoutTimer = null;
}
//
// Skip if there is no data to flush
if (this.dataBucketAlwaysAvailable.length === 0)
return;
//
// Copy everything in dataBucketAlwaysAvailable to dataBucketFlushOps
// to prevent any new incoming data to be added to the batch. This is to ensure
// that the batch is not modified while it is being processed.
this.dataBucketFlushOps = [...this.dataBucketFlushOps, ...this.dataBucketAlwaysAvailable];
this.dataBucketAlwaysAvailable = [];
//
// Process the data for batch insert
try {
// Call the insert function provided in the params to perform the actual database insertion.
if (!this.params.insertFn)
throw new Error('BATCHWRITER: No insert function provided in params');
await this.insertWithRetry(this.dataBucketFlushOps);
console.info(`BATCHWRITER [${this.params.title}]: Flush | Length: ${this.dataBucketFlushOps.length} (session: ${sessionTimerResult}) (flush: ${flushTimer.get()})`);
// Call the flush callback, if provided
if (callback)
await callback(this.dataBucketFlushOps);
// Reset the flush bucket
this.dataBucketFlushOps = [];
}
catch (error) {
console.error(`BATCHWRITER [${this.params.title}]: Error @ flush().insert(): ${error.message}`);
throw error; // Re-throw to allow retry logic at higher level
}
//
}
catch (error) {
console.error(`BATCHWRITER [${this.params.title}]: Error @ flush(): ${error.message}`);
throw error; // Re-throw to allow retry logic at higher level
}
}
/**
* Helper method to perform insert operations with retry logic for transient errors.
* This method will attempt to insert the data using the provided insert function,
* and if an error occurs, it will retry the operation with exponential backoff
* until the maximum number of retries is reached.
* @param data The data to insert.
* @returns A promise that resolves when the insert operation is successful, or rejects if all retries fail.
*/
async insertWithRetry(data) {
const maxRetries = this.params.max_retries ?? 3;
const retryBaseDelayMs = this.params.retry_base_delay_ms ?? 1000;
for (let attempt = 0; attempt <= maxRetries; attempt++) {
try {
await this.params.insertFn(data);
return;
}
catch (error) {
const parsedError = error;
const nextAttempt = attempt + 1;
const delayMs = retryBaseDelayMs * (2 ** attempt);
console.error(`BATCHWRITER [${this.params.title}]: Transient insert error (${parsedError.code ?? 'unknown'}). Retrying ${nextAttempt}/${maxRetries} in ${delayMs}ms. ${parsedError.message}`);
await new Promise(resolve => setTimeout(resolve, delayMs));
}
}
}
/**
* Write data to the batch.
* @param data The data to write.
* @param options Options for the write operation (reserved for future use).
* @param writeCallback Callback function to call after the write operation is complete.
* @param flushCallback Callback function to call after the flush operation is complete.
*/
async write(data, { flushCallback, writeCallback } = {}) {
//
//
// Invalidate the previously set idle timeout timer
// since we are performing a write operation again.
if (this.idleTimeoutTimer) {
clearTimeout(this.idleTimeoutTimer);
this.idleTimeoutTimer = null;
}
//
// Check if the batch is full
const batchSize = this.params.batch_size ?? 10_000;
if (this.dataBucketAlwaysAvailable.length >= batchSize) {
console.info(`BATCHWRITER [${this.params.title}]: Batch full. Flushing data...`);
await this.flush(flushCallback);
}
//
// Reset the session timer (for logging purposes)
if (this.dataBucketAlwaysAvailable.length === 0) {
this.sessionTimer.reset();
}
//
// Add the current data to the batch
if (Array.isArray(data)) {
const combinedDataWithOptions = data.map(item => item);
this.dataBucketAlwaysAvailable = [...this.dataBucketAlwaysAvailable, ...combinedDataWithOptions];
}
else {
this.dataBucketAlwaysAvailable.push(data);
}
//
// Call the write callback, if provided
if (writeCallback) {
await writeCallback();
}
//
// Setup the idle timeout timer to flush the data if too long has passed
// since the last write operation. Check if this functionality is enabled.
if (this.params.idle_timeout && this.params.idle_timeout > 0 && !this.idleTimeoutTimer) {
this.idleTimeoutTimer = setTimeout(async () => {
console.info(`BATCHWRITER [${this.params.title}]: Idle timeout reached. Flushing data...`);
await this.flush(flushCallback);
}, this.params.idle_timeout);
}
//
// Setup the batch timeout timer to flush the data, if the timeout value is reached,
// even if the batch is not full. Check if this functionality is enabled.
if (this.params.batch_timeout && this.params.batch_timeout > 0 && !this.batchTimeoutTimer) {
this.batchTimeoutTimer = setTimeout(async () => {
console.info(`BATCHWRITER [${this.params.title}]: Batch timeout reached. Flushing data...`);
await this.flush(flushCallback);
}, this.params.batch_timeout);
}
//
}
}