firelease
Version:
Firebase queue consumer for Node with at-least-once semantics
1,210 lines • 48.8 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.settings = exports.stats = exports.defaults = exports.RETRY = exports.TESTABLES = void 0;
exports.attachWorker = attachWorker;
exports.pingQueues = pingQueues;
exports.extendLease = extendLease;
exports.blacklist = blacklist;
exports.shutdown = shutdown;
exports.listTasksInProgress = listTasksInProgress;
const lodash_1 = __importDefault(require("lodash"));
const ms_1 = __importDefault(require("ms"));
const nodefire_1 = __importDefault(require("nodefire"));
const timers = __importStar(require("safe-timers"));
const stats_1 = require("./stats");
exports.TESTABLES = { resetBetweenTests, waitUntilDeleted, getQueueCheckCooldown };
const PING_INTERVAL = (0, ms_1.default)('1m');
const PING_KEY = 'ping';
const QUEUE_CHECK_TIMEOUT = (0, ms_1.default)('15s');
const MAX_QUEUE_CHECK_COOLDOWN = (0, ms_1.default)('30s');
const QUEUE_SIZE_HYSTERESIS = 0.15;
const QUEUE_SIZE_MISMATCH_THRESHOLD = 100;
const DEMOTION_JITTER = (0, ms_1.default)('30s');
const LEASE_TRANSACTION_DURATION_ALPHA = 0.1;
const queues = [];
const tasks = {};
const blacklistedTaskKeys = new Set();
let globalMaxConcurrent = Number.MAX_VALUE;
let globalNumConcurrent = 0;
let safeQueueSize = 6000;
let queueCheckInterval = (0, ms_1.default)('5m');
let queueLoadTimeout = (0, ms_1.default)('1m');
let shutdownResolve;
let shutdownReject;
let shutdownPromise;
const defaultCaptureError = (error) => { console.error(error.stack); };
/**
* Return this from a worker to retry after the current lease expires, and to reset the lease
* backoff to zero.
*/
exports.RETRY = {};
/** Default option values for all subsequent attachWorker calls. */
exports.defaults = {
maxConcurrent: Number.MAX_VALUE, bufferSize: Infinity, minLease: '30s', maxLease: '1h',
healthyPingLatency: '1.5s'
};
exports.stats = new stats_1.FireleaseStats(() => blacklistedTaskKeys.size);
const scanAll = lodash_1.default.debounce(() => {
lodash_1.default.forEach(tasks, task => {
void task.queue.process(task);
});
}, 100);
exports.settings = {
get globalMaxConcurrent() {
return globalMaxConcurrent;
},
set globalMaxConcurrent(value) {
globalMaxConcurrent = value;
if (value) {
shutdownReject?.(new Error('Queues restarted'));
shutdownPromise = shutdownResolve = shutdownReject = undefined;
scanAll();
}
},
get safeQueueSize() {
return safeQueueSize;
},
set safeQueueSize(value) {
if (!Number.isFinite(value) || value < 1) {
throw new Error('safeQueueSize must be a positive finite number');
}
safeQueueSize = Math.floor(value);
},
get queueCheckInterval() {
return queueCheckInterval;
},
set queueCheckInterval(value) {
const normalizedValue = duration(value);
if (!Number.isFinite(normalizedValue) || normalizedValue <= 0) {
throw new Error('queueCheckInterval must be a positive finite duration');
}
queueCheckInterval = normalizedValue;
},
get queueLoadTimeout() {
return queueLoadTimeout;
},
set queueLoadTimeout(value) {
const normalizedValue = duration(value);
if (!Number.isFinite(normalizedValue) || normalizedValue <= 0) {
throw new Error('queueLoadTimeout must be a positive finite duration');
}
queueLoadTimeout = normalizedValue;
},
captureError: defaultCaptureError
};
const firelease = Object.freeze({
RETRY: exports.RETRY,
settings: exports.settings,
defaults: exports.defaults,
stats: exports.stats,
attachWorker,
pingQueues,
extendLease,
blacklist,
shutdown,
listTasksInProgress
});
class Task {
constructor(source, snap) {
this.source = source;
this.phase = 'wait';
this.expiry = 0;
this.working = false;
this.queue = source.queue;
this.ref = snap.ref;
this.key = Task.makeKey(snap);
this.updateFrom(snap);
}
static makeKey(snap) {
return snap.ref.toString();
}
updateFrom(snap) {
const value = snap.val();
this.expiry = value?._lease?.expiry ?? this.ref.now;
// console.log('update', this.key, 'expiry', this.expiry);
delete this.removed;
}
prepare() {
if (tasks[this.key] !== this || this.removed || this.working)
return false;
const now = this.ref.now;
const busy = this.expiry > now;
// console.log('prepare', this.ref.key, 'expiry', this.expiry, 'now', now);
if (!busy) {
// Locally reserve for min lease duration to prevent concurrent transaction attempts. Expiry
// will be overwritten when transaction completes or task gets removed.
this.expiry = now + this.queue.constrainLeaseDuration(0);
}
this.timeout?.clear();
this.timeout = timers.setTimeout(this.queue.process.bind(this.queue, this), this.expiry - now);
return !busy;
}
async process() {
let startTimestamp = 0;
let acquired = false;
let contended = false;
let reschedule = true;
let firstAcquisition = false;
this.working = true;
this.phase = 'lease';
const transactionPromise = this.ref.transaction(itemValue => {
const item = itemValue;
acquired = false;
contended = false;
firstAcquisition = false;
if (tasks[this.key] !== this || this.removed)
return;
if (!item || this.ref.key === PING_KEY) {
acquired = true;
return null;
}
startTimestamp = this.ref.now;
// console.log('txn ', this.ref.key, 'lease', item._lease, 'now', startTimestamp);
// Check if another process beat us to it.
if (item._lease?.expiry && item._lease.expiry > startTimestamp) {
contended = true;
return item;
}
acquired = true;
firstAcquisition = lodash_1.default.isNil(item._lease?.initial);
item._lease ??= {};
item._lease.time = this.queue.constrainLeaseDuration((item._lease.time ?? 0) * 2);
item._lease.expiry = startTimestamp + item._lease.time;
item._lease.attempts = (item._lease.attempts ?? 0) + 1;
item._lease.initial ??= startTimestamp;
item._lease.busy = true;
return this.queue.callPreprocess(item);
}, { detectStuck: 5, prefetchValue: false, timeout: (0, ms_1.default)('15s') });
let transactionCompleted = false;
try {
const item = await transactionPromise;
transactionCompleted = true;
if (acquired && item !== null && this.ref.key !== PING_KEY) {
this.recordLeaseTransaction('acquired', transactionPromise.transaction);
if (firstAcquisition)
Object.defineProperty(item._lease, 'firstAcquisition', { value: true });
this.queue.stats.tasksAcquired++;
await this.run(item, startTimestamp);
}
else if (contended) {
this.recordLeaseTransaction('contended', transactionPromise.transaction);
}
}
catch (error) {
if (!transactionCompleted && this.ref.key !== PING_KEY) {
this.recordLeaseTransaction('failed', transactionPromise.transaction);
}
reschedule = false;
// Hardcoded retry -- hard to do anything smarter, since we failed to update the task in
// Firebase.
this.expiry = 0;
if (!/timeout/i.test(error.message) || this.source.connected) {
console.log(`Queue item ${this.key} lease transaction error: ${error.message}`);
error.firelease = lodash_1.default.assign(error.firelease ?? {}, { itemKey: this.key, phase: 'leasing' });
exports.settings.captureError(error);
timers.setTimeout(this.queue.scan, (0, ms_1.default)('3s'));
}
}
this.working = false;
this.phase = this.removed ? 'done' : 'retry';
if (!this.removed && reschedule) {
// Wait until Queue.process() releases this task's concurrency slot before re-arming its
// lease-expiry timer. Listener swaps can replay the task while it is still working.
timers.setTimeout(() => { void this.queue.process(this); }, 0);
}
}
recordLeaseTransaction(outcome, transaction) {
try {
const leaseStats = this.source.stats.leaseTransactions;
leaseStats[outcome] += 1;
if (!transaction?.tries || transaction?.duration === undefined)
return;
const tries = transaction?.tries;
const transactionDuration = (transaction?.prefetchDuration ?? 0) + transaction?.duration;
leaseStats.tries += tries;
leaseStats.duration = leaseStats.duration === 0 ?
transactionDuration || 1 :
leaseStats.duration * (1 - LEASE_TRANSACTION_DURATION_ALPHA) +
transactionDuration * LEASE_TRANSACTION_DURATION_ALPHA;
this.queue.options.captureLeaseTransactionMetrics?.(outcome, tries, transactionDuration);
}
catch (error) {
try {
const metricError = lodash_1.default.isError(error) ? error : new Error(String(error));
metricError.firelease = lodash_1.default.assign(metricError.firelease ?? {}, { itemKey: this.key, phase: 'lease-metric' });
exports.settings.captureError(metricError);
}
catch (captureError) {
try {
console.error('Error capturing lease transaction metric error:', captureError);
}
catch {
// Metric recording must never interrupt task processing.
}
}
}
}
async run(item, startTimestamp) {
Object.defineProperty(item, '$ref', { value: this.ref });
Object.defineProperty(item, '$leaseTimeRemaining', { get: () => {
if (!item._lease?.expiry)
return 0;
return Math.max(0, item._lease.expiry - this.ref.now);
} });
this.phase = 'work';
let result;
try {
try {
result = await this.queue.callWorker(item);
}
finally {
const now = this.ref.now;
if (now > item._lease.expiry) {
this.phase = 'exceed';
// If it looks like we exceeded the lease time, double-check against the current item
// before crying wolf, in case the worker extended the lease.
const currentItem = await this.ref.get({ cache: false });
// If no item, we can't tell if it's because the worker chose to delete it early, or
// because it overran its lease and another worker picked it up and completed it, so say
// nothing.
if (currentItem) {
if (!currentItem._lease) {
console.log(`Queue item ${this.key} likely exceeded its lease time by taking`, (0, ms_1.default)(now - startTimestamp), 'because the item has already been deleted and replaced with a new one.');
}
else if (currentItem._lease.expiry && now > currentItem._lease.expiry) {
console.log(`Queue item ${this.key} exceeded lease time of`, (0, ms_1.default)(currentItem._lease.expiry - startTimestamp), 'by taking', (0, ms_1.default)(now - startTimestamp));
}
}
}
}
}
catch (processingError) {
try {
if (/timeout/i.test(processingError.message) && !this.source.connected)
return;
console.log(`Queue item ${this.key} processing error: ${processingError.message}`);
processingError.firelease = lodash_1.default.assign(processingError.firelease ?? {}, { itemKey: this.key, phase: 'processing' });
processingError.level ??= 'warning';
exports.settings.captureError(processingError);
// Reset busy flag, unless we exceeded our original lease in which case we can't be sure
// whether another handler has already picked up the task so leave it be.
if (this.phase !== 'exceed')
await this.ref.child('_lease/busy').set(null);
}
catch (postProcessingError) {
this.handlePostProcessingError(postProcessingError);
}
return;
}
try {
this.phase = 'post';
if (lodash_1.default.isNil(result)) {
await this.ref.remove(); // common shortcut
return;
}
const item2 = await this.ref.transaction(itemValue => {
const currentItem = itemValue;
if (!currentItem)
return null;
let value = lodash_1.default.isFunction(result) ? result(currentItem) : result;
if (lodash_1.default.isNil(value))
return null;
if (value === firelease.RETRY) {
if (currentItem._lease)
delete currentItem._lease.time;
}
else if (lodash_1.default.isNumber(value) || lodash_1.default.isString(value)) {
value = duration(value);
currentItem._lease ??= {};
currentItem._lease.expiry =
value > 1000000000000 ? value : startTimestamp + value;
delete currentItem._lease.time;
}
else if (lodash_1.default.isObject(value)) {
currentItem._lease = value;
}
else {
throw new Error(`Unexpected return value from worker: ${value}`);
}
if (currentItem._lease)
delete currentItem._lease.busy;
return currentItem;
}, { prefetchValue: false });
if (item2)
item._lease = item2._lease;
}
catch (postProcessingError) {
this.handlePostProcessingError(postProcessingError);
}
}
handlePostProcessingError(error) {
if (/timeout/i.test(error.message) && !this.source.connected)
return;
console.log(`Queue item ${this.key} post-processing error: ${error.message}`);
error.firelease = lodash_1.default.assign(error.firelease ?? {}, { itemKey: this.key, phase: 'post-processing' });
exports.settings.captureError(error);
}
}
function getQueueCheckCooldown(previousDuration) {
return Math.min(previousDuration, MAX_QUEUE_CHECK_COOLDOWN);
}
class QueueCheckQueue {
constructor() {
this.jobs = [];
this.draining = false;
this.previousDuration = 0;
this.previousFinishedAt = 0;
}
enqueue(source, description, epoch, run) {
const duplicate = (this.active?.source === source && this.active.epoch === epoch) ||
// Match shorthand would deep-compare QueueSource, but identity is required here.
// eslint-disable-next-line lodash/matches-shorthand
lodash_1.default.some(this.jobs, job => job.source === source && job.epoch === epoch);
if (duplicate) {
source.reportError('queue-check-coalesced', 'Firelease queue check coalesced', 'warning', { description });
return Promise.resolve();
}
return new Promise(resolve => {
this.jobs.push({ source, epoch, description, run, resolve });
void this.drain();
});
}
reset() {
lodash_1.default.forEach(this.jobs.splice(0), job => { job.resolve(); });
this.previousDuration = 0;
this.previousFinishedAt = 0;
}
async drain() {
if (this.draining)
return;
this.draining = true;
while (this.jobs.length) {
const job = this.jobs.shift();
try {
if (!job.source.isCurrent(job.epoch)) {
job.resolve();
continue;
}
// Don't exceed a 50% duty cycle.
const minimumStart = this.previousFinishedAt + getQueueCheckCooldown(this.previousDuration);
const now = performance.now();
if (minimumStart > now) {
await new Promise(resolve => {
timers.setTimeout(resolve, minimumStart - now);
});
}
if (!job.source.isCurrent(job.epoch)) {
job.resolve();
continue;
}
this.active = job;
const start = performance.now();
try {
await job.run();
}
catch (error) {
job.source.crash('queue-check-failed', 'Firelease queue check failed', error, { description: job.description });
}
finally {
const finishedAt = performance.now();
this.previousDuration = finishedAt - start;
this.previousFinishedAt = finishedAt;
this.active = undefined;
job.resolve();
}
}
catch (error) {
job.source.crash('queue-check-queue-failed', 'Firelease queue check queue failed', error, { description: job.description });
break;
}
}
this.draining = false;
}
}
const queueCheckQueue = new QueueCheckQueue();
class QueueListener {
constructor(source, mode) {
this.source = source;
this.mode = mode;
this.snapshots = new Map();
this.loaded = false;
this.stopped = false;
this.observers = new Set();
this.onAdd = (snap) => {
this.snapshots.set(Task.makeKey(snap), snap);
this.source.addTask(snap);
this.notify();
};
this.onRemove = (snap) => {
this.snapshots.delete(Task.makeKey(snap));
this.source.removeTask(snap);
this.notify();
};
this.onValue = () => {
if (this.stopped)
return;
this.loaded = true;
this.query.off('value', this.onValue);
this.notify();
};
this.onError = (error) => {
this.source.crash('queue-listener-failed', 'Firelease queue listener failed', error, { mode: this.mode });
this.stop();
};
const limit = source.listenerLimit(mode);
this.query =
mode === 'full' ? source.ref : source.ref.orderByChild('_lease/expiry').limitToFirst(limit);
this.changeEvent = mode === 'full' ? 'child_changed' : 'child_moved';
if (source.adaptive && mode === 'full')
this.observers.add(source.onListenerSize);
}
start() {
this.query.on('child_added', this.onAdd, this.onError);
this.query.on('child_removed', this.onRemove, this.onError);
this.query.on(this.changeEvent, this.onAdd, this.onError);
this.query.on('value', this.onValue, this.onError);
}
waitForLoad(timeout) {
if (this.loaded)
return Promise.resolve('loaded');
if (this.stopped)
return Promise.resolve('stopped');
return new Promise(resolve => {
let timeoutHandle; // eslint-disable-line prefer-const
const onChange = () => {
if (!this.loaded && !this.stopped)
return;
this.observers.delete(onChange);
timeoutHandle?.clear();
resolve(this.loaded ? 'loaded' : 'stopped');
};
this.observers.add(onChange);
timeoutHandle = timers.setTimeout(() => {
this.observers.delete(onChange);
resolve('timed-out');
}, timeout);
});
}
stop() {
if (this.stopped)
return;
this.stopped = true;
this.query.off('child_added', this.onAdd);
this.query.off('child_removed', this.onRemove);
this.query.off(this.changeEvent, this.onAdd);
this.query.off('value', this.onValue);
this.observers.delete(this.source.onListenerSize);
this.notify();
}
notify() {
for (const observer of this.observers)
observer(this);
}
}
class QueueSource {
constructor(queue, ref) {
this.queue = queue;
this.ref = ref;
this.mode = 'safe';
this.epoch = 0;
this.connected = false;
this.crashing = false;
this.connectionStartedAt = 0;
this.promotionNotBeforeTimestamp = 0;
this.initialStartupComplete = false;
this.onConnection = (snap) => {
const connected = Boolean(snap.val());
if (this.connected === connected)
return;
this.connected = connected;
this.stats.connected = connected;
this.epoch++;
const epoch = this.epoch;
this.cancelPendingWork();
if (connected) {
this.stats.healthy = true;
this.connectionStartedAt = performance.now();
void this.enqueueStartup(epoch);
}
else {
this.activeListener?.stop();
this.activeListener = undefined;
this.clearTasks();
if (this.stats.size !== null && this.stats.sizeTimestamp === undefined) {
this.stats.sizeTimestamp = Date.now();
}
this.stats.healthy = false;
}
};
this.onListenerSize = (listener) => {
if (listener !== this.activeListener || !listener.loaded)
return;
const size = listener.snapshots.size;
this.stats.size = size;
delete this.stats.sizeDelta;
delete this.stats.sizeTimestamp;
if (size > exports.settings.safeQueueSize) {
this.scheduleDemotion();
}
else if (this.demotionTimer) {
this.demotionTimer.clear();
this.demotionTimer = undefined;
}
};
this.connectionRef = this.ref.root.child('.info/connected');
this.stats = new stats_1.QueueSourceStats(ref.toString());
}
get adaptive() {
return this.queue.options.bufferSize === Infinity;
}
start() {
this.connectionRef.on('value', this.onConnection);
}
reset() {
this.connectionRef.off('value', this.onConnection);
this.connected = false;
this.epoch++;
this.cancelPendingWork();
this.exitTimer?.clear();
this.exitTimer = undefined;
this.activeListener?.stop();
this.activeListener = undefined;
this.clearTasks();
}
async enqueueStartup(epoch) {
await queueCheckQueue.enqueue(this, 'startup', epoch, () => this.initialize(epoch));
}
async initialize(epoch) {
try {
let targetMode = 'safe';
if (this.adaptive) {
const count = await this.probeSize('startup');
if (!this.isCurrent(epoch))
return;
targetMode = count !== null && count < fullQueueSize() ? 'full' : 'safe';
}
const loadedMode = await this.loadListener(targetMode, epoch, 'startup');
if (!loadedMode)
return;
const loadedTaskCount = this.activeListener?.snapshots.size ?? 0;
const connectionDuration = Math.round(performance.now() - this.connectionStartedAt);
console.log(`Queue worker ${this.ref} loaded ${loadedTaskCount} tasks in ${loadedMode} mode` +
` (${(0, ms_1.default)(connectionDuration)})`);
if (loadedMode === 'safe')
this.scheduleSafeCheck();
this.initialStartupComplete = true;
}
catch (e) {
if (this.initialStartupComplete)
throw e;
this.crash('queue-startup-failed', 'Firelease queue startup failed', e, { description: 'startup' });
}
}
isCurrent(epoch) {
return !this.crashing && this.connected && this.epoch === epoch;
}
listenerLimit(mode) {
if (mode === 'full')
return Infinity;
const limit = this.adaptive ? exports.settings.safeQueueSize : this.queue.options.bufferSize;
return Math.max(1, Math.floor(limit));
}
async probeSize(reason) {
try {
const keys = await this.ref.childrenKeys({ timeout: QUEUE_CHECK_TIMEOUT });
this.stats.size = keys.length;
delete this.stats.sizeDelta;
this.stats.sizeTimestamp = Date.now();
return keys.length;
}
catch (error) {
this.reportError('queue-count-failed', 'Firelease queue count failed', 'warning', { cause: error.message, reason });
return null;
}
}
scheduleSafeCheck() {
this.checkTimer?.clear();
if (!(this.adaptive && this.connected && this.mode === 'safe'))
return;
const interval = queueCheckInterval;
const jitteredInterval = Math.max(0, Math.round(interval * (0.95 + Math.random() * 0.1)));
const epoch = this.epoch;
this.checkTimer = timers.setTimeout(() => {
this.checkTimer = undefined;
this.scheduleSafeCheck();
void queueCheckQueue.enqueue(this, 'scheduled shallow count', epoch, () => this.runScheduledCheck(epoch));
}, jitteredInterval);
}
async runScheduledCheck(epoch) {
if (!this.isCurrent(epoch) || this.mode !== 'safe')
return;
const count = await this.probeSize('scheduled');
if (!this.isCurrent(epoch) || count === null || this.mode !== 'safe')
return;
const liveCount = this.activeListener?.snapshots.size ?? 0;
const listenerLimit = this.listenerLimit('safe');
const delta = count - liveCount;
if (liveCount < listenerLimit)
this.stats.sizeDelta = delta;
else
delete this.stats.sizeDelta;
if (this.stats.sizeDelta !== undefined && delta >= QUEUE_SIZE_MISMATCH_THRESHOLD) {
this.reportError('safe-queue-size-mismatch', 'Firelease safe queue size mismatch', 'error', { count, delta, listenerLimit, liveCount });
}
if (count < fullQueueSize() && performance.now() >= this.promotionNotBeforeTimestamp) {
await this.promote(epoch);
}
}
async promote(epoch) {
if (!this.isCurrent(epoch) || this.mode !== 'safe')
return;
const loadedMode = await this.loadListener('full', epoch, 'promotion');
if (!loadedMode)
return;
if (loadedMode === 'safe') {
this.scheduleSafeCheck();
return;
}
console.log(`Queue worker ${this.ref} promoted to full mode with` +
` ${this.activeListener?.snapshots.size ?? 0} tasks`);
}
scheduleDemotion() {
if (!(this.adaptive && this.connected && this.mode === 'full' && !this.demotionTimer))
return;
const epoch = this.epoch;
const delay = Math.round(Math.random() * DEMOTION_JITTER);
console.log(`Queue worker ${this.ref} scheduling demotion with ${(0, ms_1.default)(delay)} jitter`);
this.demotionTimer = timers.setTimeout(() => {
this.demotionTimer = undefined;
void queueCheckQueue.enqueue(this, 'live-count demotion', epoch, () => this.demote(epoch));
}, delay);
}
async demote(epoch) {
if (!this.isCurrent(epoch) || this.mode !== 'full' ||
(this.activeListener?.snapshots.size ?? 0) <= exports.settings.safeQueueSize)
return;
const lastFullSize = this.activeListener?.snapshots.size ?? 0;
if (!await this.loadListener('safe', epoch, 'demotion'))
return;
this.stats.size = lastFullSize;
delete this.stats.sizeDelta;
this.stats.sizeTimestamp = Date.now();
console.log(`Queue worker ${this.ref} demoted to safe mode with` +
` ${this.activeListener?.snapshots.size ?? 0} buffered tasks`);
this.scheduleSafeCheck();
}
async loadListener(mode, epoch, description) {
let details = { description, mode, timeout: queueLoadTimeout };
const result = await this.replaceListener(mode, epoch);
if (result === 'loaded') {
if (mode === 'full')
this.promotionNotBeforeTimestamp = 0;
return mode;
}
if (result === 'stopped')
return;
if (mode === 'full') {
this.reportError('queue-load-timeout', 'Firelease queue load timed out', 'warning', details);
this.promotionNotBeforeTimestamp = performance.now() + queueCheckInterval * 3;
const fallbackResult = await this.replaceListener('safe', epoch);
if (fallbackResult === 'loaded')
return 'safe';
if (fallbackResult === 'stopped')
return;
details = { description: `${description} fallback`, mode: 'safe', timeout: queueLoadTimeout };
}
this.crash('queue-load-timeout', 'Firelease queue load timed out', new Error('timeout'), details);
}
async replaceListener(mode, epoch) {
if (!this.isCurrent(epoch))
return 'stopped';
this.activeListener?.stop();
this.activeListener = undefined;
this.clearTasks();
if (mode === 'full') {
this.checkTimer?.clear();
this.checkTimer = undefined;
}
else {
this.demotionTimer?.clear();
this.demotionTimer = undefined;
}
const listener = new QueueListener(this, mode);
this.activeListener = listener;
listener.start();
const result = await listener.waitForLoad(queueLoadTimeout);
if (result !== 'loaded' || !this.isCurrent(epoch) || this.activeListener !== listener) {
listener.stop();
if (this.activeListener === listener) {
this.activeListener = undefined;
this.clearTasks();
}
return result === 'timed-out' && this.isCurrent(epoch) ? 'timed-out' : 'stopped';
}
this.mode = mode;
this.stats.mode = mode;
listener.notify();
return 'loaded';
}
cancelPendingWork() {
this.checkTimer?.clear();
this.checkTimer = undefined;
this.demotionTimer?.clear();
this.demotionTimer = undefined;
this.promotionNotBeforeTimestamp = 0;
}
async checkPing() {
const startedAt = performance.now();
const timestamp = Date.now();
const pingRef = this.ref.child(PING_KEY);
let pingFree = false;
try {
await pingRef.transaction(item => {
pingFree = !item;
return item ?? { timestamp, _lease: { expiry: nodefire_1.default.SERVER_TIMESTAMP } };
}, { prefetchValue: false, timeout: (0, ms_1.default)('10s') });
}
catch (error) {
this.recordPingResult(startedAt, false);
throw error;
}
if (!pingFree)
return; // another process is currently pinging
try {
await waitUntilDeleted(pingRef, this.queue.options.healthyPingLatency + (0, ms_1.default)('10s'));
}
catch {
this.recordPingResult(startedAt, false);
return;
}
this.recordPingResult(startedAt, true);
}
recordPingResult(startedAt, succeeded) {
const latency = Math.round(performance.now() - startedAt);
this.stats.latency = latency;
this.stats.healthy = succeeded && latency < this.queue.options.healthyPingLatency;
this.stats.pingTimestamp = Date.now();
}
reportError(code, message, level, details = {}) {
const error = new Error(message);
error.level = level;
error.firelease = {
...details, code, phase: 'queue-sizing', queue: this.queue.ref.toString(),
source: this.ref.toString()
};
if (level === 'error')
console.error(message, error.firelease);
else
console.warn(message, error.firelease);
exports.settings.captureError(error);
}
crash(code, message, cause, details = {}) {
if (this.crashing)
return;
this.crashing = true;
this.cancelPendingWork();
const error = new Error(message);
error.level = 'fatal';
error.firelease = {
...details, cause: cause.message, code, phase: 'crashing',
queue: this.queue.ref.toString(), source: this.ref.toString()
};
console.error(message, error.firelease);
exports.settings.captureError(error);
// Give the error capture a chance to process before exiting.
this.exitTimer = timers.setTimeout(() => { process.exit(1); }, (0, ms_1.default)('3s'));
}
addTask(snap) {
const taskKey = Task.makeKey(snap);
let task = tasks[taskKey];
if (blacklistedTaskKeys.has(taskKey)) {
if (task)
this.removeTask(taskKey);
return;
}
if (task) {
task.updateFrom(snap);
}
else {
task = tasks[taskKey] = new Task(this, snap);
}
void this.queue.process(task);
}
removeTask(snapOrKey) {
const taskKey = lodash_1.default.isString(snapOrKey) ? snapOrKey : Task.makeKey(snapOrKey);
const task = tasks[taskKey];
if (task?.source !== this)
return;
task.removed = true;
if (task.timeout) {
task.timeout.clear();
delete task.timeout;
}
if (!task.working)
delete tasks[taskKey];
}
clearTasks() {
lodash_1.default.forEach(tasks, (task, taskKey) => {
if (task.source === this)
this.removeTask(taskKey);
});
}
}
class Queue {
constructor(refOrRefs, options, worker) {
this.numConcurrent = 0;
if (lodash_1.default.isFunction(options)) {
worker = options;
options = {};
}
const refs = (0, lodash_1.default)(refOrRefs).castArray().uniqBy(item => item.toString()).value();
if (!refs.length)
throw new Error('At least one queue ref is required');
this.refs = refs;
this.ref = this.refs[0];
const filledOptions = lodash_1.default.defaults({}, options, firelease.defaults);
filledOptions.minLease = duration(filledOptions.minLease);
filledOptions.maxLease = duration(filledOptions.maxLease);
filledOptions.healthyPingLatency = duration(filledOptions.healthyPingLatency);
this.options = filledOptions;
this.worker = worker;
this.sources = lodash_1.default.map(this.refs, sourceRef => new QueueSource(this, sourceRef));
this.stats = new stats_1.QueueStats(this.ref.toString(), this.ref.key, lodash_1.default.map(this.sources, 'stats'));
exports.stats.queues.push(this.stats);
// Need each queue's scan function to be debounced separately.
this.scan = lodash_1.default.debounce(this.scan.bind(this), 100);
}
start() {
lodash_1.default.forEach(this.sources, source => { source.start(); });
}
reset() {
this.scan.cancel();
lodash_1.default.forEach(this.sources, source => { source.reset(); });
}
scan() {
lodash_1.default.forEach(tasks, task => {
if (task.queue === this)
void this.process(task);
});
}
hasQuota() {
return this.numConcurrent < this.options.maxConcurrent &&
globalNumConcurrent < globalMaxConcurrent;
}
constrainLeaseDuration(time) {
return Math.min(this.options.maxLease, Math.max(time, this.options.minLease));
}
async process(task) {
if (task.source.connected && this.hasQuota() && task.prepare()) {
globalNumConcurrent++;
this.numConcurrent++;
try {
await task.process();
globalNumConcurrent--;
this.numConcurrent--;
if (task.removed)
delete tasks[task.key];
if (globalNumConcurrent === globalMaxConcurrent - 1) {
scanAll();
}
else if (this.numConcurrent === this.options.maxConcurrent - 1) {
this.scan();
}
if (!globalMaxConcurrent && !globalNumConcurrent)
shutdownResolve?.();
if (!globalMaxConcurrent) {
console.log(`Queues draining, tasks in progress: ${globalNumConcurrent}`);
}
}
catch (error) {
error.message = `Unexpected error in Queue.process: ${error.message}`;
exports.settings.captureError(error);
}
}
}
callPreprocess(item) {
if (this.options.preprocess)
item = this.options.preprocess(item);
return item;
}
async callWorker(item) {
return this.worker(item);
}
}
function attachWorker(refOrRefs, options, worker) {
const queue = new Queue(refOrRefs, options, worker);
queues.push(queue);
queue.start();
}
function duration(value) {
if (lodash_1.default.isNumber(value))
return value;
return (0, ms_1.default)(value);
}
function fullQueueSize() {
return exports.settings.safeQueueSize * (1 - QUEUE_SIZE_HYSTERESIS);
}
let pinging = false;
let pingIntervalHandle;
let pingCallback;
/**
* Sets up regular pinging of all queues. Can be called either before or after workers are
* attached, and will always ping all queues. Can be called more than once to change the
* parameters.
*
* All durations can be specified as either a human-readable string, or a number of milliseconds.
*
* @param {Function(Object) | null} callback The callback to invoke with the live `stats` object
* each time we ping all the queues. It retains the existing global fields and adds
* structured results for every logical queue and physical source. If not specified,
* reports are silently dropped.
* @param {number | string} interval The interval at which to ping queues, to both check the
* current response latency and make sure no tasks are stuck. Defaults to 1 minute.
*/
function pingQueues(callback, interval) {
const normalizedInterval = interval ? duration(interval) : PING_INTERVAL;
pingIntervalHandle?.clear();
pingCallback = callback;
pingIntervalHandle = timers.setInterval(() => {
void runPingCheck();
}, normalizedInterval);
}
async function runPingCheck() {
try {
await checkPings();
}
catch (error) {
error.firelease = lodash_1.default.assign(error.firelease ?? {}, { phase: 'pinging' });
error.level = 'warning';
exports.settings.captureError(error);
pinging = false;
}
}
async function checkPings() {
if (pinging)
return;
pinging = true;
await Promise.all((0, lodash_1.default)(queues)
.flatMap(queue => queue.sources)
.map(source => source.checkPing())
.value());
// Backup scan in case tasks are stuck on a queue due to bugs.
scanAll();
pingCallback?.(exports.stats);
pinging = false;
}
function waitUntilDeleted(ref, timeout) {
return new Promise((resolve, reject) => {
let settled = false;
let timeoutHandle; // eslint-disable-line prefer-const
function finish(error) {
if (settled)
return;
settled = true;
timeoutHandle?.clear();
ref.off('value', onValue);
if (error)
reject(error);
else
resolve();
}
function onValue(snap) {
if (snap.val())
return;
finish();
}
timeoutHandle = timeout ?
timers.setTimeout(() => { finish(new Error('timeout')); }, timeout) : undefined;
try {
ref.on('value', onValue, finish);
}
catch (error) {
finish(error);
}
});
}
/**
* Extends the lease on a task to give the worker more time to finish. Checks a bunch of validity
* constraints along the way and throws an error if the worker needs to abort.
*
* All durations can be specified as either a human-readable string, or a number of milliseconds.
*
* @param {Object} item The original task object provided to a worker function.
* @param {number | string} timeNeeded The minimum time needed counting from the current time. The
actual lease may be extended by up to twice this amount, to prevent excessive churn.
* @return {Promise} A promise that will be resolved when the lease has been extended, and rejected
* if something went wrong and the worker should abort.
*/
function extendLease(item, timeNeeded) {
if (!item?._lease?.expiry)
throw new Error('Invalid task');
item._lease.timeNeeded = Math.max(item._lease.timeNeeded ?? 0, duration(timeNeeded));
if (!item._lease.extendLeasePromise) {
if (!globalMaxConcurrent)
return Promise.reject(new Error('shutdown in progress'));
item._lease.extendLeasePromise = updateLease(item, timeNeeded);
}
return item._lease.extendLeasePromise;
}
async function updateLease(item, timeNeeded) {
let error;
let timeNeededUsed = null;
const itemValue = await item.$ref.transaction(currentValue => {
let currentItem = currentValue;
error = undefined;
timeNeededUsed = null;
const now = item.$ref.now;
if (!currentItem) {
error = new Error('Task disappeared, unable to extend lease.');
error.firelease = { code: 'gone' };
currentItem = null; // make sure we attempt a write to force sha check
}
else if (!currentItem._lease) {
error = new Error('Task recreated, unable to extend lease.');
error.firelease = { code: 'recreated' };
}
else if (item._lease.expiry !== currentItem._lease.expiry) {
error = new Error('Task leased by another worker, unable to extend lease.');
error.firelease = { code: 'stolen' };
}
else if (currentItem._lease.expiry <= now) {
error = new Error('Lease expired, unable to extend.');
error.firelease = { code: 'lost' };
}
else {
const currentLease = currentItem._lease;
timeNeededUsed = item._lease.timeNeeded ?? 0;
// Expiry is monotonically increasing, so safe to do early abort if it's high enough.
if (currentLease.expiry >= now + timeNeededUsed)
return;
currentLease.expiry += timeNeededUsed;
}
return currentItem;
}, { prefetchValue: false });
const currentItem = itemValue;
const activeLease = item._lease;
const moreTimeNeeded = (activeLease?.timeNeeded ?? 0) > (timeNeededUsed ?? 0) ?
activeLease?.timeNeeded : undefined;
if (activeLease) {
delete activeLease.extendLeasePromise;
delete activeLease.timeNeeded;
}
if (error) {
error.firelease = lodash_1.default.assign(error.firelease ?? {}, { itemKey: item.$ref.toString(), timeNeeded });
throw error;
}
if (currentItem && activeLease) {
activeLease.expiry = currentItem._lease.expiry;
}
if (moreTimeNeeded) {
// If an extendLease raced with the transaction then retry it.
await firelease.extendLease(item, moreTimeNeeded);
}
}
/**
* Blacklist the given task key from ever being processed again.
* @param {string} taskKey The task key to blacklist. This is the full Firebase URL of the task and
* can be obtained from an error using `error.firelease.itemKey`.
* @return {boolean} True if the task key was added to the list, false if it was already present.
*/
function blacklist(taskKey) {
if (blacklistedTaskKeys.has(taskKey))
return false;
blacklistedTaskKeys.add(taskKey);
const task = tasks[taskKey];
if (task)
task.source.removeTask(taskKey);
return true;
}
/**
* Shuts down firelease by refusing to take new tasks.
* @return {Promise<void>} A promise that resolves when the shutdown is complete.
*/
function shutdown() {
globalMaxConcurrent = 0;
if (!shutdownPromise) {
shutdownPromise = new Promise((resolve, reject) => {
shutdownResolve = resolve;
shutdownReject = reject;
});
}
if (!globalNumConcurrent)
shutdownResolve?.();
return shutdownPromise;
}
/**
* Lists the URLs of all tasks that are currently being worked on.
*/
function listTasksInProgress() {
return (0, lodash_1.default)(tasks).pickBy('working').keys().value();
}
function resetBetweenTests() {
scanAll.cancel();
pingIntervalHandle?.clear();
pingIntervalHandle = undefined;
pingCallback = undefined;
pinging = false;
lodash_1.default.forEach(queues, queue => { queue.reset(); });
queueCheckQueue.reset();
lodash_1.default.forEach(tasks, (task, taskKey) => {
task.timeout?.clear();
delete tasks[taskKey];
});
queues.length = 0;
exports.stats.queues.length = 0;
blacklistedTaskKeys.clear();
globalMaxConcurrent = Number.MAX_VALUE;
globalNumConcurrent = 0;
safeQueueSize = 6000;
queueCheckInterval = (0, ms_1.default)('5m');
queueLoadTimeout = (0, ms_1.default)('1m');
shutdownResolve = shutdownReject = shutdownPromise = undefined;
delete exports.defaults.preprocess;
delete exports.defaults.captureLeaseTransactionMetrics;
lodash_1.default.assign(exports.defaults, {
maxConcurrent: Number.MAX_VALUE,
bufferSize: Infinity,
minLease: '30s',
maxLease: '1h',
healthyPingLatency: '1.5s'
});
exports.settings.captureError = defaultCaptureError;
}
exports.default = firelease;
//# sourceMappingURL=firelease.js.map