@net3/queuer
Version:
537 lines (536 loc) • 24.8 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.QueueManager = void 0;
const pieces_framework_1 = require("@activepieces/pieces-framework");
/**
* Central utility handling all queue storage and scheduling logic.
* All data is persisted via the piece `context.store` (PROJECT scope).
*/
class QueueManager {
/* ---------------------------------------------------------------------- */
/* Helpers – configuration, state & usage */
/* ---------------------------------------------------------------------- */
static async getQueueConfiguration(context, queueId) {
return await context.store.get(`queue_config_${queueId}`, pieces_framework_1.StoreScope.PROJECT);
}
/**
* NOTE: Activepieces key-value store cannot list keys, so a full “list queues”
* is not possible without maintaining an external index. Stub returns [].
*/
static async listAllQueues( /* context: any */) {
return [];
}
static async getQueueState(context, queueId) {
const key = `queue_state_${queueId}`;
return ((await context.store.get(key, pieces_framework_1.StoreScope.PROJECT)) || {
queueId,
lastReleaseTime: 0,
itemCount: 0,
currentExecutingItem: null,
lastExecutedTime: 0,
version: 0,
});
}
static async getQueueUsage(context, queueId) {
const now = new Date();
const dateKey = now.toISOString().split('T')[0];
const hourKey = now.toISOString().slice(0, 13);
const key = `queue_usage_${queueId}`;
const usage = (await context.store.get(key, pieces_framework_1.StoreScope.PROJECT)) || {
date: dateKey,
hour: hourKey,
dailyCount: 0,
hourlyCount: 0,
};
// daily rollover
if (usage.date !== dateKey) {
usage.date = dateKey;
usage.dailyCount = 0;
}
// hourly rollover
if (usage.hour !== hourKey) {
usage.hour = hourKey;
usage.hourlyCount = 0;
}
return usage;
}
/* ---------------------------------------------------------------------- */
/* Status & rate-limit helpers */
/* ---------------------------------------------------------------------- */
static async getQueueStatus(context, queueId) {
const queue = await this.getQueueConfiguration(context, queueId);
if (!queue)
return null;
const state = await this.getQueueState(context, queueId);
const usage = await this.getQueueUsage(context, queueId);
const isWithinActiveHours = queue.activeHours
? this.isWithinActiveHours(queue.activeHours)
: true;
const nextActiveWindow = queue.activeHours && !isWithinActiveHours
? this.getNextActiveWindow(queue.activeHours)
: undefined;
return {
pendingItems: state.itemCount,
currentlyExecuting: state.currentExecutingItem,
lastExecutedTime: state.lastExecutedTime
? new Date(state.lastExecutedTime).toISOString()
: null,
dailyUsed: usage.dailyCount,
dailyLimit: queue.dailyLimit,
dailyRemaining: queue.dailyLimit > 0 ? queue.dailyLimit - usage.dailyCount : null,
hourlyUsed: usage.hourlyCount,
hourlyLimit: queue.hourlyLimit,
hourlyRemaining: queue.hourlyLimit > 0 ? queue.hourlyLimit - usage.hourlyCount : null,
isWithinActiveHours,
nextActiveWindow,
};
}
/** Check daily/hourly rate limits and return availability. */
static async checkRateLimits(context, queueId) {
const queue = await this.getQueueConfiguration(context, queueId);
if (!queue)
return { canSend: true };
if (queue.dailyLimit === 0 && queue.hourlyLimit === 0)
return { canSend: true };
const usage = await this.getQueueUsage(context, queueId);
const now = new Date();
// daily
if (queue.dailyLimit > 0 && usage.dailyCount >= queue.dailyLimit) {
const tomorrow = new Date(now);
tomorrow.setDate(tomorrow.getDate() + 1);
tomorrow.setHours(0, 0, 0, 0);
return {
canSend: false,
nextAvailableTime: tomorrow.getTime(),
dailyRemaining: 0,
hourlyRemaining: queue.hourlyLimit > 0 ? queue.hourlyLimit - usage.hourlyCount : 0,
};
}
// hourly
if (queue.hourlyLimit > 0 && usage.hourlyCount >= queue.hourlyLimit) {
const nextHour = new Date(now);
nextHour.setHours(nextHour.getHours() + 1, 0, 0, 0);
return {
canSend: false,
nextAvailableTime: nextHour.getTime(),
dailyRemaining: queue.dailyLimit > 0 ? queue.dailyLimit - usage.dailyCount : undefined,
hourlyRemaining: 0,
};
}
return {
canSend: true,
dailyRemaining: queue.dailyLimit > 0 ? queue.dailyLimit - usage.dailyCount : undefined,
hourlyRemaining: queue.hourlyLimit > 0 ? queue.hourlyLimit - usage.hourlyCount : undefined,
};
}
/* ---------------------------------------------------------------------- */
/* Delay calculations */
/* ---------------------------------------------------------------------- */
static calculateDelayMs(delayType, delayUnit, delayValue, delayMin, delayMax) {
const mult = {
seconds: 1000,
minutes: 60 * 1000,
hours: 60 * 60 * 1000,
days: 24 * 60 * 60 * 1000,
};
if (delayType === 'fixed') {
return Math.floor((delayValue || 0) * mult[delayUnit]);
}
const min = delayMin || 0;
const max = delayMax || 0;
const rnd = Math.random() * (max - min) + min;
return Math.floor(rnd * mult[delayUnit]);
}
static async calculateItemDelay(context, queue, limits) {
let baseDelay = this.calculateDelayMs(queue.delayType, queue.delayUnit, queue.delayValue, queue.delayMin, queue.delayMax);
let limitDelay = 0;
let activeHoursDelay = 0;
if (!limits.canSend && limits.nextAvailableTime) {
const now = Date.now();
limitDelay = Math.max(0, limits.nextAvailableTime - now);
baseDelay = Math.max(baseDelay, limitDelay);
}
if (queue.activeHours) {
// First check if we're currently within active hours
const isCurrentlyActive = this.isWithinActiveHours(queue.activeHours);
if (!isCurrentlyActive) {
// Only apply active hours delay if we're outside active hours
const nextActive = this.getNextActiveTime(queue.activeHours);
const now = Date.now();
if (nextActive > now) {
activeHoursDelay = nextActive - now;
baseDelay = Math.max(baseDelay, activeHoursDelay);
}
}
// If we're within active hours, no additional delay needed
}
return { delayMs: baseDelay, limitDelay, activeHoursDelay };
}
/* ---------------------------------------------------------------------- */
/* Active-hours helpers */
/* ---------------------------------------------------------------------- */
static isWithinActiveHours(activeHours) {
if (!activeHours?.schedule)
return true;
const now = new Date();
const tz = activeHours.timezone || 'UTC';
const local = new Date(now.toLocaleString('en-US', { timeZone: tz }));
const day = local
.toLocaleDateString('en-US', { weekday: 'long' })
.toLowerCase();
const time = local.toTimeString().slice(0, 5); // HH:MM
const sched = activeHours.schedule[day];
if (!sched || !sched.enabled)
return false;
if (sched.start && sched.end) {
return time >= sched.start && time <= sched.end;
}
return true;
}
static getNextActiveTime(activeHours) {
if (!activeHours?.schedule)
return Date.now();
const now = new Date();
const tz = activeHours.timezone || 'UTC';
for (let i = 0; i < 7; i++) {
const cand = new Date(now);
cand.setDate(cand.getDate() + i);
const local = new Date(cand.toLocaleString('en-US', { timeZone: tz }));
const day = local
.toLocaleDateString('en-US', { weekday: 'long' })
.toLowerCase();
const sched = activeHours.schedule[day];
if (sched && sched.enabled && sched.start) {
const [h, m] = sched.start.split(':').map(Number);
cand.setHours(h, m, 0, 0);
if (cand.getTime() > now.getTime())
return cand.getTime();
}
}
return now.getTime() + 24 * 60 * 60 * 1000;
}
static getNextActiveWindow(activeHours) {
return new Date(this.getNextActiveTime(activeHours)).toISOString();
}
/* ---------------------------------------------------------------------- */
/* Core scheduling – add, validate, update, cleanup */
/* ---------------------------------------------------------------------- */
static async addItemToQueue(context, queueId, targetAction, actionConfig, delayMs) {
const MAX_RETRIES = 5;
let attempt = 0;
const queue = await this.getQueueConfiguration(context, queueId);
if (!queue)
throw new Error(`Queue not found: ${queueId}`);
while (attempt < MAX_RETRIES) {
try {
const ts = Date.now();
const rand = Math.random().toString(36).substr(2, 9);
const queueItemId = `${queueId}_${ts}_${rand}`;
const stateKey = `queue_state_${queueId}`;
const currentState = (await context.store.get(stateKey, pieces_framework_1.StoreScope.PROJECT)) || {
queueId,
lastReleaseTime: 0,
itemCount: 0,
currentExecutingItem: null,
lastExecutedTime: 0,
version: 0,
};
// Get actual latest release time from queued items (FIFO)
const itemsListKey = `queue_${queueId}_items`;
const itemIds = (await context.store.get(itemsListKey, pieces_framework_1.StoreScope.PROJECT)) || [];
let latestReleaseTime = 0;
for (const itemId of itemIds) {
const item = await this.getQueueItem(context, itemId);
if (item && item.status === 'scheduled' && item.releaseTime > latestReleaseTime) {
latestReleaseTime = item.releaseTime;
}
}
const earliest = ts + delayMs;
const nextRelease = Math.max(earliest, latestReleaseTime + delayMs);
const uniqueOffset = Math.floor(Math.random() * 1000);
let finalReleaseTime = nextRelease + uniqueOffset;
// Apply active hours constraints AFTER calculating queue position
if (queue.activeHours) {
const isCurrentlyActive = this.isWithinActiveHours(queue.activeHours);
if (!isCurrentlyActive) {
// Only push to next active window if we're outside active hours
const nextActive = this.getNextActiveTime(queue.activeHours);
if (nextActive > finalReleaseTime) {
finalReleaseTime = nextActive;
}
}
// If we're within active hours, keep the calculated time
}
// slot reservation
const slotKey = `queue_time_${queueId}_${finalReleaseTime}`;
if (await context.store.get(slotKey, pieces_framework_1.StoreScope.PROJECT)) {
attempt++;
await new Promise((r) => setTimeout(r, 50 * attempt));
continue;
}
await context.store.put(slotKey, queueItemId, pieces_framework_1.StoreScope.PROJECT);
const queueItem = {
id: queueItemId,
queueId,
releaseTime: finalReleaseTime,
targetAction,
actionConfig,
status: 'scheduled',
createdAt: ts,
};
await context.store.put(`queue_item_${queueItemId}`, queueItem, pieces_framework_1.StoreScope.PROJECT);
const newState = {
...currentState,
lastReleaseTime: finalReleaseTime,
itemCount: currentState.itemCount + 1,
version: currentState.version + 1,
};
const verify = await context.store.get(stateKey, pieces_framework_1.StoreScope.PROJECT);
if (verify && verify.version !== currentState.version) {
// optimistic lock failed – cleanup
await context.store.delete(`queue_item_${queueItemId}`, pieces_framework_1.StoreScope.PROJECT);
await context.store.delete(slotKey, pieces_framework_1.StoreScope.PROJECT);
attempt++;
await new Promise((r) => setTimeout(r, 100 * attempt));
continue;
}
await context.store.put(stateKey, newState, pieces_framework_1.StoreScope.PROJECT);
// track item list
const list = (await context.store.get(itemsListKey, pieces_framework_1.StoreScope.PROJECT)) || [];
list.push(queueItemId);
await context.store.put(itemsListKey, list, pieces_framework_1.StoreScope.PROJECT);
return { queueItemId, releaseTime: finalReleaseTime };
}
catch (err) {
attempt++;
if (attempt >= MAX_RETRIES)
throw new Error(`Failed to add item after ${MAX_RETRIES} attempts: ${err.message}`);
await new Promise((r) => setTimeout(r, 100 * attempt));
}
}
throw new Error('Failed to add item to queue');
}
static async getQueueItem(context, queueItemId) {
return await context.store.get(`queue_item_${queueItemId}`, pieces_framework_1.StoreScope.PROJECT);
}
static async updateItemStatus(context, queueItemId, status, error) {
const item = await this.getQueueItem(context, queueItemId);
if (item) {
item.status = status;
if (error)
item.lastError = error;
await context.store.put(`queue_item_${queueItemId}`, item, pieces_framework_1.StoreScope.PROJECT);
}
}
static async validateQueuePosition(context, queueItemId) {
const item = await this.getQueueItem(context, queueItemId);
if (!item)
return { isValid: false, shouldRepause: false, reason: 'Item not found' };
const now = Date.now();
const state = await this.getQueueState(context, item.queueId);
if (state.currentExecutingItem &&
state.currentExecutingItem !== queueItemId) {
return {
isValid: false,
shouldRepause: true,
newReleaseTime: now + 5000,
reason: 'Another item executing',
};
}
const nextOk = await this.checkIfNextInQueue(context, item);
if (!nextOk) {
const queue = await this.getQueueConfiguration(context, item.queueId);
if (!queue)
return {
isValid: false,
shouldRepause: false,
reason: 'Queue not found',
};
const delayMs = this.calculateDelayMs(queue.delayType, queue.delayUnit, queue.delayValue, queue.delayMin, queue.delayMax);
const newRelease = state.lastReleaseTime + delayMs;
item.releaseTime = newRelease;
await context.store.put(`queue_item_${item.id}`, item, pieces_framework_1.StoreScope.PROJECT);
return {
isValid: false,
shouldRepause: true,
newReleaseTime: newRelease,
reason: 'Not next in queue',
};
}
if (now >= item.releaseTime) {
state.currentExecutingItem = queueItemId;
await context.store.put(`queue_state_${item.queueId}`, state, pieces_framework_1.StoreScope.PROJECT);
return { isValid: true, shouldRepause: false };
}
return {
isValid: false,
shouldRepause: true,
newReleaseTime: item.releaseTime,
reason: 'Not time yet',
};
}
static async checkIfNextInQueue(context, currentItem) {
const list = (await context.store.get(`queue_${currentItem.queueId}_items`, pieces_framework_1.StoreScope.PROJECT)) || [];
for (const id of list) {
if (id === currentItem.id)
continue;
const it = await this.getQueueItem(context, id);
if (it && it.status === 'scheduled' && it.releaseTime < currentItem.releaseTime)
return false;
}
return true;
}
static async markExecutionComplete(context, queueId, queueItemId) {
const state = await this.getQueueState(context, queueId);
state.currentExecutingItem = null;
state.lastExecutedTime = Date.now();
state.itemCount = Math.max(0, state.itemCount - 1);
await context.store.put(`queue_state_${queueId}`, state, pieces_framework_1.StoreScope.PROJECT);
const listKey = `queue_${queueId}_items`;
const list = (await context.store.get(listKey, pieces_framework_1.StoreScope.PROJECT)) || [];
await context.store.put(listKey, list.filter((id) => id !== queueItemId), pieces_framework_1.StoreScope.PROJECT);
}
static async cleanupQueueItem(context, queueId, queueItemId, releaseTime) {
await context.store.delete(`queue_time_${queueId}_${releaseTime}`, pieces_framework_1.StoreScope.PROJECT);
await context.store.delete(`queue_item_${queueItemId}`, pieces_framework_1.StoreScope.PROJECT);
}
static async incrementUsageCounters(context, queueId) {
const usage = await this.getQueueUsage(context, queueId);
usage.dailyCount++;
usage.hourlyCount++;
await context.store.put(`queue_usage_${queueId}`, usage, pieces_framework_1.StoreScope.PROJECT);
const queue = await this.getQueueConfiguration(context, queueId);
if (queue) {
queue.totalProcessed++;
queue.lastUsed = Date.now();
await context.store.put(`queue_config_${queueId}`, queue, pieces_framework_1.StoreScope.PROJECT);
}
}
/* ---------------------------------------------------------------------- */
/* Queue Management Operations */
/* ---------------------------------------------------------------------- */
/**
* Get all items in a queue with their details
*/
static async listQueueItems(context, queueId) {
const listKey = `queue_${queueId}_items`;
const itemIds = (await context.store.get(listKey, pieces_framework_1.StoreScope.PROJECT)) || [];
const items = [];
for (const itemId of itemIds) {
const item = await this.getQueueItem(context, itemId);
if (item) {
items.push(item);
}
}
// Sort by release time
return items.sort((a, b) => a.releaseTime - b.releaseTime);
}
/**
* Clear all items from a queue
*/
static async clearQueue(context, queueId) {
if (!queueId) {
throw new Error('Queue ID is required for clearing queue');
}
if (!context) {
throw new Error('Context is required for clearing queue');
}
try {
const listKey = `queue_${queueId}_items`;
let itemIds;
try {
itemIds = (await context.store.get(listKey, pieces_framework_1.StoreScope.PROJECT)) || [];
}
catch (error) {
throw new Error(`Failed to get queue items list: ${error.message}`);
}
let clearedCount = 0;
const errors = [];
// Delete all queue items and their time slots
for (const itemId of itemIds) {
try {
const item = await this.getQueueItem(context, itemId);
if (item) {
// Delete the item
try {
await context.store.delete(`queue_item_${itemId}`, pieces_framework_1.StoreScope.PROJECT);
}
catch (error) {
errors.push(`Failed to delete item ${itemId}: ${error.message}`);
continue;
}
// Delete the time slot reservation
try {
const slotKey = `queue_time_${queueId}_${item.releaseTime}`;
await context.store.delete(slotKey, pieces_framework_1.StoreScope.PROJECT);
}
catch (error) {
errors.push(`Failed to delete time slot for item ${itemId}: ${error.message}`);
}
clearedCount++;
}
}
catch (error) {
errors.push(`Failed to process item ${itemId}: ${error.message}`);
}
}
// Clear the items list
try {
await context.store.delete(listKey, pieces_framework_1.StoreScope.PROJECT);
}
catch (error) {
throw new Error(`Failed to clear items list: ${error.message}`);
}
// Reset queue state
const stateKey = `queue_state_${queueId}`;
try {
const currentState = (await context.store.get(stateKey, pieces_framework_1.StoreScope.PROJECT)) || {
queueId,
lastReleaseTime: 0,
itemCount: 0,
currentExecutingItem: null,
lastExecutedTime: 0,
version: 0,
};
const resetState = {
...currentState,
itemCount: 0,
currentExecutingItem: null,
lastReleaseTime: 0, // Reset this so new items start fresh
version: currentState.version + 1,
};
await context.store.put(stateKey, resetState, pieces_framework_1.StoreScope.PROJECT);
}
catch (error) {
throw new Error(`Failed to reset queue state: ${error.message}`);
}
// If there were partial errors, include them in the response
if (errors.length > 0) {
throw new Error(`Queue cleared with ${errors.length} errors: ${errors.join('; ')}`);
}
return {
clearedCount,
queueId,
};
}
catch (error) {
throw new Error(`Clear queue operation failed: ${error.message}`);
}
}
/**
* Get complete queue information including configuration and status
*/
static async getQueueDetails(context, queueId) {
const configuration = await this.getQueueConfiguration(context, queueId);
const status = await this.getQueueStatus(context, queueId);
const state = await this.getQueueState(context, queueId);
const usage = await this.getQueueUsage(context, queueId);
return {
configuration,
status,
state,
usage,
};
}
}
exports.QueueManager = QueueManager;