pg-boss
Version:
Queueing jobs in Postgres from Node.js like a boss
254 lines (253 loc) • 16 kB
JavaScript
import assert from 'node:assert';
import { DEFAULT_SCHEMA } from "./plans.js";
const POLICY = {
MAX_EXPIRATION_HOURS: 24,
MIN_POLLING_INTERVAL_MS: 500,
MAX_RETENTION_DAYS: 365
};
function assertObjectName(value, name = 'Name') {
assert(/^[\w.\-/]+$/.test(value), `${name} can only contain alphanumeric characters, underscores, hyphens, periods, or forward slashes`);
}
function validateQueueArgs(config = {}) {
assert(!('deadLetter' in config) || config.deadLetter === null || (typeof config.deadLetter === 'string'), 'deadLetter must be a string');
if (config.deadLetter) {
assertObjectName(config.deadLetter, 'deadLetter');
}
validateRetryConfig(config);
validateExpirationConfig(config);
validateRetentionConfig(config);
validateDeletionConfig(config);
validateHeartbeatConfig(config);
}
function checkSendArgs(args) {
let name, data, options;
if (typeof args[0] === 'string') {
name = args[0];
data = args[1];
assert(typeof data !== 'function', 'send() cannot accept a function as the payload. Did you intend to use work()?');
options = args[2];
}
else if (typeof args[0] === 'object') {
assert(args.length === 1, 'send object API only accepts 1 argument');
const job = args[0];
assert(job, 'boss requires all jobs to have a name');
name = job.name;
data = job.data;
options = job.options;
}
options = options || {};
assert(name, 'boss requires all jobs to have a queue name');
assert(typeof options === 'object', 'options should be an object');
options = { ...options };
assert(!('priority' in options) || (Number.isInteger(options.priority)), 'priority must be an integer');
options.priority = options.priority || 0;
options.startAfter = (options.startAfter instanceof Date && typeof options.startAfter.toISOString === 'function')
? options.startAfter.toISOString()
: (+options.startAfter > 0)
? '' + options.startAfter
: (typeof options.startAfter === 'string')
? options.startAfter
: undefined;
validateRetryConfig(options);
validateExpirationConfig(options);
validateRetentionConfig(options);
validateDeletionConfig(options);
validateGroupConfig(options);
validateHeartbeatConfig(options);
return { name, data, options };
}
function validateGroupConfig(config) {
if (!('group' in config) || config.group === undefined || config.group === null) {
return;
}
assert(typeof config.group === 'object', 'group must be an object');
assert(typeof config.group.id === 'string' && config.group.id.length > 0, 'group.id must be a non-empty string');
assert(!('tier' in config.group) || (typeof config.group.tier === 'string' && config.group.tier.length > 0), 'group.tier must be a non-empty string if provided');
}
function validateGroupConcurrencyValue(value, optionName) {
if (typeof value === 'number') {
assert(Number.isInteger(value) && value >= 1, `${optionName} must be an integer >= 1`);
return;
}
assert(typeof value === 'object', `${optionName} must be a number or an object with { default, tiers? }`);
assert(Number.isInteger(value.default) && value.default >= 1, `${optionName}.default must be an integer >= 1`);
if ('tiers' in value && value.tiers) {
assert(typeof value.tiers === 'object', `${optionName}.tiers must be an object`);
for (const [tier, limit] of Object.entries(value.tiers)) {
assert(typeof tier === 'string' && tier.length > 0, `${optionName} tier keys must be non-empty strings`);
assert(Number.isInteger(limit) && limit >= 1, `${optionName}.tiers["${tier}"] must be an integer >= 1`);
}
}
}
function validatePriorityRangeConfig(config) {
if (config.minPriority !== undefined) {
assert(Number.isInteger(config.minPriority), 'minPriority must be an integer');
}
if (config.maxPriority !== undefined) {
assert(Number.isInteger(config.maxPriority), 'maxPriority must be an integer');
}
if (config.minPriority !== undefined && config.maxPriority !== undefined) {
assert(config.minPriority <= config.maxPriority, 'minPriority must be <= maxPriority');
}
}
function validateGroupConcurrencyConfig(config) {
const hasGlobal = config.groupConcurrency != null;
const hasLocal = config.localGroupConcurrency != null;
assert(!(hasGlobal && hasLocal), 'cannot specify both groupConcurrency and localGroupConcurrency - choose one');
if (hasGlobal)
validateGroupConcurrencyValue(config.groupConcurrency, 'groupConcurrency');
if (hasLocal) {
validateGroupConcurrencyValue(config.localGroupConcurrency, 'localGroupConcurrency');
validateLocalGroupConcurrencyLimit(config.localGroupConcurrency, config.localConcurrency);
}
}
function validateLocalGroupConcurrencyLimit(localGroupConcurrency, localConcurrency) {
const effectiveLocalConcurrency = localConcurrency ?? 1;
if (typeof localGroupConcurrency === 'number') {
assert(localGroupConcurrency <= effectiveLocalConcurrency, `localGroupConcurrency (${localGroupConcurrency}) cannot exceed localConcurrency (${effectiveLocalConcurrency})`);
}
else if (typeof localGroupConcurrency === 'object') {
assert(localGroupConcurrency.default <= effectiveLocalConcurrency, `localGroupConcurrency.default (${localGroupConcurrency.default}) cannot exceed localConcurrency (${effectiveLocalConcurrency})`);
if (localGroupConcurrency.tiers) {
for (const [tier, limit] of Object.entries(localGroupConcurrency.tiers)) {
assert(limit <= effectiveLocalConcurrency, `localGroupConcurrency.tiers["${tier}"] (${limit}) cannot exceed localConcurrency (${effectiveLocalConcurrency})`);
}
}
}
}
function checkWorkArgs(name, args) {
let options, callback;
assert(name, 'queue name is required');
if (args.length === 1) {
callback = args[0];
options = {};
}
else if (args.length > 1) {
options = args[0] || {};
callback = args[1];
}
assert(typeof callback === 'function', 'expected callback to be a function');
assert(typeof options === 'object', 'expected config to be an object');
options = { ...options };
applyPollingInterval(options);
assert(!('batchSize' in options) || (Number.isInteger(options.batchSize) && options.batchSize >= 1), 'batchSize must be an integer > 0');
assert(!('includeMetadata' in options) || typeof options.includeMetadata === 'boolean', 'includeMetadata must be a boolean');
assert(!('priority' in options) || typeof options.priority === 'boolean', 'priority must be a boolean');
assert(!('localConcurrency' in options) || (Number.isInteger(options.localConcurrency) && options.localConcurrency >= 1), 'localConcurrency must be an integer >= 1');
validatePriorityRangeConfig(options);
validateGroupConcurrencyConfig(options);
validateHeartbeatRefreshConfig(options);
return { options, callback };
}
function checkFetchArgs(name, options) {
assert(name, 'missing queue name');
assert(!('batchSize' in options) || (Number.isInteger(options.batchSize) && options.batchSize >= 1), 'batchSize must be an integer > 0');
assert(!('includeMetadata' in options) || typeof options.includeMetadata === 'boolean', 'includeMetadata must be a boolean');
assert(!('priority' in options) || typeof options.priority === 'boolean', 'priority must be a boolean');
assert(!('ignoreStartAfter' in options) || typeof options.ignoreStartAfter === 'boolean', 'ignoreStartAfter must be a boolean');
validatePriorityRangeConfig(options);
}
function getConfig(value) {
assert(value && (typeof value === 'object' || typeof value === 'string'), 'configuration assert: string or config object is required to connect to postgres');
const config = (typeof value === 'string')
? { connectionString: value }
: { ...value };
config.schedule = ('schedule' in config) ? config.schedule : true;
config.supervise = ('supervise' in config) ? config.supervise : true;
config.migrate = ('migrate' in config) ? config.migrate : true;
config.createSchema = ('createSchema' in config) ? config.createSchema : true;
applySchemaConfig(config);
applyOpsConfig(config);
applyScheduleConfig(config);
applyBamConfig(config);
validateWarningConfig(config);
return config;
}
function applySchemaConfig(config) {
if (config.schema) {
assertPostgresObjectName(config.schema);
}
config.schema = config.schema || DEFAULT_SCHEMA;
}
function validateWarningConfig(config) {
assert(!('warningQueueSize' in config) || config.warningQueueSize >= 1, 'configuration assert: warningQueueSize must be at least 1');
assert(!('warningSlowQuerySeconds' in config) || config.warningSlowQuerySeconds >= 1, 'configuration assert: warningSlowQuerySeconds must be at least 1');
assert(!('warningRetentionDays' in config) || (Number.isInteger(config.warningRetentionDays) && config.warningRetentionDays >= 1), 'configuration assert: warningRetentionDays must be an integer >= 1');
assert(!('warningRetentionDays' in config) || config.warningRetentionDays <= POLICY.MAX_RETENTION_DAYS, `configuration assert: warningRetentionDays cannot exceed ${POLICY.MAX_RETENTION_DAYS} days`);
}
function assertPostgresObjectName(name) {
assert(typeof name === 'string', 'Name must be a string');
assert(name.length <= 50, 'Name cannot exceed 50 characters');
assert(!/\W/.test(name), 'Name can only contain alphanumeric characters or underscores');
assert(!/^\d/.test(name), 'Name cannot start with a number');
}
function assertQueueName(name) {
assert(name, 'Name is required');
assert(typeof name === 'string', 'Name must be a string');
assertObjectName(name);
}
function assertKey(key) {
if (!key)
return;
assert(typeof key === 'string', 'Key must be a string');
assertObjectName(key, 'Key');
}
function validateRetentionConfig(config) {
assert(!('retentionSeconds' in config) || config.retentionSeconds >= 1, 'configuration assert: retentionSeconds must be at least every second');
}
function validateExpirationConfig(config) {
assert(!('expireInSeconds' in config) || config.expireInSeconds >= 1, 'configuration assert: expireInSeconds must be at least every second');
assert(!config.expireInSeconds || config.expireInSeconds / 60 / 60 < POLICY.MAX_EXPIRATION_HOURS, `configuration assert: expiration cannot exceed ${POLICY.MAX_EXPIRATION_HOURS} hours`);
}
function validateRetryConfig(config) {
assert(!('retryDelay' in config) || (Number.isInteger(config.retryDelay) && config.retryDelay >= 0), 'retryDelay must be an integer >= 0');
assert(!('retryLimit' in config) || (Number.isInteger(config.retryLimit) && config.retryLimit >= 0), 'retryLimit must be an integer >= 0');
assert(!('retryBackoff' in config) || (config.retryBackoff === true || config.retryBackoff === false), 'retryBackoff must be either true or false');
assert(!('retryDelayMax' in config) || config.retryDelayMax === null || config.retryBackoff === true, 'retryDelayMax can only be set if retryBackoff is true');
assert(!('retryDelayMax' in config) || config.retryDelayMax === null || (Number.isInteger(config.retryDelayMax) && config.retryDelayMax >= 0), 'retryDelayMax must be an integer >= 0');
}
function validateHeartbeatConfig(config) {
assert(!('heartbeatSeconds' in config) || config.heartbeatSeconds === null || (Number.isInteger(config.heartbeatSeconds) && config.heartbeatSeconds >= 10), 'heartbeatSeconds must be an integer >= 10');
}
function validateHeartbeatRefreshConfig(config) {
if (!('heartbeatRefreshSeconds' in config) || config.heartbeatRefreshSeconds == null)
return;
assert(typeof config.heartbeatRefreshSeconds === 'number' && config.heartbeatRefreshSeconds > 0, 'heartbeatRefreshSeconds must be a number > 0');
}
function applyPollingInterval(config) {
assert(!('pollingIntervalSeconds' in config) || config.pollingIntervalSeconds >= POLICY.MIN_POLLING_INTERVAL_MS / 1000, `configuration assert: pollingIntervalSeconds must be at least every ${POLICY.MIN_POLLING_INTERVAL_MS}ms`);
config.pollingInterval = ('pollingIntervalSeconds' in config)
? config.pollingIntervalSeconds * 1000
: 2000;
}
function applyOpsConfig(config) {
assert(!('superviseIntervalSeconds' in config) || config.superviseIntervalSeconds >= 1, 'configuration assert: superviseIntervalSeconds must be at least every second');
config.superviseIntervalSeconds = config.superviseIntervalSeconds || 60;
assert(config.superviseIntervalSeconds / 60 / 60 <= POLICY.MAX_EXPIRATION_HOURS, `configuration assert: superviseIntervalSeconds cannot exceed ${POLICY.MAX_EXPIRATION_HOURS} hours`);
assert(!('maintenanceIntervalSeconds' in config) || config.maintenanceIntervalSeconds >= 1, 'configuration assert: maintenanceIntervalSeconds must be at least every second');
config.maintenanceIntervalSeconds = config.maintenanceIntervalSeconds || POLICY.MAX_EXPIRATION_HOURS * 60 * 60;
assert(config.maintenanceIntervalSeconds / 60 / 60 <= POLICY.MAX_EXPIRATION_HOURS, `configuration assert: maintenanceIntervalSeconds cannot exceed ${POLICY.MAX_EXPIRATION_HOURS} hours`);
assert(!('monitorIntervalSeconds' in config) || config.monitorIntervalSeconds >= 1, 'configuration assert: monitorIntervalSeconds must be at least every second');
config.monitorIntervalSeconds = config.monitorIntervalSeconds || 60;
assert(config.monitorIntervalSeconds / 60 / 60 <= POLICY.MAX_EXPIRATION_HOURS, `configuration assert: monitorIntervalSeconds cannot exceed ${POLICY.MAX_EXPIRATION_HOURS} hours`);
assert(!('queueCacheIntervalSeconds' in config) || config.queueCacheIntervalSeconds >= 1, 'configuration assert: queueCacheIntervalSeconds must be at least every second');
config.queueCacheIntervalSeconds = config.queueCacheIntervalSeconds || 60;
assert(config.queueCacheIntervalSeconds / 60 / 60 <= POLICY.MAX_EXPIRATION_HOURS, `configuration assert: queueCacheIntervalSeconds cannot exceed ${POLICY.MAX_EXPIRATION_HOURS} hours`);
}
function validateDeletionConfig(config) {
assert(!('deleteAfterSeconds' in config) || config.deleteAfterSeconds >= 0, 'configuration assert: deleteAfterSeconds must be at least 0 (0 disables deletion)');
}
function applyScheduleConfig(config) {
assert(!('clockMonitorIntervalSeconds' in config) || (config.clockMonitorIntervalSeconds >= 1 && config.clockMonitorIntervalSeconds <= 600), 'configuration assert: clockMonitorIntervalSeconds must be between 1 second and 10 minutes');
config.clockMonitorIntervalSeconds = config.clockMonitorIntervalSeconds || 600;
assert(!('cronMonitorIntervalSeconds' in config) || (config.cronMonitorIntervalSeconds >= 1 && config.cronMonitorIntervalSeconds <= 45), 'configuration assert: cronMonitorIntervalSeconds must be between 1 and 45 seconds');
config.cronMonitorIntervalSeconds = config.cronMonitorIntervalSeconds || 30;
assert(!('cronWorkerIntervalSeconds' in config) || (config.cronWorkerIntervalSeconds >= 1 && config.cronWorkerIntervalSeconds <= 45), 'configuration assert: cronWorkerIntervalSeconds must be between 1 and 45 seconds');
config.cronWorkerIntervalSeconds = config.cronWorkerIntervalSeconds || 5;
}
function applyBamConfig(config) {
const minInterval = config.__test__bypass_bam_interval_check ? 1 : 10;
assert(!('bamIntervalSeconds' in config) || config.bamIntervalSeconds >= minInterval, `configuration assert: bamIntervalSeconds must be at least ${minInterval} seconds`);
config.bamIntervalSeconds = config.bamIntervalSeconds || 60;
}
export { assertKey, assertPostgresObjectName, assertQueueName, checkFetchArgs, checkSendArgs, checkWorkArgs, getConfig, POLICY, validateQueueArgs };