@coko/server
Version:
Reusable server for use by Coko's projects
346 lines • 13.9 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.jobManager = exports.JobManager = void 0;
const pg_boss_1 = require("pg-boss");
const cronstrue_1 = __importDefault(require("cronstrue"));
const cron_validator_1 = require("cron-validator");
const zod_1 = require("zod");
const isMatch_1 = __importDefault(require("lodash/isMatch"));
const db_1 = require("../db");
const logger_1 = __importDefault(require("../logger"));
const internals_1 = __importDefault(require("../logger/internals"));
const errors_1 = require("./errors");
const wait_1 = __importDefault(require("../utils/wait"));
const filesystem_1 = require("../utils/filesystem");
const defaultJobQueueNames_1 = __importDefault(require("./defaultJobQueueNames"));
const defaultJobQueues_1 = require("./defaultJobQueues");
const sendOptionsSchema = zod_1.z.strictObject({
startAfter: zod_1.z.number().int().positive().optional(),
});
const cron = zod_1.z
.string()
.refine(val => (0, cron_validator_1.isValidCron)(val, { alias: true, seconds: false }), {
message: 'Invalid cron string',
});
const JobQueueName = zod_1.z.string().refine(val => {
const reservedQueueNames = Object.keys(defaultJobQueueNames_1.default).map(key => defaultJobQueueNames_1.default[key]);
// reserving email, as it will be implemented
const disallowed = [...reservedQueueNames, 'email'];
return !disallowed.includes(val.toLowerCase());
}, {
message: 'The provided string is a reserved job queue name and is not allowed.',
});
const Timezone = zod_1.z
.string()
.optional()
.refine(val => {
if (!val)
return false;
try {
new Intl.DateTimeFormat('en-US', { timeZone: val }).format();
return true;
}
catch (_error) {
return false;
}
}, {
message: 'Not a valid timezone.',
});
const JobHandlerArgumentsSchema = zod_1.z.strictObject({
id: zod_1.z.string(),
name: zod_1.z.string(),
data: zod_1.z.any(),
});
const JobQueueSchema = zod_1.z.strictObject({
name: JobQueueName,
handler: zod_1.z.function().input([JobHandlerArgumentsSchema]),
batchSize: zod_1.z.number().int().positive().optional(),
concurrency: zod_1.z.number().int().positive().optional(),
schedule: cron.optional(),
scheduleTimezone: Timezone.optional(),
});
const JobQueuesArraySchema = zod_1.z.array(JobQueueSchema);
const newSchemaName = 'pgboss_v12';
class JobManager {
#boss;
// Exposing the boss instance can be used for testing
#exposeBossInstance;
constructor(options = {}) {
this.#exposeBossInstance = options.exposeBossInstance;
}
get boss() {
if (!this.#exposeBossInstance) {
throw new errors_1.JobManagerError("Access denied: 'boss' property was not exposed during JobManager initialization.");
}
return this.#boss;
}
async init(passedQueues = []) {
internals_1.default.section('Set up job manager');
const connectionConfig = (0, db_1.getDbConnectionConfig)('jobQueueDb');
const bossInstance = new pg_boss_1.PgBoss({
schema: newSchemaName,
...connectionConfig,
});
this.#boss = bossInstance;
this.#boss.on('error', error => logger_1.default.error(error));
await this.#boss.start();
internals_1.default.success('Connected to job queue');
internals_1.default.section('Register built-in job queues');
await this.#registerQueues(defaultJobQueues_1.defaultJobQueues, false);
internals_1.default.point('Registering custom job queues');
const jobQueuesFile = await (0, filesystem_1.readConfigurationFile)('jobQueues');
let queues = jobQueuesFile?.default || [];
try {
JobQueuesArraySchema.parse(queues);
}
catch (e) {
throw new errors_1.JobManagerError(`Malformed jobQueues file: ${e}`);
}
if (passedQueues)
queues = [...queues, ...passedQueues];
if (queues.length === 0) {
internals_1.default.point('No custom job queues found', 2);
}
await this.#registerQueues(queues);
internals_1.default.point('Cleaning up orphaned schedules');
const schedules = await this.#boss.getSchedules();
let orphanFound = false;
await Promise.all(schedules.map(async (schedule) => {
const queue = queues.find(q => q.name === schedule.name);
if (!queue) {
orphanFound = true;
await this.#boss.unschedule(schedule.name);
internals_1.default.success(`Removed schedule on queue "${schedule.name}", as queue definition no longer exists.`, 2);
}
if (queue && !queue.schedule) {
orphanFound = true;
await this.#boss.unschedule(schedule.name);
internals_1.default.success(`Removed schedule on queue "${schedule.name}", as schedule option no longer exists on queue definition.`, 2);
}
}));
if (!orphanFound) {
internals_1.default.point('No orphaned schedules found', 2);
}
await this.#migrate();
}
async #migrate() {
const metaTableData = await db_1.migrationsMeta.getData();
if (metaTableData.pgBossSchema === newSchemaName) {
internals_1.default.point('Job queue schema unchanged, no migration necessary.');
return;
}
const existingSchema = metaTableData.pgBossSchema || 'pgboss';
internals_1.default.point(`Migrating job queues from existing schema '${existingSchema}' to new schema '${newSchemaName}'`);
const existingTableExists = await db_1.db.schema
.withSchema(existingSchema)
.hasTable('job');
if (!existingTableExists) {
internals_1.default.point('There is no existing job queue table to migrate from.');
return;
}
const queues = await this.#boss.getQueues();
for (const queue of queues) {
try {
const sql = `
INSERT INTO ${newSchemaName}.job (
id,
name,
priority,
data,
retry_limit,
retry_count,
retry_delay,
retry_backoff,
start_after,
singleton_key,
singleton_on,
expire_seconds,
created_on,
keep_until,
output,
policy
)
SELECT
id,
name,
priority,
data,
retryLimit,
retryCount,
retryDelay,
retryBackoff,
startAfter,
singletonKey,
singletonOn,
EXTRACT(EPOCH FROM expireIn)::integer,
createdOn,
keepUntil,
output jsonb,
'${queue.policy}' as policy
FROM ${existingSchema}.job
WHERE name = '${queue.name}'
AND state = 'created'
ON CONFLICT DO NOTHING
`;
/* eslint-disable-next-line no-await-in-loop */
const { rowCount } = await db_1.db.raw(sql);
if (rowCount) {
internals_1.default.success(`Migrated ${rowCount} job${rowCount > 1 ? 's' : ''} in queue ${queue.name}`, 2);
}
}
catch (error) {
throw new errors_1.JobManagerError(`Migration error while copying jobs from '${queue.name}': ${error.message}`);
}
}
await db_1.migrationsMeta.setPgBossSchema(newSchemaName);
internals_1.default.success(`Saved applied job queue schema: ${newSchemaName}.`);
}
async #registerQueues(queues, indent = true) {
await Promise.all(queues.map(async (q) => {
const options = {};
if (q.batchSize)
options.batchSize = q.batchSize;
if (q.concurrency)
options.localConcurrency = q.concurrency;
const handler = async (jobs) => {
const [job] = jobs;
return q.handler(job);
};
const exists = await this.#boss.getQueue(q.name);
if (!exists)
await this.#boss.createQueue(q.name);
await this.#boss.work(q.name, options, handler);
internals_1.default.success(`Registered queue "${q.name}"`, indent ? 2 : 0);
if (q.schedule) {
const scheduleOptions = {};
if (q.scheduleTimezone)
scheduleOptions.tz = q.scheduleTimezone;
await this.#boss.schedule(q.name, q.schedule, null, scheduleOptions);
const readablePattern = cronstrue_1.default.toString(q.schedule, {
verbose: true,
});
internals_1.default.success(`Set up schedule on queue "${q.name}" to run: ${readablePattern}, in the ${q.scheduleTimezone || 'UTC'} timezone`, 4);
}
}));
}
async stop() {
internals_1.default.section('Shut down job manager');
await this.#boss.stop();
/**
* await boss.stop() doesn't wait until boss is in a stopped state,
* so we're trapping the function until boss.stopped is true
*/
const pollingInterval = 100;
const timeout = 5000;
const endTime = Date.now() + timeout;
while (true) {
// @ts-ignore
if (this.#boss.stopped) {
internals_1.default.success('Successfully shut down job manager');
break;
}
if (Date.now() >= endTime) {
internals_1.default.success(`Job manager shutdown timed out after ${timeout} ms`);
break;
}
/* eslint-disable-next-line no-await-in-loop */
await (0, wait_1.default)(pollingInterval);
}
}
async getQueueSize(queueName) {
const stats = await this.#boss.getQueueStats(queueName);
return stats.queuedCount + stats.activeCount;
}
/* eslint-disable-next-line class-methods-use-this */
async waitForJobsToFinish(queueName, filterData, options = {}) {
const interval = options.interval || 500;
const timeout = options.timeout || 60000;
return new Promise((resolve, reject) => {
let intervalId;
let timeoutId;
async function checkIsDone() {
try {
const res = await db_1.db.raw(`
SELECT
id,
name,
state,
data
FROM
${newSchemaName}.job
WHERE
name = '${queueName}'
`);
const match = res.rows.filter(row => (0, isMatch_1.default)(row.data, filterData));
if (match.length === 0) {
throw new Error(`No jobs match the filter ${JSON.stringify(filterData)}`);
}
const pending = match.filter(row => row.state === 'created');
if (pending.length > 0)
return;
const active = match.filter(row => row.state === 'active');
if (active.length > 0)
return;
clearInterval(intervalId);
clearTimeout(timeoutId);
resolve();
}
catch (e) {
clearInterval(intervalId);
clearTimeout(timeoutId);
reject(e);
}
}
intervalId = setInterval(checkIsDone, interval);
timeoutId = setTimeout(() => {
clearInterval(intervalId);
reject(new Error(`Job in queue "${queueName}" with data matching ${JSON.stringify(filterData)} did not complete before timeout of ${timeout} ms.`));
}, timeout);
});
}
async waitForQueueToEmpty(queueName, options = {}) {
const interval = options.interval || 500;
const timeout = options.timeout || 60000;
const self = this;
return new Promise((resolve, reject) => {
let intervalId;
let timeoutId;
async function checkIsEmpty() {
try {
const size = await self.getQueueSize(queueName);
if (size === 0) {
clearInterval(intervalId);
clearTimeout(timeoutId);
resolve();
}
}
catch (e) {
clearInterval(intervalId);
clearTimeout(timeoutId);
reject(e);
}
}
intervalId = setInterval(checkIsEmpty, interval);
timeoutId = setTimeout(() => {
clearInterval(intervalId);
reject(new Error(`Waiting for queue ${queueName} to empty timed out after ${timeout} ms.`));
}, timeout);
});
}
async sendToQueue(queueName, data, options = {}) {
try {
sendOptionsSchema.parse(options);
}
catch (error) {
throw new errors_1.JobManagerOptionsError(error.message);
}
await this.#boss.send(queueName, data, options);
}
}
exports.JobManager = JobManager;
const jobManager = new JobManager();
exports.jobManager = jobManager;
//# sourceMappingURL=JobManager.js.map