@sidequest/engine
Version:
@sidequest/engine is the core engine of SideQuest, a distributed background job processing system for Node.js and TypeScript.
206 lines (202 loc) • 9.05 kB
JavaScript
'use strict';
var backend = require('@sidequest/backend');
var core = require('@sidequest/core');
var child_process = require('child_process');
var os = require('os');
var path = require('path');
var constants = require('./job/constants.cjs');
var jobBuilder = require('./job/job-builder.cjs');
var grantQueueConfig = require('./queue/grant-queue-config.cjs');
var shutdown = require('./utils/shutdown.cjs');
const workerPath = path.resolve(__dirname, "workers", "main.js");
/**
* The main engine for managing job queues and workers in Sidequest.
*/
class Engine {
/**
* Backend instance used by the engine.
* This is initialized when the engine is configured or started.
*/
backend;
/**
* Current configuration of the engine.
* This is set when the engine is configured or started.
* It contains all the necessary settings for the engine to operate, such as backend, queues, logger options, and job defaults.
*/
config;
/**
* Main worker process that runs the Sidequest engine.
* This is created when the engine is started and handles job processing.
*/
mainWorker;
/**
* Flag indicating whether the engine is currently shutting down.
* This is used to prevent multiple shutdown attempts and ensure graceful shutdown behavior.
*/
shuttingDown = false;
/**
* Configures the Sidequest engine with the provided configuration.
* @param config Optional configuration object.
* @returns The resolved configuration.
*/
async configure(config) {
if (this.config) {
core.logger("Engine").debug("Sidequest already configured");
return this.config;
}
this.config = {
queues: config?.queues ?? [],
backend: {
driver: config?.backend?.driver ?? "@sidequest/sqlite-backend",
config: config?.backend?.config ?? "./sidequest.sqlite",
},
cleanupFinishedJobsIntervalMin: config?.cleanupFinishedJobsIntervalMin ?? 60,
cleanupFinishedJobsOlderThan: config?.cleanupFinishedJobsOlderThan ?? 30 * 24 * 60 * 60 * 1000,
releaseStaleJobsIntervalMin: config?.releaseStaleJobsIntervalMin ?? 60,
maxConcurrentJobs: config?.maxConcurrentJobs ?? 10,
skipMigration: config?.skipMigration ?? false,
logger: {
level: config?.logger?.level ?? "info",
json: config?.logger?.json ?? false,
},
gracefulShutdown: config?.gracefulShutdown ?? true,
minThreads: config?.minThreads ?? os.cpus().length,
maxThreads: config?.maxThreads ?? os.cpus().length * 2,
idleWorkerTimeout: config?.idleWorkerTimeout ?? 10_000,
releaseStaleJobsMaxStaleMs: config?.releaseStaleJobsMaxStaleMs ?? backend.MISC_FALLBACK.maxStaleMs, // 10 minutes
releaseStaleJobsMaxClaimedMs: config?.releaseStaleJobsMaxClaimedMs ?? backend.MISC_FALLBACK.maxClaimedMs, // 1 minute
jobDefaults: {
queue: config?.jobDefaults?.queue ?? constants.JOB_BUILDER_FALLBACK.queue,
maxAttempts: config?.jobDefaults?.maxAttempts ?? constants.JOB_BUILDER_FALLBACK.maxAttempts,
// This here does not use a fallback default because it is a getter.
// It needs to be set at job creation time.
availableAt: config?.jobDefaults?.availableAt,
timeout: config?.jobDefaults?.timeout ?? constants.JOB_BUILDER_FALLBACK.timeout,
uniqueness: config?.jobDefaults?.uniqueness ?? constants.JOB_BUILDER_FALLBACK.uniqueness,
},
queueDefaults: {
concurrency: config?.queueDefaults?.concurrency ?? backend.QUEUE_FALLBACK.concurrency,
priority: config?.queueDefaults?.priority ?? backend.QUEUE_FALLBACK.priority,
state: config?.queueDefaults?.state ?? backend.QUEUE_FALLBACK.state,
},
};
if (this.config.maxConcurrentJobs !== undefined && this.config.maxConcurrentJobs < 1) {
throw new Error(`Invalid "maxConcurrentJobs" value: must be at least 1.`);
}
core.logger("Engine").debug(`Configuring Sidequest engine: ${JSON.stringify(this.config)}`);
if (this.config.logger) {
core.configureLogger(this.config.logger);
}
this.backend = new backend.LazyBackend(this.config.backend);
if (!this.config.skipMigration) {
await this.backend.migrate();
}
return this.config;
}
/**
* Starts the Sidequest engine and worker process.
* @param config Optional configuration object.
*/
async start(config) {
if (this.mainWorker) {
core.logger("Engine").warn("Sidequest engine already started");
return;
}
await this.configure(config);
core.logger("Engine").info(`Starting Sidequest using backend ${this.config.backend.driver}`);
if (this.config.queues) {
for (const queue of this.config.queues) {
await grantQueueConfig.grantQueueConfig(this.backend, queue, this.config.queueDefaults, true);
}
}
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error("Timeout on starting sidequest fork!"));
}, 5000);
if (!this.mainWorker) {
const runWorker = () => {
core.logger("Engine").debug("Starting main worker...");
this.mainWorker = child_process.fork(workerPath);
core.logger("Engine").debug(`Worker PID: ${this.mainWorker.pid}`);
this.mainWorker.on("message", (msg) => {
if (msg === "ready") {
core.logger("Engine").debug("Main worker is ready");
this.mainWorker?.send({ type: "start", sidequestConfig: this.config });
clearTimeout(timeout);
resolve();
}
});
this.mainWorker.on("exit", () => {
if (!this.shuttingDown) {
core.logger("Engine").error("Sidequest main exited, creating new...");
runWorker();
}
});
};
runWorker();
shutdown.gracefulShutdown(this.close.bind(this), "Engine", this.config.gracefulShutdown);
}
});
}
/**
* Gets the current engine configuration.
* @returns The current configuration, if set.
*/
getConfig() {
return this.config;
}
/**
* Gets the backend instance in use by the engine.
* @returns The backend instance, if set.
*/
getBackend() {
return this.backend;
}
/**
* Closes the engine and releases resources.
*/
async close() {
if (!this.shuttingDown) {
this.shuttingDown = true;
core.logger("Engine").debug("Closing Sidequest engine...");
if (this.mainWorker) {
const promise = new Promise((resolve) => {
this.mainWorker.on("exit", resolve);
});
this.mainWorker.send({ type: "shutdown" });
await promise;
}
await this.backend?.close();
this.config = undefined;
this.backend = undefined;
this.mainWorker = undefined;
// Reset the shutting down flag after closing
// This allows the engine to be reconfigured or restarted later
shutdown.clearGracefulShutdown();
core.logger("Engine").debug("Sidequest engine closed.");
this.shuttingDown = false;
}
}
/**
* Builds a job using the provided job class.
* @param JobClass The job class constructor.
* @returns A new JobBuilder instance for the job class.
*/
build(JobClass) {
if (!this.config || !this.backend) {
throw new Error("Engine not configured. Call engine.configure() or engine.start() first.");
}
if (this.shuttingDown) {
throw new Error("Engine is shutting down, cannot build job.");
}
core.logger("Engine").debug(`Building job for class: ${JobClass.name}`);
return new jobBuilder.JobBuilder(this.backend, JobClass, {
...this.config.jobDefaults,
// We need to do this check again because available at is a getter. It needs to be set at job creation time.
// If not set, it will use the fallback value which is outdated from config.
availableAt: this.config.jobDefaults.availableAt ?? constants.JOB_BUILDER_FALLBACK.availableAt,
});
}
}
exports.Engine = Engine;
//# sourceMappingURL=engine.cjs.map