@sidequest/engine
Version:
@sidequest/engine is the core engine of SideQuest, a distributed background job processing system for Node.js and TypeScript.
250 lines (246 loc) • 11.7 kB
JavaScript
;
var backend = require('@sidequest/backend');
var core = require('@sidequest/core');
var child_process = require('child_process');
var fs = require('fs');
var os = require('os');
var url = require('url');
var util = require('util');
var constants$1 = require('./constants.cjs');
var dependencyRegistry = require('./dependency-registry.cjs');
var constants = require('./job/constants.cjs');
var cronRegistry = require('./job/cron-registry.cjs');
var jobBuilder = require('./job/job-builder.cjs');
var grantQueueConfig = require('./queue/grant-queue-config.cjs');
require('node:fs');
require('node:url');
var shutdown = require('./utils/shutdown.cjs');
var manualLoader = require('./shared-runner/manual-loader.cjs');
require('piscina');
/**
* The main engine for managing job queues and workers in Sidequest.
*/
class Engine {
/**
* Main worker process that runs the Sidequest engine.
* This is created when the engine is started and handles job processing.
*/
mainWorker;
/**
* Flag indicating whether the engine is currently shutting down.
* This is used to prevent multiple shutdown attempts and ensure graceful shutdown behavior.
*/
shuttingDown = false;
/**
* Configures the Sidequest engine with the provided configuration.
* @param config Optional configuration object.
* @returns The resolved configuration.
*/
async configure(config) {
if (this.getConfig()) {
core.logger("Engine").debug("Sidequest already configured");
return this.getConfig();
}
const nonNullConfig = {
queues: config?.queues ?? [],
backend: {
driver: config?.backend?.driver ?? "@sidequest/sqlite-backend",
config: config?.backend?.config ?? "./sidequest.sqlite",
},
cleanupFinishedJobsIntervalMin: config?.cleanupFinishedJobsIntervalMin ?? 60,
cleanupFinishedJobsOlderThan: config?.cleanupFinishedJobsOlderThan ?? 30 * 24 * 60 * 60 * 1000,
releaseStaleJobsIntervalMin: config?.releaseStaleJobsIntervalMin ?? 60,
maxConcurrentJobs: config?.maxConcurrentJobs ?? 10,
skipMigration: config?.skipMigration ?? false,
logger: {
level: config?.logger?.level ?? "info",
json: config?.logger?.json ?? false,
},
gracefulShutdown: config?.gracefulShutdown ?? true,
minThreads: config?.minThreads ?? os.cpus().length,
maxThreads: config?.maxThreads ?? os.cpus().length * 2,
idleWorkerTimeout: config?.idleWorkerTimeout ?? 10_000,
releaseStaleJobsMaxStaleMs: config?.releaseStaleJobsMaxStaleMs ?? backend.MISC_FALLBACK.maxStaleMs, // 10 minutes
releaseStaleJobsMaxClaimedMs: config?.releaseStaleJobsMaxClaimedMs ?? backend.MISC_FALLBACK.maxClaimedMs, // 1 minute
jobDefaults: {
queue: config?.jobDefaults?.queue ?? constants.JOB_BUILDER_FALLBACK.queue,
maxAttempts: config?.jobDefaults?.maxAttempts ?? constants.JOB_BUILDER_FALLBACK.maxAttempts,
// This here does not use a fallback default because it is a getter.
// It needs to be set at job creation time.
availableAt: config?.jobDefaults?.availableAt,
timeout: config?.jobDefaults?.timeout ?? constants.JOB_BUILDER_FALLBACK.timeout,
uniqueness: config?.jobDefaults?.uniqueness ?? constants.JOB_BUILDER_FALLBACK.uniqueness,
},
queueDefaults: {
concurrency: config?.queueDefaults?.concurrency ?? backend.QUEUE_FALLBACK.concurrency,
priority: config?.queueDefaults?.priority ?? backend.QUEUE_FALLBACK.priority,
state: config?.queueDefaults?.state ?? backend.QUEUE_FALLBACK.state,
},
manualJobResolution: config?.manualJobResolution ?? false,
jobsFilePath: config?.jobsFilePath?.trim() ?? "",
jobPollingInterval: config?.jobPollingInterval ?? 100,
};
dependencyRegistry.dependencyRegistry.register(dependencyRegistry.Dependency.Config, nonNullConfig);
this.validateConfig();
core.logger("Engine").debug(`Configuring Sidequest engine: ${util.inspect(nonNullConfig)}`);
if (nonNullConfig.logger) {
core.configureLogger(nonNullConfig.logger);
}
const backend$1 = dependencyRegistry.dependencyRegistry.register(dependencyRegistry.Dependency.Backend, new backend.LazyBackend(nonNullConfig.backend));
if (!nonNullConfig.skipMigration) {
await backend$1.migrate();
}
return nonNullConfig;
}
/**
* Validates the engine configuration settings.
*
* This method also resolves the jobs file path to a file URL if manual job resolution is enabled.
*
* @throws {Error} When `maxConcurrentJobs` is defined but less than 1
* @throws {Error} When `manualJobResolution` is enabled but the specified `jobsFilePath` does not exist
* @throws {Error} When `manualJobResolution` is enabled but no jobs script can be found in parent directories
*
* @remarks
* - Ensures `maxConcurrentJobs` is at least 1 if specified
* - When `manualJobResolution` is enabled, validates that either:
* - A valid `jobsFilePath` exists and resolves it to an absolute URL
* - A sidequest jobs script can be found in parent directories
* - Logs the resolved jobs file path when using manual job resolution
*/
validateConfig() {
const config = this.getConfig();
if (config.maxConcurrentJobs !== undefined && config.maxConcurrentJobs < 1) {
throw new Error(`Invalid "maxConcurrentJobs" value: must be at least 1.`);
}
if (config.manualJobResolution) {
if (config.jobsFilePath) {
const scriptUrl = manualLoader.resolveScriptPath(config.jobsFilePath);
if (!fs.existsSync(url.fileURLToPath(scriptUrl))) {
throw new Error(`The specified jobsFilePath does not exist. Resolved to: ${scriptUrl}`);
}
core.logger("Engine").info(`Using manual jobs file at: ${config.jobsFilePath}`);
config.jobsFilePath = scriptUrl;
}
else {
// This should throw an error if not found
manualLoader.findSidequestJobsScriptInParentDirs();
}
}
}
/**
* Starts the Sidequest engine and worker process.
* @param config Optional configuration object.
*/
async start(config) {
if (this.mainWorker) {
core.logger("Engine").warn("Sidequest engine already started");
return;
}
const nonNullConfig = await this.configure(config);
core.logger("Engine").info(`Starting Sidequest using backend ${nonNullConfig.backend.driver}`);
if (nonNullConfig.queues) {
for (const queue of nonNullConfig.queues) {
await grantQueueConfig.grantQueueConfig(dependencyRegistry.dependencyRegistry.get(dependencyRegistry.Dependency.Backend), queue, nonNullConfig.queueDefaults, true);
}
}
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error("Timeout on starting sidequest fork!"));
}, 5000);
if (!this.mainWorker) {
const runWorker = () => {
core.logger("Engine").debug("Starting main worker...");
this.mainWorker = child_process.fork(constants$1.DEFAULT_WORKER_PATH);
core.logger("Engine").debug(`Worker PID: ${this.mainWorker.pid}`);
this.mainWorker.on("message", (msg) => {
if (msg === "ready") {
core.logger("Engine").debug("Main worker is ready");
this.mainWorker?.send({ type: "start", sidequestConfig: nonNullConfig });
clearTimeout(timeout);
resolve();
}
});
this.mainWorker.on("exit", () => {
if (!this.shuttingDown) {
core.logger("Engine").error("Sidequest main exited, creating new...");
runWorker();
}
});
};
runWorker();
shutdown.gracefulShutdown(this.close.bind(this), "Engine", nonNullConfig.gracefulShutdown);
}
});
}
/**
* Gets the current engine configuration.
* @returns The current configuration, if set.
*/
getConfig() {
return dependencyRegistry.dependencyRegistry.get(dependencyRegistry.Dependency.Config);
}
/**
* Gets the backend instance in use by the engine.
* @returns The backend instance, if set.
*/
getBackend() {
return dependencyRegistry.dependencyRegistry.get(dependencyRegistry.Dependency.Backend);
}
/**
* Closes the engine and releases resources.
*/
async close() {
if (!this.shuttingDown) {
this.shuttingDown = true;
core.logger("Engine").debug("Closing Sidequest engine...");
// Stop all scheduled cron jobs first
await cronRegistry.ScheduledJobRegistry.stopAll();
if (this.mainWorker) {
const promise = new Promise((resolve) => {
this.mainWorker.on("exit", resolve);
});
this.mainWorker.send({ type: "shutdown" });
await promise;
}
try {
await dependencyRegistry.dependencyRegistry.get(dependencyRegistry.Dependency.Backend)?.close();
}
catch (error) {
core.logger("Engine").error("Error closing backend:", error);
}
this.mainWorker = undefined;
// Reset the shutting down flag after closing
// This allows the engine to be reconfigured or restarted later
shutdown.clearGracefulShutdown();
core.logger("Engine").debug("Sidequest engine closed.");
this.shuttingDown = false;
// Clear the dependency registry to allow fresh configuration later
dependencyRegistry.dependencyRegistry.clear();
}
}
/**
* Builds a job using the provided job class.
* @param JobClass The job class constructor.
* @returns A new JobBuilder instance for the job class.
*/
build(JobClass) {
const backend = this.getBackend();
const config = this.getConfig();
if (!config || !backend) {
throw new Error("Engine not configured. Call engine.configure() or engine.start() first.");
}
if (this.shuttingDown) {
throw new Error("Engine is shutting down, cannot build job.");
}
core.logger("Engine").debug(`Building job for class: ${JobClass.name}`);
return new jobBuilder.JobBuilder(backend, JobClass, {
...config.jobDefaults,
// We need to do this check again because available at is a getter. It needs to be set at job creation time.
// If not set, it will use the fallback value which is outdated from config.
availableAt: config.jobDefaults.availableAt ?? constants.JOB_BUILDER_FALLBACK.availableAt,
}, config.manualJobResolution);
}
}
exports.Engine = Engine;
//# sourceMappingURL=engine.cjs.map