@sidequest/engine
Version:
@sidequest/engine is the core engine of SideQuest, a distributed background job processing system for Node.js and TypeScript.
248 lines (245 loc) • 11.3 kB
JavaScript
import { QUEUE_FALLBACK, MISC_FALLBACK, LazyBackend } from '@sidequest/backend';
import { logger, configureLogger } from '@sidequest/core';
import { fork } from 'child_process';
import { existsSync } from 'fs';
import { cpus } from 'os';
import { fileURLToPath } from 'url';
import { inspect } from 'util';
import { DEFAULT_WORKER_PATH } from './constants.js';
import { dependencyRegistry, Dependency } from './dependency-registry.js';
import { JOB_BUILDER_FALLBACK } from './job/constants.js';
import { ScheduledJobRegistry } from './job/cron-registry.js';
import { JobBuilder } from './job/job-builder.js';
import { grantQueueConfig } from './queue/grant-queue-config.js';
import 'node:fs';
import 'node:url';
import { gracefulShutdown, clearGracefulShutdown } from './utils/shutdown.js';
import { resolveScriptPath, findSidequestJobsScriptInParentDirs } from './shared-runner/manual-loader.js';
import 'piscina';
/**
* The main engine for managing job queues and workers in Sidequest.
*/
class Engine {
/**
* Main worker process that runs the Sidequest engine.
* This is created when the engine is started and handles job processing.
*/
mainWorker;
/**
* Flag indicating whether the engine is currently shutting down.
* This is used to prevent multiple shutdown attempts and ensure graceful shutdown behavior.
*/
shuttingDown = false;
/**
* Configures the Sidequest engine with the provided configuration.
* @param config Optional configuration object.
* @returns The resolved configuration.
*/
async configure(config) {
if (this.getConfig()) {
logger("Engine").debug("Sidequest already configured");
return this.getConfig();
}
const nonNullConfig = {
queues: config?.queues ?? [],
backend: {
driver: config?.backend?.driver ?? "@sidequest/sqlite-backend",
config: config?.backend?.config ?? "./sidequest.sqlite",
},
cleanupFinishedJobsIntervalMin: config?.cleanupFinishedJobsIntervalMin ?? 60,
cleanupFinishedJobsOlderThan: config?.cleanupFinishedJobsOlderThan ?? 30 * 24 * 60 * 60 * 1000,
releaseStaleJobsIntervalMin: config?.releaseStaleJobsIntervalMin ?? 60,
maxConcurrentJobs: config?.maxConcurrentJobs ?? 10,
skipMigration: config?.skipMigration ?? false,
logger: {
level: config?.logger?.level ?? "info",
json: config?.logger?.json ?? false,
},
gracefulShutdown: config?.gracefulShutdown ?? true,
minThreads: config?.minThreads ?? cpus().length,
maxThreads: config?.maxThreads ?? cpus().length * 2,
idleWorkerTimeout: config?.idleWorkerTimeout ?? 10_000,
releaseStaleJobsMaxStaleMs: config?.releaseStaleJobsMaxStaleMs ?? MISC_FALLBACK.maxStaleMs, // 10 minutes
releaseStaleJobsMaxClaimedMs: config?.releaseStaleJobsMaxClaimedMs ?? MISC_FALLBACK.maxClaimedMs, // 1 minute
jobDefaults: {
queue: config?.jobDefaults?.queue ?? JOB_BUILDER_FALLBACK.queue,
maxAttempts: config?.jobDefaults?.maxAttempts ?? JOB_BUILDER_FALLBACK.maxAttempts,
// This here does not use a fallback default because it is a getter.
// It needs to be set at job creation time.
availableAt: config?.jobDefaults?.availableAt,
timeout: config?.jobDefaults?.timeout ?? JOB_BUILDER_FALLBACK.timeout,
uniqueness: config?.jobDefaults?.uniqueness ?? JOB_BUILDER_FALLBACK.uniqueness,
},
queueDefaults: {
concurrency: config?.queueDefaults?.concurrency ?? QUEUE_FALLBACK.concurrency,
priority: config?.queueDefaults?.priority ?? QUEUE_FALLBACK.priority,
state: config?.queueDefaults?.state ?? QUEUE_FALLBACK.state,
},
manualJobResolution: config?.manualJobResolution ?? false,
jobsFilePath: config?.jobsFilePath?.trim() ?? "",
jobPollingInterval: config?.jobPollingInterval ?? 100,
};
dependencyRegistry.register(Dependency.Config, nonNullConfig);
this.validateConfig();
logger("Engine").debug(`Configuring Sidequest engine: ${inspect(nonNullConfig)}`);
if (nonNullConfig.logger) {
configureLogger(nonNullConfig.logger);
}
const backend = dependencyRegistry.register(Dependency.Backend, new LazyBackend(nonNullConfig.backend));
if (!nonNullConfig.skipMigration) {
await backend.migrate();
}
return nonNullConfig;
}
/**
* Validates the engine configuration settings.
*
* This method also resolves the jobs file path to a file URL if manual job resolution is enabled.
*
* @throws {Error} When `maxConcurrentJobs` is defined but less than 1
* @throws {Error} When `manualJobResolution` is enabled but the specified `jobsFilePath` does not exist
* @throws {Error} When `manualJobResolution` is enabled but no jobs script can be found in parent directories
*
* @remarks
* - Ensures `maxConcurrentJobs` is at least 1 if specified
* - When `manualJobResolution` is enabled, validates that either:
* - A valid `jobsFilePath` exists and resolves it to an absolute URL
* - A sidequest jobs script can be found in parent directories
* - Logs the resolved jobs file path when using manual job resolution
*/
validateConfig() {
const config = this.getConfig();
if (config.maxConcurrentJobs !== undefined && config.maxConcurrentJobs < 1) {
throw new Error(`Invalid "maxConcurrentJobs" value: must be at least 1.`);
}
if (config.manualJobResolution) {
if (config.jobsFilePath) {
const scriptUrl = resolveScriptPath(config.jobsFilePath);
if (!existsSync(fileURLToPath(scriptUrl))) {
throw new Error(`The specified jobsFilePath does not exist. Resolved to: ${scriptUrl}`);
}
logger("Engine").info(`Using manual jobs file at: ${config.jobsFilePath}`);
config.jobsFilePath = scriptUrl;
}
else {
// This should throw an error if not found
findSidequestJobsScriptInParentDirs();
}
}
}
/**
* Starts the Sidequest engine and worker process.
* @param config Optional configuration object.
*/
async start(config) {
if (this.mainWorker) {
logger("Engine").warn("Sidequest engine already started");
return;
}
const nonNullConfig = await this.configure(config);
logger("Engine").info(`Starting Sidequest using backend ${nonNullConfig.backend.driver}`);
if (nonNullConfig.queues) {
for (const queue of nonNullConfig.queues) {
await grantQueueConfig(dependencyRegistry.get(Dependency.Backend), queue, nonNullConfig.queueDefaults, true);
}
}
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error("Timeout on starting sidequest fork!"));
}, 5000);
if (!this.mainWorker) {
const runWorker = () => {
logger("Engine").debug("Starting main worker...");
this.mainWorker = fork(DEFAULT_WORKER_PATH);
logger("Engine").debug(`Worker PID: ${this.mainWorker.pid}`);
this.mainWorker.on("message", (msg) => {
if (msg === "ready") {
logger("Engine").debug("Main worker is ready");
this.mainWorker?.send({ type: "start", sidequestConfig: nonNullConfig });
clearTimeout(timeout);
resolve();
}
});
this.mainWorker.on("exit", () => {
if (!this.shuttingDown) {
logger("Engine").error("Sidequest main exited, creating new...");
runWorker();
}
});
};
runWorker();
gracefulShutdown(this.close.bind(this), "Engine", nonNullConfig.gracefulShutdown);
}
});
}
/**
* Gets the current engine configuration.
* @returns The current configuration, if set.
*/
getConfig() {
return dependencyRegistry.get(Dependency.Config);
}
/**
* Gets the backend instance in use by the engine.
* @returns The backend instance, if set.
*/
getBackend() {
return dependencyRegistry.get(Dependency.Backend);
}
/**
* Closes the engine and releases resources.
*/
async close() {
if (!this.shuttingDown) {
this.shuttingDown = true;
logger("Engine").debug("Closing Sidequest engine...");
// Stop all scheduled cron jobs first
await ScheduledJobRegistry.stopAll();
if (this.mainWorker) {
const promise = new Promise((resolve) => {
this.mainWorker.on("exit", resolve);
});
this.mainWorker.send({ type: "shutdown" });
await promise;
}
try {
await dependencyRegistry.get(Dependency.Backend)?.close();
}
catch (error) {
logger("Engine").error("Error closing backend:", error);
}
this.mainWorker = undefined;
// Reset the shutting down flag after closing
// This allows the engine to be reconfigured or restarted later
clearGracefulShutdown();
logger("Engine").debug("Sidequest engine closed.");
this.shuttingDown = false;
// Clear the dependency registry to allow fresh configuration later
dependencyRegistry.clear();
}
}
/**
* Builds a job using the provided job class.
* @param JobClass The job class constructor.
* @returns A new JobBuilder instance for the job class.
*/
build(JobClass) {
const backend = this.getBackend();
const config = this.getConfig();
if (!config || !backend) {
throw new Error("Engine not configured. Call engine.configure() or engine.start() first.");
}
if (this.shuttingDown) {
throw new Error("Engine is shutting down, cannot build job.");
}
logger("Engine").debug(`Building job for class: ${JobClass.name}`);
return new JobBuilder(backend, JobClass, {
...config.jobDefaults,
// We need to do this check again because available at is a getter. It needs to be set at job creation time.
// If not set, it will use the fallback value which is outdated from config.
availableAt: config.jobDefaults.availableAt ?? JOB_BUILDER_FALLBACK.availableAt,
}, config.manualJobResolution);
}
}
export { Engine };
//# sourceMappingURL=engine.js.map