UNPKG

trigger.dev

Version:

A Command-Line Interface for Trigger.dev projects

714 lines • 31.7 kB
var _a; import { spawn } from "node:child_process"; import { readFileSync, writeFileSync, renameSync, unlinkSync, existsSync, mkdirSync, } from "node:fs"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { setTimeout as awaitTimeout } from "node:timers/promises"; import { eventBus } from "../utilities/eventBus.js"; import { logger } from "../utilities/logger.js"; import { resolveSourceFiles } from "../utilities/sourceFiles.js"; import { BackgroundWorker } from "./backgroundWorker.js"; import { copySkillFolders } from "../build/bundleSkills.js"; import { cliLink, prettyError } from "../utilities/cliOutput.js"; import { DevRunController } from "../entryPoints/dev-run-controller.js"; import { io } from "socket.io-client"; import pLimit from "p-limit"; import { resolveLocalEnvVars } from "../utilities/localEnvVars.js"; import { TaskRunProcessPool } from "./taskRunProcessPool.js"; import { tryCatch } from "@trigger.dev/core/utils"; import { devBranchPathSegment } from "../utilities/devBranch.js"; import { getTmpRoot } from "../utilities/tempDirectories.js"; export async function startWorkerRuntime(options) { const runtime = new DevSupervisor(options); await runtime.init(); return runtime; } /** * The DevSupervisor is used when you run the `trigger.dev dev` command (with engine 2.0+) * It's responsible for: * - Creating/registering BackgroundWorkers * - Pulling runs from the queue * - Delegating executing the runs to DevRunController * - Receiving snapshot update pings (via socket) */ class DevSupervisor { options; config; disconnectPresence; lastManifest; latestWorkerId; /** Receive notifications when runs change state */ socket; socketIsReconnecting = false; /** Workers are versions of the code */ workers = new Map(); /** Map of run friendly id to run controller. They process runs from start to finish. */ runControllers = new Map(); socketConnections = new Set(); runLimiter; taskRunProcessPool; /** Detached watchdog process that cancels runs if the CLI is killed */ watchdogProcess; activeRunsPath; watchdogPidPath; constructor(options) { this.options = options; } async init() { logger.debug("[DevSupervisor] initialized worker runtime", { options: this.options }); //get the settings for dev const settings = await this.options.client.dev.config(); if (!settings.success) { throw new Error(`Failed to connect to ${this.options.client.apiURL}. Couldn't retrieve settings: ${settings.error}`); } logger.debug("[DevSupervisor] Got dev settings", { settings: settings.data }); this.config = settings.data; this.options.client.dev.setEngineURL(this.config.engineUrl); const maxConcurrentRuns = Math.min(this.config.maxConcurrentRuns, this.options.args.maxConcurrentRuns ?? this.config.maxConcurrentRuns); logger.debug("[DevSupervisor] Using maxConcurrentRuns", { maxConcurrentRuns }); this.runLimiter = pLimit(maxConcurrentRuns); // Initialize the task run process pool const env = await this.#getEnvVars(); const processKeepAlive = this.options.config.processKeepAlive ?? this.options.config.experimental_processKeepAlive; const enableProcessReuse = typeof processKeepAlive === "boolean" ? processKeepAlive : typeof processKeepAlive === "object" ? processKeepAlive.enabled : false; const maxPoolSize = typeof processKeepAlive === "object" ? (processKeepAlive.devMaxPoolSize ?? 25) : 25; const maxExecutionsPerProcess = typeof processKeepAlive === "object" ? (processKeepAlive.maxExecutionsPerProcess ?? 50) : 50; if (enableProcessReuse) { logger.debug("[DevSupervisor] Enabling process reuse", { enableProcessReuse, maxPoolSize, maxExecutionsPerProcess, }); } this.taskRunProcessPool = new TaskRunProcessPool({ env, cwd: this.options.config.workingDir, enableProcessReuse, maxPoolSize, maxExecutionsPerProcess, }); this.socket = this.#createSocket(); //start an SSE connection for presence this.disconnectPresence = await this.#startPresenceConnection(); // Handle SIGTERM/SIGINT to gracefully stop all run controllers process.on("SIGTERM", this.#handleSigterm); process.on("SIGINT", this.#handleSigterm); // Spawn detached watchdog to cancel runs if CLI is killed (e.g. pnpm SIGKILL) this.#spawnWatchdog(); //start dequeuing await this.#dequeueRuns(); } #handleSigterm = async () => { logger.debug("[DevSupervisor] Received SIGTERM/SIGINT, stopping all run controllers"); await this.shutdown(); // Must exit explicitly since registering a custom SIGINT handler // overrides Node's default process termination behavior. process.exit(0); }; async shutdown() { process.off("SIGTERM", this.#handleSigterm); process.off("SIGINT", this.#handleSigterm); // Stop all local run controllers first so active-runs.json is up-to-date const stopPromises = Array.from(this.runControllers.values()).map((controller) => controller.stop()); await Promise.allSettled(stopPromises); // Kill watchdog on clean shutdown — no disconnect needed since runs are stopped locally this.#killWatchdog(); this.disconnectPresence?.(); try { this.socket?.close(); } catch (error) { logger.debug("[DevSupervisor] shutdown, socket failed to close", { error }); } // Shutdown the task run process pool if (this.taskRunProcessPool) { const [shutdownError] = await tryCatch(this.taskRunProcessPool.shutdown()); if (shutdownError) { logger.debug("[DevSupervisor] shutdown, task run process pool failed to shutdown", { error: shutdownError, }); } } } #spawnWatchdog() { const triggerDir = join(this.options.config.workingDir, ".trigger"); if (!existsSync(triggerDir)) { mkdirSync(triggerDir, { recursive: true }); } // Namespace watchdog state per branch so concurrent dev sessions on // different branches don't share a single watchdog instance (the // single-instance guard would otherwise kill the other branch's watchdog). const safeBranch = devBranchPathSegment(this.options.branch); const suffix = safeBranch ? `-${safeBranch}` : ""; this.activeRunsPath = join(triggerDir, `active-runs${suffix}.json`); this.watchdogPidPath = join(triggerDir, `watchdog${suffix}.pid`); // Write empty active-runs file this.#updateActiveRunsFile(); // Resolve the compiled watchdog script path relative to this file const thisDir = fileURLToPath(new URL(".", import.meta.url)); const watchdogScript = join(thisDir, "devWatchdog.js"); if (!existsSync(watchdogScript)) { logger.debug("[DevSupervisor] Watchdog script not found, skipping", { watchdogScript }); return; } try { this.watchdogProcess = spawn(process.execPath, [watchdogScript], { detached: true, stdio: "ignore", env: { ...process.env, WATCHDOG_PARENT_PID: process.pid.toString(), WATCHDOG_API_URL: this.config?.engineUrl ?? this.options.client.apiURL, WATCHDOG_API_KEY: this.options.client.accessToken ?? "", WATCHDOG_ACTIVE_RUNS: this.activeRunsPath, WATCHDOG_PID_FILE: this.watchdogPidPath, WATCHDOG_TMP_DIR: getTmpRoot(this.options.config.workingDir, this.options.branch), }, }); this.watchdogProcess.unref(); logger.debug("[DevSupervisor] Spawned watchdog", { watchdogPid: this.watchdogProcess.pid, parentPid: process.pid, }); } catch (error) { logger.debug("[DevSupervisor] Failed to spawn watchdog", { error }); } } #killWatchdog() { const knownPid = this.watchdogProcess?.pid; if (knownPid) { try { process.kill(knownPid, "SIGTERM"); } catch { // Already dead } this.watchdogProcess = undefined; } // Fallback: try via PID file, but only if the PID matches our spawned watchdog // to avoid killing an unrelated process that reused a stale PID if (this.watchdogPidPath) { try { const content = readFileSync(this.watchdogPidPath, "utf8"); const prefix = "trigger-watchdog:"; if (content.startsWith(prefix)) { const pid = parseInt(content.slice(prefix.length), 10); if (pid && (!knownPid || pid === knownPid)) { process.kill(pid, "SIGTERM"); } } } catch { // Already dead or no file } } // Clean up files try { if (this.activeRunsPath) unlinkSync(this.activeRunsPath); } catch { } try { if (this.watchdogPidPath) unlinkSync(this.watchdogPidPath); } catch { } } #updateActiveRunsFile() { if (!this.activeRunsPath) return; try { const data = { parentPid: process.pid, runFriendlyIds: Array.from(this.runControllers.keys()), }; // Atomic write: write to temp file then rename to avoid corrupt reads const tmpPath = this.activeRunsPath + ".tmp"; writeFileSync(tmpPath, JSON.stringify(data)); renameSync(tmpPath, this.activeRunsPath); } catch (error) { logger.debug("[DevSupervisor] Failed to update active-runs file", { error }); } } async initializeWorker(manifest, metafile, stop) { if (this.lastManifest && this.lastManifest.contentHash === manifest.contentHash) { logger.debug("worker skipped", { lastManifestContentHash: this.lastManifest?.contentHash }); eventBus.emit("workerSkipped"); stop(); return; } const env = await this.#getEnvVars(); const backgroundWorker = new BackgroundWorker(manifest, metafile, { env, cwd: this.options.config.workingDir, stop, }); logger.debug("initializing background worker", { manifest }); await backgroundWorker.initialize(); if (!backgroundWorker.manifest) { stop(); throw new Error("Could not initialize worker"); } // Copy registered skill folders into `${workingDir}/.trigger/skills/{id}/` // so `skill.local()` can read them at runtime. The main indexer already // discovered skills; we just do the file IO here. const discoveredSkills = backgroundWorker.manifest.skills ?? []; if (discoveredSkills.length > 0) { try { await copySkillFolders({ skills: discoveredSkills, destinationRoot: join(this.options.config.workingDir, ".trigger", "skills"), workingDir: this.options.config.workingDir, logger, }); } catch (err) { prettyError("Skill bundling failed", err.message); stop(); return; } } const validationIssue = validateWorkerManifest(backgroundWorker.manifest); if (validationIssue) { prettyError(generationValidationIssueHeader(validationIssue), generateValidationIssueMessage(validationIssue, backgroundWorker.manifest, manifest), generateValidationIssueFooter(validationIssue)); stop(); return; } const sourceFiles = resolveSourceFiles(manifest.sources, backgroundWorker.manifest.tasks); const backgroundWorkerBody = { localOnly: true, metadata: { packageVersion: manifest.packageVersion, cliPackageVersion: manifest.cliPackageVersion, tasks: backgroundWorker.manifest.tasks, prompts: backgroundWorker.manifest.prompts, queues: backgroundWorker.manifest.queues, contentHash: manifest.contentHash, sourceFiles, runtime: backgroundWorker.manifest.runtime, runtimeVersion: backgroundWorker.manifest.runtimeVersion, }, engine: "V2", supportsLazyAttempts: true, }; const backgroundWorkerRecord = await this.options.client.createBackgroundWorker(this.options.config.project, backgroundWorkerBody); if (!backgroundWorkerRecord.success) { stop(); throw new Error(backgroundWorkerRecord.error); } backgroundWorker.serverWorker = backgroundWorkerRecord.data; this.#registerWorker(backgroundWorker); this.lastManifest = manifest; this.latestWorkerId = backgroundWorker.serverWorker.id; eventBus.emit("backgroundWorkerInitialized", backgroundWorker); } /** * Tries to dequeue runs for all the active versions running. * For the latest version we will pull from the main queue, so we don't specify that. */ async #dequeueRuns() { if (!this.config) { throw new Error("No config, can't dequeue runs"); } if (!this.latestWorkerId) { //try again later logger.debug(`[DevSupervisor] dequeueRuns. No latest worker ID, trying again later`); setTimeout(() => this.#dequeueRuns(), this.config.dequeueIntervalWithoutRun); return; } if (this.runLimiter && this.runLimiter.activeCount + this.runLimiter.pendingCount > this.runLimiter.concurrency) { logger.debug(`[DevSupervisor] dequeueRuns. Run limit reached, trying again later`); setTimeout(() => this.#dequeueRuns(), this.config.dequeueIntervalWithoutRun); return; } try { //todo later we should track available resources and machines used, and pass them in here (it supports it) const result = await this.options.client.dev.dequeue({ currentWorker: this.latestWorkerId, oldWorkers: [], // This isn't even used on the server side, so we can just pass an empty array }); if (!result.success) { logger.debug(`[DevSupervisor] dequeueRuns. Failed to dequeue runs`, { error: result.error, }); setTimeout(() => this.#dequeueRuns(), this.config.dequeueIntervalWithoutRun); return; } //no runs, try again later if (result.data.dequeuedMessages.length === 0) { setTimeout(() => this.#dequeueRuns(), this.config.dequeueIntervalWithoutRun); return; } logger.debug(`[DevSupervisor] dequeueRuns. Results`, { dequeuedMessages: JSON.stringify(result.data.dequeuedMessages), }); //start runs for (const message of result.data.dequeuedMessages) { const worker = this.workers.get(message.backgroundWorker.friendlyId); if (!worker) { logger.debug(`[DevSupervisor] dequeueRuns. Dequeued a run but there's no BackgroundWorker so we can't execute it`, { run: message.run.friendlyId, workerId: message.backgroundWorker.friendlyId, }); //todo call the API to crash the run with a good message continue; } let runController = this.runControllers.get(message.run.friendlyId); if (runController) { logger.debug(`[DevSupervisor] dequeueRuns. Dequeuing a run that already has a runController`, { runController: message.run.friendlyId, }); //todo, what do we do here? //todo I think the run shouldn't exist and we should kill the process but TBC continue; } if (!worker.serverWorker) { logger.debug(`[DevSupervisor] dequeueRuns. Worker doesn't have a serverWorker`, { run: message.run.friendlyId, worker, }); continue; } if (!worker.manifest) { logger.debug(`[DevSupervisor] dequeueRuns. Worker doesn't have a manifest`, { run: message.run.friendlyId, worker, }); continue; } if (!this.taskRunProcessPool) { logger.debug(`[DevSupervisor] dequeueRuns. No task run process pool`, { run: message.run.friendlyId, worker, }); continue; } logger.debug("[DevSupervisor] dequeueRuns. Creating run controller", { run: message.run.friendlyId, worker, config: this.options.config, }); const legacyDevProcessCwdBehaviour = typeof this.options.config.legacyDevProcessCwdBehaviour === "boolean" ? this.options.config.legacyDevProcessCwdBehaviour : true; const cwd = legacyDevProcessCwdBehaviour === true ? undefined : worker.build.outputPath; //new run runController = new DevRunController({ runFriendlyId: message.run.friendlyId, worker: worker, httpClient: this.options.client, logLevel: this.options.args.logLevel, taskRunProcessPool: this.taskRunProcessPool, cwd, onFinished: () => { logger.debug("[DevSupervisor] Run finished", { runId: message.run.friendlyId }); //stop the run controller, and remove it runController?.stop(); this.runControllers.delete(message.run.friendlyId); this.#updateActiveRunsFile(); this.#unsubscribeFromRunNotifications(message.run.friendlyId); //stop the worker if it is deprecated and there are no more runs if (worker.deprecated) { this.#tryDeleteWorker(message.backgroundWorker.friendlyId).catch((err) => { logger.debug("[DevSupervisor] Failed to delete worker", { error: err }); }); } }, onSubscribeToRunNotifications: async (run, snapshot) => { this.#subscribeToRunNotifications(); }, onUnsubscribeFromRunNotifications: async (run, snapshot) => { this.#unsubscribeFromRunNotifications(run.friendlyId); }, }); this.runControllers.set(message.run.friendlyId, runController); this.#updateActiveRunsFile(); if (this.runLimiter) { this.runLimiter(() => runController.start(message)).then(() => { logger.debug("[DevSupervisor] Run started", { runId: message.run.friendlyId }); }); } else { //don't await for run completion, we want to dequeue more runs runController.start(message).then(() => { logger.debug("[DevSupervisor] Run started", { runId: message.run.friendlyId }); }); } } setTimeout(() => this.#dequeueRuns(), this.config.dequeueIntervalWithRun); } catch (error) { logger.debug(`[DevSupervisor] dequeueRuns. Error thrown`, { error }); //dequeue again setTimeout(() => this.#dequeueRuns(), this.config.dequeueIntervalWithoutRun); } } async #startPresenceConnection() { try { const eventSource = this.options.client.dev.presenceConnection(); // Regular "ping" messages eventSource.addEventListener("presence", (event) => { // logger.debug(`Presence ping received`, { event }); }); // Connection was lost and successfully reconnected eventSource.addEventListener("reconnect", (event) => { logger.debug("[DevSupervisor] Presence connection restored"); }); // Handle messages that might have been missed during disconnection eventSource.addEventListener("missed_events", (event) => { logger.debug("[DevSupervisor] Missed some presence events during disconnection"); }); // If you need to close it manually return () => { logger.info("[DevSupervisor] Closing presence connection"); eventSource.close(); }; // eslint-disable-next-line no-useless-catch } catch (error) { throw error; } } async #getEnvVars() { const environmentVariablesResponse = await this.options.client.getEnvironmentVariables(this.options.config.project); const OTEL_IMPORT_HOOK_INCLUDES = (this.options.config.instrumentedPackageNames ?? []).join(","); return { ...resolveLocalEnvVars(this.options.args.envFile, environmentVariablesResponse.success ? environmentVariablesResponse.data.variables : {}), NODE_ENV: "development", TRIGGER_API_URL: this.options.client.apiURL, TRIGGER_SECRET_KEY: this.options.client.accessToken, OTEL_EXPORTER_OTLP_COMPRESSION: "none", OTEL_IMPORT_HOOK_INCLUDES, }; } async #registerWorker(worker) { if (!worker.serverWorker) { return; } //deprecate other workers for (const [workerId, existingWorker] of this.workers.entries()) { if (workerId === worker.serverWorker.id) { continue; } existingWorker.deprecate(); this.#tryDeleteWorker(workerId).catch((err) => { logger.debug("[DevSupervisor] Failed to delete worker", { error: err }); }); } this.workers.set(worker.serverWorker.id, worker); } #createSocket() { const wsUrl = new URL(this.options.client.apiURL); wsUrl.pathname = "/dev-worker"; const socket = io(wsUrl.href, { transports: ["websocket"], extraHeaders: { Authorization: `Bearer ${this.options.client.accessToken}`, }, }); socket.on("run:notify", async ({ version, run }) => { logger.debug("[DevSupervisor] Received run notification", { version, run }); this.options.client.dev.sendDebugLog(run.friendlyId, { time: new Date(), message: "run:notify received by runner", }); const controller = this.runControllers.get(run.friendlyId); if (!controller) { logger.debug("[DevSupervisor] Ignoring notification, no local run ID", { runId: run.friendlyId, }); return; } await controller.getLatestSnapshot(); }); socket.on("connect", () => { logger.debug("[DevSupervisor] Connected to supervisor"); if (socket.recovered || this.socketIsReconnecting) { logger.debug("[DevSupervisor] Socket recovered"); eventBus.emit("socketConnectionReconnected", `Connection was recovered`); } this.socketIsReconnecting = false; for (const controller of this.runControllers.values()) { controller.resubscribeToRunNotifications(); } }); socket.on("connect_error", (error) => { logger.debug("[DevSupervisor] Connection error", { error }); }); socket.on("disconnect", (reason, description) => { logger.debug("[DevSupervisor] socket was disconnected", { reason, description, active: socket.active, }); if (reason === "io server disconnect") { // the disconnection was initiated by the server, you need to manually reconnect socket.connect(); } else { this.socketIsReconnecting = true; eventBus.emit("socketConnectionDisconnected", reason); } }); const interval = setInterval(() => { logger.debug("[DevSupervisor] Socket connections", { connections: Array.from(this.socketConnections), }); }, 5000); return socket; } #subscribeToRunNotifications() { const runFriendlyIds = Array.from(this.runControllers.keys()); if (!this.socket) { logger.debug("[DevSupervisor] Socket not connected"); return; } for (const id of runFriendlyIds) { this.socketConnections.add(id); } logger.debug("[DevSupervisor] Subscribing to run notifications", { runFriendlyIds, connections: Array.from(this.socketConnections), }); this.socket.emit("run:subscribe", { version: "1", runFriendlyIds }); } #unsubscribeFromRunNotifications(friendlyId) { if (!this.socket) { logger.debug("[DevSupervisor] Socket not connected"); return; } this.socketConnections.delete(friendlyId); logger.debug("[DevSupervisor] Unsubscribing from run notifications", { runFriendlyId: friendlyId, connections: Array.from(this.socketConnections), }); this.socket.emit("run:unsubscribe", { version: "1", runFriendlyIds: [friendlyId] }); } /** Deletes the worker if there are no active runs, after a delay */ /** * Maximum number of deprecated workers to keep around. * We retain a small buffer of old workers because the server may still * dequeue runs locked to a recently-deprecated worker version. * When the limit is exceeded, the oldest deprecated workers are cleaned up. */ static MAX_DEPRECATED_WORKERS = 2; async #tryDeleteWorker(friendlyId) { await awaitTimeout(5_000); this.#cleanupWorker(friendlyId); } #hasActiveRunsForWorker(friendlyId) { for (const controller of this.runControllers.values()) { try { if (controller.workerFriendlyId === friendlyId) return true; } catch { // workerFriendlyId may throw if the controller is in an unexpected state } } return false; } #cleanupWorker(friendlyId) { const worker = this.workers.get(friendlyId); if (!worker) { return; } if (this.#hasActiveRunsForWorker(friendlyId)) { logger.debug("[DevSupervisor] Worker still has active runs, skipping cleanup", { workerId: friendlyId, }); return; } if (worker.serverWorker?.version) { this.taskRunProcessPool?.deprecateVersion(worker.serverWorker?.version); } // Enforce limit on deprecated workers to bound disk usage. // We keep a few around because the server may still dequeue runs for them. this.#pruneDeprecatedWorkers(); } #pruneDeprecatedWorkers() { const deprecatedWorkers = []; for (const [id, worker] of this.workers.entries()) { if (!worker.deprecated) continue; if (!this.#hasActiveRunsForWorker(id)) { deprecatedWorkers.push({ id, worker }); } } // Keep the most recent deprecated workers, remove the rest if (deprecatedWorkers.length <= _a.MAX_DEPRECATED_WORKERS) { return; } // Remove oldest first (they appear first in insertion order of the Map) const toRemove = deprecatedWorkers.slice(0, deprecatedWorkers.length - _a.MAX_DEPRECATED_WORKERS); for (const { id, worker } of toRemove) { logger.debug("[DevSupervisor] Pruning old deprecated worker and cleaning up build dir", { workerId: id, version: worker.serverWorker?.version, }); if (worker.serverWorker?.version) { this.taskRunProcessPool?.deprecateVersion(worker.serverWorker.version); } worker.stop(); this.workers.delete(id); } } } _a = DevSupervisor; // Duplicate task ids (including across task types, e.g. a schedule and a // regular task sharing an id) are enforced server-side when the background // worker is registered, so both `dev` and `deploy` surface a single, // authoritative error from the backend rather than a separate client check. function validateWorkerManifest(manifest) { if (!manifest.tasks || manifest.tasks.length === 0) { return { type: "noTasksDefined" }; } return undefined; } function generationValidationIssueHeader(issue) { switch (issue.type) { case "noTasksDefined": { return `No tasks exported from your trigger files`; } } } function generateValidationIssueFooter(issue) { switch (issue.type) { case "noTasksDefined": { return cliLink("View the task docs", "https://trigger.dev/docs/tasks/overview"); } } } function generateValidationIssueMessage(issue, manifest, buildManifest) { switch (issue.type) { case "noTasksDefined": { return ` Files: ${buildManifest.files.map((file) => file.entry).join("\n")} Make sure you have at least one task exported from your trigger files. You may have defined a task and forgot to add the export statement: \`\`\`ts import { task } from "@trigger.dev/sdk/v3"; 👇 Don't forget this export const myTask = task({ id: "myTask", async run() { // Your task logic here } }); \`\`\` `.replace(/^ {8}/gm, ""); } default: { return `Unknown validation issue: ${issue}`; } } } //# sourceMappingURL=devSupervisor.js.map