inngest
Version:
Official SDK for Inngest.com. Inngest is the reliability layer for modern applications. Inngest combines durable execution, events, and queues into a zero-infra platform with built-in observability.
429 lines (427 loc) • 16.6 kB
JavaScript
const require_rolldown_runtime = require('../../../../_virtual/rolldown_runtime.cjs');
const require_connect = require('../../../../proto/src/components/connect/protobuf/connect.cjs');
const require_util = require('../../util.cjs');
const require_buffer = require('../../buffer.cjs');
const require_types = require('../../types.cjs');
const require_url = require('../../../../helpers/url.cjs');
const require_messages = require('../../messages.cjs');
const require_handshake = require('./handshake.cjs');
const require_types$1 = require('./types.cjs');
const require_heartbeat = require('./heartbeat.cjs');
const require_requestProcessor = require('./requestProcessor.cjs');
const require_statusReporter = require('./statusReporter.cjs');
let __jpwilliams_waitgroup = require("@jpwilliams/waitgroup");
//#region src/components/connect/strategies/core/connection.ts
/**
* Shared connection core logic used by both SameThreadStrategy and
* WorkerThreadStrategy.
*
* This module uses a **reconcile loop** that continuously ensures a live
* WebSocket connection is open. Reconnection, drain, and shutdown are
* expressed as state changes that wake the loop rather than recursive
* calls or callback-driven control flow.
*
* Domain-specific logic is delegated to focused sub-modules:
* - {@link HeartbeatManager} — periodic heartbeat pings
* - {@link RequestProcessor} — executor requests, lease extensions, reply ACKs
* - {@link establishConnection} — HTTP start + WebSocket handshake
*/
/**
* Core connection manager that handles WebSocket connection lifecycle,
* handshake, heartbeat, lease extension, and reconnection.
*
* Uses a reconcile loop that:
* - Ensures a WebSocket connection is always open
* - Manages a single heartbeat interval targeting the active connection
* - Handles reconnection, drain, and shutdown as state changes
*/
var ConnectionCore = class ConnectionCore {
config;
callbacks;
_activeConnection;
_drainingConnection;
_shutdownRequested = false;
_inProgressRequests = {
wg: new __jpwilliams_waitgroup.WaitGroup(),
requestLeases: {},
requestMeta: {}
};
_lastHeartbeatSentAt;
_lastHeartbeatReceivedAt;
_lastMessageReceivedAt;
excludeGateways = /* @__PURE__ */ new Set();
wakeSignal;
pendingWakeReasons = [];
hasConnectedBefore = false;
shutdownDumpInterval;
static SHUTDOWN_DUMP_INTERVAL_MS = 6e4;
loopPromise;
closePromise;
resolveFirstReady;
rejectFirstReady;
useSigningKey;
heartbeatManager;
statusReporter;
requestProcessor;
constructor(config, callbacks) {
this.config = config;
this.callbacks = callbacks;
this.useSigningKey = config.hashedSigningKey;
let resolve;
this.wakeSignal = {
promise: new Promise((r) => {
resolve = r;
}),
resolve
};
const accessor = {
get activeConnection() {
return self._activeConnection;
},
get drainingConnection() {
return self._drainingConnection;
},
get shutdownRequested() {
return self._shutdownRequested;
},
get inProgressRequests() {
return self._inProgressRequests;
},
get appIds() {
return self.config.appIds;
}
};
const wakeSignalRef = { wake: (reason) => this.wake(reason) };
const self = this;
this.heartbeatManager = new require_heartbeat.HeartbeatManager(accessor, wakeSignalRef, callbacks.logger);
this.heartbeatManager.onHeartbeatSent = () => {
this._lastHeartbeatSentAt = Date.now();
};
this.statusReporter = new require_statusReporter.StatusReporter(accessor, callbacks.logger);
this.requestProcessor = new require_requestProcessor.RequestProcessor(accessor, wakeSignalRef, callbacks, callbacks.logger);
}
get connectionId() {
return this._activeConnection?.id;
}
/**
* Wait for all in-progress requests to complete.
*/
async waitForInProgress() {
await this._inProgressRequests.wg.wait();
}
/**
* Return a snapshot of debug/health information for this connection.
*/
getDebugState() {
return {
state: this.callbacks.getState(),
activeConnectionId: this._activeConnection?.id,
drainingConnectionId: this._drainingConnection?.id,
lastHeartbeatSentAt: this._lastHeartbeatSentAt,
lastHeartbeatReceivedAt: this._lastHeartbeatReceivedAt,
lastMessageReceivedAt: this._lastMessageReceivedAt,
shutdownRequested: this._shutdownRequested,
inFlightRequestCount: Object.keys(this._inProgressRequests.requestLeases).length,
inFlightRequests: Object.values(this._inProgressRequests.requestMeta)
};
}
/**
* Start the reconcile loop. Resolves when the first connection is active.
* The loop continues running in the background.
*/
async start(attempt = 0) {
if (typeof WebSocket === "undefined") throw new Error("WebSockets not supported in current environment");
if (this.callbacks.getState() === require_types.ConnectionState.CLOSED) throw new Error("Connection already closed");
const firstReadyPromise = new Promise((resolve, reject) => {
this.resolveFirstReady = resolve;
this.rejectFirstReady = reject;
});
this.loopPromise = this.reconcileLoop(attempt);
this.loopPromise.catch((err) => {
this.rejectFirstReady?.(err);
});
await firstReadyPromise;
}
/**
* Request graceful shutdown. Resolves when fully closed (in-flight done,
* connection closed).
*/
async close() {
if (this.closePromise) return this.closePromise;
this.closePromise = this.closeOnce();
return this.closePromise;
}
async closeOnce() {
const inFlightCount = Object.keys(this._inProgressRequests.requestLeases).length;
this.callbacks.logger.info({ inFlightCount }, "Shutting down, waiting for in-flight requests");
this._shutdownRequested = true;
this.dumpInFlightForShutdown("drain-start");
this.startShutdownInFlightDumpTimer();
if (this._activeConnection?.ws.readyState === WebSocket.OPEN) {
this._activeConnection.ws.send(require_buffer.ensureUnsharedArrayBuffer(require_connect.ConnectMessage.encode(require_connect.ConnectMessage.create({ kind: require_connect.GatewayMessageType.WORKER_PAUSE })).finish()));
this.callbacks.logger.info({ connectionId: this._activeConnection.id }, "Sent WORKER_PAUSE, draining");
}
this.wake(require_types$1.WAKE_REASON.ShutdownRequested);
if (this.loopPromise) await this.loopPromise;
this.callbacks.logger.info("Connection closed");
}
async getApiBaseUrl() {
return require_url.resolveApiBaseUrl({
apiBaseUrl: this.config.apiBaseUrl,
mode: this.config.mode
});
}
resetWakeSignal() {
let resolve;
this.wakeSignal = {
promise: new Promise((r) => {
resolve = r;
}),
resolve
};
}
wake(reason = require_types$1.WAKE_REASON.Unknown) {
const shouldResolve = this.pendingWakeReasons.length === 0;
this.pendingWakeReasons.push(reason);
if (shouldResolve) this.wakeSignal.resolve();
}
switchAuthKey() {
const switchToFallback = this.useSigningKey === this.config.hashedSigningKey;
if (switchToFallback) this.callbacks.logger.debug("Switching to fallback signing key");
this.useSigningKey = switchToFallback ? this.config.hashedFallbackKey : this.config.hashedSigningKey;
}
hasInFlightRequests() {
return Object.keys(this._inProgressRequests.requestLeases).length > 0;
}
/**
* Debug-level "still draining" dump emitted at drain start and periodically
* thereafter while in-flight requests are holding the shutdown. One summary
* line plus one line per request carrying `requestId`, `runId`, `stepId`,
* `functionSlug`, `ageMs`, and `sinceLastLeaseExtendMs`. Does not affect
* info/warn logs.
*
* `requestLeases` drives the reconcile-loop exit gate, so use it as the
* single source of truth for the in-flight set; `requestMeta` carries the
* enrichment fields and is kept in sync alongside the lease map.
*/
dumpInFlightForShutdown(reason) {
const leaseIds = Object.keys(this._inProgressRequests.requestLeases);
if (leaseIds.length === 0) return;
const now = Date.now();
const ages = [];
for (const id of leaseIds) {
const m = this._inProgressRequests.requestMeta[id];
if (m?.leaseAcquiredAt) ages.push(now - m.leaseAcquiredAt);
}
this.callbacks.logger.debug({
reason,
inFlightCount: leaseIds.length,
oldestAgeMs: ages.length > 0 ? Math.max(...ages) : void 0
}, "Shutdown: still draining");
for (const id of leaseIds) {
const m = this._inProgressRequests.requestMeta[id];
if (!m) continue;
this.callbacks.logger.debug({
reason,
requestId: m.requestId,
runId: m.runId,
stepId: m.stepId,
functionSlug: m.functionSlug,
appId: m.appId,
ageMs: m.leaseAcquiredAt ? now - m.leaseAcquiredAt : void 0,
sinceLastLeaseExtendMs: m.leaseLastExtendedAt ? now - m.leaseLastExtendedAt : void 0
}, "Shutdown: still draining in-flight request");
}
}
startShutdownInFlightDumpTimer() {
if (this.shutdownDumpInterval) return;
this.shutdownDumpInterval = setInterval(() => {
if (!this._shutdownRequested) return;
this.dumpInFlightForShutdown("periodic");
this.wake(require_types$1.WAKE_REASON.ShutdownStillPending);
}, ConnectionCore.SHUTDOWN_DUMP_INTERVAL_MS);
}
stopShutdownInFlightDumpTimer() {
if (this.shutdownDumpInterval) {
clearInterval(this.shutdownDumpInterval);
this.shutdownDumpInterval = void 0;
}
}
async reconcileLoop(initialAttempt) {
let attempt = initialAttempt;
this.callbacks.logger.debug({ initialAttempt }, "Reconcile loop entered");
while (true) {
if (this._shutdownRequested && !this.hasInFlightRequests()) break;
if (!this._activeConnection || this._activeConnection.dead) {
this.callbacks.logger.debug({
hasActiveConnection: !!this._activeConnection,
activeConnectionDead: this._activeConnection?.dead,
hasDrainingConnection: !!this._drainingConnection,
drainingConnectionId: this._drainingConnection?.id
}, "No active connection");
if (this.hasConnectedBefore) this.callbacks.logger.info({ attempt }, "Reconnecting");
else this.callbacks.logger.info("Connecting");
this.callbacks.onStateChange(this.hasConnectedBefore ? require_types.ConnectionState.RECONNECTING : require_types.ConnectionState.CONNECTING);
try {
const { conn, gatewayGroup } = await require_handshake.establishConnection(this.config, this.useSigningKey, attempt, this.excludeGateways, this.callbacks.logger);
this.attachHandlers(conn, gatewayGroup);
if (this._drainingConnection) {
this.callbacks.logger.info({
oldConnectionId: this._drainingConnection.id,
newConnectionId: conn.id
}, "Replaced draining connection");
this._drainingConnection.close();
this._drainingConnection = void 0;
}
this._activeConnection = conn;
this.heartbeatManager.updateInterval(conn.heartbeatIntervalMs);
this.statusReporter.updateInterval(conn.statusIntervalMs);
attempt = 0;
this.hasConnectedBefore = true;
this.callbacks.logger.info({
connectionId: conn.id,
gatewayGroup
}, "Connection active");
this.callbacks.onStateChange(require_types.ConnectionState.ACTIVE);
if (this._shutdownRequested) {
conn.ws.send(require_buffer.ensureUnsharedArrayBuffer(require_connect.ConnectMessage.encode(require_connect.ConnectMessage.create({ kind: require_connect.GatewayMessageType.WORKER_PAUSE })).finish()));
this.callbacks.logger.info({ connectionId: conn.id }, "Sent WORKER_PAUSE on reconnect during shutdown");
} else {
conn.ws.send(require_buffer.ensureUnsharedArrayBuffer(require_connect.ConnectMessage.encode(require_connect.ConnectMessage.create({ kind: require_connect.GatewayMessageType.WORKER_READY })).finish()));
this.callbacks.logger.info({ connectionId: conn.id }, "Sent WORKER_READY");
}
this.callbacks.onConnectionActive?.(this.useSigningKey);
this.resolveFirstReady?.();
this.resolveFirstReady = void 0;
this.rejectFirstReady = void 0;
} catch (err) {
if (!(err instanceof require_util.ReconnectError)) throw err;
attempt = err.attempt + 1;
if (err instanceof require_util.AuthError) this.switchAuthKey();
if (err instanceof require_util.ConnectionLimitError) this.callbacks.logger.error("Max concurrent connections reached");
if (err.message?.includes("connect_gateway_closing")) {
const jitter = 500 + Math.random() * 1e3;
this.callbacks.logger.info({
attempt,
delay: Math.round(jitter),
error: err.message
}, "Gateway draining, retrying");
if (await require_util.waitWithCancel(jitter, () => {
return this._shutdownRequested && !this.hasInFlightRequests();
})) break;
continue;
}
const delay = require_util.expBackoff(attempt);
this.callbacks.logger.info({
attempt,
delay
}, "Reconnecting after failure");
if (await require_util.waitWithCancel(delay, () => {
return this._shutdownRequested && !this.hasInFlightRequests();
})) break;
continue;
}
}
if (this.pendingWakeReasons.length === 0) await this.wakeSignal.promise;
const reasons = this.pendingWakeReasons;
this.pendingWakeReasons = [];
this.resetWakeSignal();
this.callbacks.logger.debug({
reasons,
shutdownRequested: this._shutdownRequested,
hasActiveConnection: !!this._activeConnection,
activeConnectionDead: this._activeConnection?.dead
}, "Reconcile loop woken");
}
this.callbacks.logger.debug({
shutdownRequested: this._shutdownRequested,
inFlightCount: Object.keys(this._inProgressRequests.requestLeases).length
}, "Reconcile loop exiting");
this.heartbeatManager.stop();
this.statusReporter.stop();
this.stopShutdownInFlightDumpTimer();
this._activeConnection?.close();
this._activeConnection = void 0;
this._drainingConnection?.close();
this._drainingConnection = void 0;
}
/**
* Wire up error, close, and message handlers on a newly-handshaked connection.
*/
attachHandlers(conn, gatewayGroup) {
const { ws } = conn;
const connectionId = conn.id;
ws.onerror = (ev) => {
if (conn.dead) return;
const uptimeMs = Date.now() - conn.connectedAt;
this.callbacks.logger.warn({
connectionId,
gatewayGroup,
uptimeMs,
error: ev?.message
}, "Connection lost (error)");
conn.dead = true;
this.excludeGateways.add(gatewayGroup);
if (this._activeConnection?.id === connectionId) this._activeConnection = void 0;
this.wake(require_types$1.WAKE_REASON.WsError);
};
ws.onclose = (ev) => {
if (conn.dead) return;
const uptimeMs = Date.now() - conn.connectedAt;
this.callbacks.logger.warn({
connectionId,
gatewayGroup,
uptimeMs,
code: ev.code,
reason: ev.reason
}, "Connection lost (close)");
conn.dead = true;
this.excludeGateways.add(gatewayGroup);
if (this._activeConnection?.id === connectionId) this._activeConnection = void 0;
this.wake(require_types$1.WAKE_REASON.WsClose);
};
ws.onmessage = async (event) => {
this._lastMessageReceivedAt = Date.now();
const connectMessage = require_messages.parseConnectMessage(new Uint8Array(event.data));
if (connectMessage.kind === require_connect.GatewayMessageType.GATEWAY_CLOSING) {
const uptimeMs = Date.now() - conn.connectedAt;
this.callbacks.logger.info({
connectionId: conn.id,
gatewayGroup,
uptimeMs
}, "Gateway draining, opening new connection");
this._drainingConnection = this._activeConnection;
this._activeConnection = void 0;
this.wake(require_types$1.WAKE_REASON.GatewayClosing);
return;
}
if (connectMessage.kind === require_connect.GatewayMessageType.GATEWAY_HEARTBEAT) {
this._lastHeartbeatReceivedAt = Date.now();
conn.pendingHeartbeats = 0;
this.callbacks.logger.debug({ connectionId }, "Handled gateway heartbeat");
return;
}
if (connectMessage.kind === require_connect.GatewayMessageType.GATEWAY_EXECUTOR_REQUEST) {
await this.requestProcessor.handleExecutorRequest(connectMessage, conn);
return;
}
if (connectMessage.kind === require_connect.GatewayMessageType.WORKER_REPLY_ACK) {
this.requestProcessor.handleReplyAck(connectMessage, connectionId);
return;
}
if (connectMessage.kind === require_connect.GatewayMessageType.WORKER_REQUEST_EXTEND_LEASE_ACK) {
this.requestProcessor.handleExtendLeaseAck(connectMessage, connectionId);
return;
}
this.callbacks.logger.warn({
kind: require_connect.gatewayMessageTypeToJSON(connectMessage.kind),
rawKind: connectMessage.kind,
state: this.callbacks.getState(),
connectionId
}, "Unexpected message type");
};
}
};
//#endregion
exports.ConnectionCore = ConnectionCore;
//# sourceMappingURL=connection.cjs.map