@temporalio/worker
Version:
Temporal.io SDK Worker sub-package
516 lines • 21.2 kB
JavaScript
"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Runtime = void 0;
exports.makeTelemetryFilterString = makeTelemetryFilterString;
const node_util_1 = require("node:util");
const v8 = __importStar(require("node:v8"));
const fs = __importStar(require("node:fs"));
const os = __importStar(require("node:os"));
const heap_js_1 = require("heap-js");
const native = __importStar(require("@temporalio/core-bridge"));
const core_bridge_1 = require("@temporalio/core-bridge");
const internal_non_workflow_1 = require("@temporalio/common/lib/internal-non-workflow");
const common_1 = require("@temporalio/common");
const proto_1 = require("@temporalio/proto");
const time_1 = require("@temporalio/common/lib/time");
const logger_1 = require("./logger");
const connection_options_1 = require("./connection-options");
const utils_1 = require("./utils");
const pkg_1 = __importDefault(require("./pkg"));
function isForwardingLogger(opts) {
return Object.hasOwnProperty.call(opts, 'forward');
}
function isOtelCollectorExporter(opts) {
return Object.hasOwnProperty.call(opts, 'otel');
}
/**
* A helper to build a filter string for use in `RuntimeOptions.telemetryOptions.tracingFilter`.
*
* Example:
* ```
* telemetryOptions: {
* logging: {
* filter: makeTelemetryFilterString({ core: 'TRACE', other: 'DEBUG' });
* // ...
* },
* }
* ```
*/
function makeTelemetryFilterString(options) {
const { core, other } = { other: 'INFO', ...options };
return `${other},temporal_sdk_core=${core},temporal_client=${core},temporal_sdk=${core}`;
}
/** A logger that buffers logs from both Node.js and Rust Core and emits logs in the right order */
class BufferedLogger extends logger_1.DefaultLogger {
constructor(next) {
super('TRACE', (entry) => this.buffer.add(entry));
this.next = next;
this.buffer = new heap_js_1.Heap((a, b) => Number(a.timestampNanos - b.timestampNanos));
}
/** Flush all buffered logs into the logger supplied to the constructor */
flush() {
for (const entry of this.buffer) {
this.next.log(entry.level, entry.message, {
...entry.meta,
[logger_1.LogTimestamp]: entry.timestampNanos,
});
}
this.buffer.clear();
}
}
/**
* Core singleton representing an instance of the Rust Core SDK
*
* Use {@link install} in order to customize the server connection options or other global process options.
*/
class Runtime {
constructor(native, options) {
this.native = native;
this.options = options;
/** Track the number of pending creation calls into the tokio runtime to prevent shut down */
this.pendingCreations = 0;
/** Track the registered native objects to automatically shutdown when all have been deregistered */
this.backRefs = new Set();
this.stopPollingForLogs = false;
this.shutdownSignalCallbacks = new Set();
this.state = 'RUNNING';
/**
* Bound to `this` for use with `process.on` and `process.off`
*/
this.startShutdownSequence = () => {
this.state = 'SHUTTING_DOWN';
this.teardownShutdownHook();
for (const callback of this.shutdownSignalCallbacks) {
queueMicrotask(callback); // Run later
this.deregisterShutdownSignalCallback(callback);
}
};
if (this.isForwardingLogs()) {
const logger = (this.logger = new BufferedLogger(this.options.logger));
this.logPollPromise = this.initLogPolling(logger);
}
else {
this.logger = this.options.logger;
this.logPollPromise = Promise.resolve();
}
this.checkHeapSizeLimit();
this.setupShutdownHook();
}
/**
* Instantiate a new Core object and set it as the singleton instance
*
* If Core has already been instantiated with {@link instance} or this method,
* will throw a {@link IllegalStateError}.
*/
static install(options) {
if (this._instance !== undefined) {
if (this.instantiator === 'install') {
throw new common_1.IllegalStateError('Runtime singleton has already been installed');
}
else if (this.instantiator === 'instance') {
throw new common_1.IllegalStateError('Runtime singleton has already been instantiated. Did you start a Worker before calling `install`?');
}
}
return this.create(options, 'install');
}
/**
* Get or instantiate the singleton Core object
*
* If Core has not been instantiated with {@link install} or this method,
* a new Core instance will be installed and configured to connect to
* a local server.
*/
static instance() {
const existingInst = this._instance;
if (existingInst !== undefined) {
return existingInst;
}
return this.create(this.defaultOptions, 'instance');
}
/**
* Factory function for creating a new Core instance, not exposed because Core is meant to be used as a singleton
*/
static create(options, instantiator) {
const compiledOptions = this.compileOptions(options);
const runtime = (0, core_bridge_1.newRuntime)(compiledOptions.telemetryOptions);
// Remember the provided options in case Core is reinstantiated after being shut down
this.defaultOptions = options;
this.instantiator = instantiator;
this._instance = new this(runtime, compiledOptions);
return this._instance;
}
// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
static compileOptions(options) {
// eslint-disable-next-line deprecation/deprecation
const { logging, metrics, tracingFilter, ...otherTelemetryOpts } = options.telemetryOptions ?? {};
const defaultFilter = tracingFilter ??
makeTelemetryFilterString({
core: 'WARN',
other: 'ERROR',
});
const loggingFilter = logging?.filter;
// eslint-disable-next-line deprecation/deprecation
const forwardLevel = logging?.forward?.level;
const forwardLevelFilter = forwardLevel &&
makeTelemetryFilterString({
core: forwardLevel,
other: forwardLevel,
});
return {
shutdownSignals: options.shutdownSignals ?? ['SIGINT', 'SIGTERM', 'SIGQUIT', 'SIGUSR2'],
telemetryOptions: {
logging: !!logging && isForwardingLogger(logging)
? {
filter: loggingFilter ?? forwardLevelFilter ?? defaultFilter,
forward: {},
}
: {
filter: loggingFilter ?? defaultFilter,
console: {},
},
metrics: metrics && {
temporality: metrics.temporality,
...(isOtelCollectorExporter(metrics)
? {
otel: {
url: metrics.otel.url,
headers: metrics.otel.headers ?? {},
metricsExportInterval: (0, time_1.msToNumber)(metrics.otel.metricsExportInterval ?? '1s'),
useSecondsForDurations: metrics.otel.useSecondsForDurations,
},
}
: {
prometheus: {
bindAddress: metrics.prometheus.bindAddress,
unitSuffix: metrics.prometheus.unitSuffix,
countersTotalSuffix: metrics.prometheus.countersTotalSuffix,
useSecondsForDurations: metrics.prometheus.useSecondsForDurations,
},
}),
},
...(0, internal_non_workflow_1.filterNullAndUndefined)(otherTelemetryOpts ?? {}),
},
logger: options.logger ?? new logger_1.DefaultLogger('INFO'),
};
}
async initLogPolling(logger) {
if (!this.isForwardingLogs()) {
return;
}
const poll = (0, node_util_1.promisify)(core_bridge_1.pollLogs);
const doPoll = async () => {
const logs = await poll(this.native);
for (const log of logs) {
const meta = {
[logger_1.LogTimestamp]: (0, logger_1.timeOfDayToBigint)(log.timestamp),
sdkComponent: common_1.SdkComponent.core,
...log.fields,
};
logger.log(log.level, log.message, meta);
}
};
try {
for (;;) {
await doPoll();
logger.flush();
if (this.stopPollingForLogs) {
break;
}
await new Promise((resolve) => {
setTimeout(resolve, 3);
this.stopPollingForLogsCallback = resolve;
});
}
}
catch (error) {
// Log using the original logger instead of buffering
this.options.logger.warn('Error gathering forwarded logs from core', {
error,
sdkComponent: common_1.SdkComponent.worker,
});
}
finally {
logger.flush();
}
}
isForwardingLogs() {
const logger = this.options.telemetryOptions.logging;
return logger != null && isForwardingLogger(logger);
}
/**
* Flush any buffered logs.
*
* This is a noop in case the instance is configured with
* `logForwardingLevel=OFF`.
*/
flushLogs() {
if (this.isForwardingLogs()) {
const logger = this.logger;
logger.flush();
}
}
/**
* Create a Core Connection object to power Workers
*
* Hidden in the docs because it is only meant to be used internally by the Worker.
* @hidden
*/
async createNativeClient(options) {
const compiledServerOptions = (0, connection_options_1.compileConnectionOptions)({
...(0, connection_options_1.getDefaultConnectionOptions)(),
...(0, internal_non_workflow_1.filterNullAndUndefined)(options ?? {}),
});
if (options?.apiKey && compiledServerOptions.metadata?.['Authorization']) {
throw new TypeError('Both `apiKey` option and `Authorization` header were provided. Only one makes sense to use at a time.');
}
const clientOptions = {
...compiledServerOptions,
tls: (0, internal_non_workflow_1.normalizeTlsConfig)(compiledServerOptions.tls),
url: options?.tls ? `https://${compiledServerOptions.address}` : `http://${compiledServerOptions.address}`,
};
return await this.createNative((0, node_util_1.promisify)(core_bridge_1.newClient), this.native, clientOptions);
}
/**
* Close a native Client, if this is the last registered Client or Worker, shutdown the core and unset the singleton instance
*
* Hidden in the docs because it is only meant to be used internally by the Worker.
* @hidden
*/
async closeNativeClient(client) {
native.clientClose(client);
this.backRefs.delete(client);
await this.shutdownIfIdle();
}
/**
* Register a Worker, this is required for automatically shutting down when all Workers have been deregistered
*
* Hidden in the docs because it is only meant to be used internally by the Worker.
* @hidden
*/
async registerWorker(client, options) {
return await this.createNative((0, node_util_1.promisify)(native.newWorker), client, options);
}
/** @hidden */
async createReplayWorker(options) {
return await this.createNativeNoBackRef(async () => {
const fn = (0, node_util_1.promisify)(native.newReplayWorker);
const replayWorker = await fn(this.native, options);
this.backRefs.add(replayWorker.worker);
return replayWorker;
});
}
/**
* Push history to a replay worker's history pusher stream.
*
* Hidden in the docs because it is only meant to be used internally by the Worker.
*
* @hidden
*/
async pushHistory(pusher, workflowId, history) {
const encoded = (0, utils_1.byteArrayToBuffer)(proto_1.temporal.api.history.v1.History.encodeDelimited(history).finish());
return await (0, node_util_1.promisify)(native.pushHistory)(pusher, workflowId, encoded);
}
/**
* Close a replay worker's history pusher stream.
*
* Hidden in the docs because it is only meant to be used internally by the Worker.
*
* @hidden
*/
closeHistoryStream(pusher) {
native.closeHistoryStream(pusher);
}
/**
* Deregister a Worker, if this is the last registered Worker or Client, shutdown the core and unset the singleton instance
*
* Hidden in the docs because it is only meant to be used internally by the Worker.
* @hidden
*/
async deregisterWorker(worker) {
native.workerFinalizeShutdown(worker);
this.backRefs.delete(worker);
await this.shutdownIfIdle();
}
/**
* Create an ephemeral Temporal server.
*
* Hidden since it is meant to be used internally by the testing framework.
* @hidden
*/
async createEphemeralServer(options) {
return await this.createNative((0, node_util_1.promisify)(native.startEphemeralServer), this.native, options, pkg_1.default.version);
}
/**
* Shut down an ephemeral Temporal server.
*
* Hidden since it is meant to be used internally by the testing framework.
* @hidden
*/
async shutdownEphemeralServer(server) {
await (0, node_util_1.promisify)(native.shutdownEphemeralServer)(server);
this.backRefs.delete(server);
await this.shutdownIfIdle();
}
async createNative(f, ...args) {
return this.createNativeNoBackRef(async () => {
const ref = await f(...args);
this.backRefs.add(ref);
return ref;
});
}
async createNativeNoBackRef(f, ...args) {
this.pendingCreations++;
try {
try {
return await f(...args);
}
finally {
this.pendingCreations--;
}
}
catch (err) {
// Attempt to shutdown the runtime in case there's an error creating the
// native object to avoid leaving an idle Runtime.
await this.shutdownIfIdle();
throw err;
}
}
isIdle() {
return this.pendingCreations === 0 && this.backRefs.size === 0;
}
async shutdownIfIdle() {
if (this.isIdle())
await this.shutdown();
}
/**
* Shutdown and unset the singleton instance.
*
* If the runtime is polling on Core logs, wait for those logs to be collected.
*
* Hidden in the docs because it is only meant to be used for testing.
* @hidden
*/
async shutdown() {
delete Runtime._instance;
this.teardownShutdownHook();
this.stopPollingForLogs = true;
this.stopPollingForLogsCallback?.();
// This will effectively drain all logs
await this.logPollPromise;
await (0, node_util_1.promisify)(core_bridge_1.runtimeShutdown)(this.native);
}
/**
* Used by Workers to register for shutdown signals
*
* Hidden in the docs because it is only meant to be used internally by the Worker.
* @hidden
*/
registerShutdownSignalCallback(callback) {
if (this.state === 'RUNNING') {
this.shutdownSignalCallbacks.add(callback);
}
else {
queueMicrotask(callback);
}
}
/**
* Used by Workers to deregister handlers registered with {@link registerShutdownSignalCallback}
*
* Hidden in the docs because it is only meant to be used internally by the Worker.
* @hidden
*/
deregisterShutdownSignalCallback(callback) {
this.shutdownSignalCallbacks.delete(callback);
}
/**
* Set up the shutdown hook, listen on shutdownSignals
*/
setupShutdownHook() {
for (const signal of this.options.shutdownSignals) {
process.on(signal, this.startShutdownSequence);
}
}
/**
* Stop listening on shutdownSignals
*/
teardownShutdownHook() {
for (const signal of this.options.shutdownSignals) {
process.off(signal, this.startShutdownSequence);
}
}
checkHeapSizeLimit() {
if (process.platform === 'linux') {
// References:
// - https://facebookmicrosites.github.io/cgroup2/docs/memory-controller.html
// - https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt
const cgroupMemoryConstraint = this.tryReadNumberFileSync(/* cgroup v2 */ '/sys/fs/cgroup/memory.high') ??
this.tryReadNumberFileSync(/* cgroup v2 */ '/sys/fs/cgroup/memory.max') ??
this.tryReadNumberFileSync(/* cgroup v1 */ '/sys/fs/cgroup/memory/memory.limit_in_bytes');
const cgroupMemoryReservation = this.tryReadNumberFileSync(/* cgroup v2 */ '/sys/fs/cgroup/memory.low') ??
this.tryReadNumberFileSync(/* cgroup v2 */ '/sys/fs/cgroup/memory.min') ??
this.tryReadNumberFileSync(/* cgroup v1 */ '/sys/fs/cgroup/memory/soft_limit_in_bytes');
const applicableMemoryConstraint = cgroupMemoryReservation ?? cgroupMemoryConstraint;
if (applicableMemoryConstraint &&
applicableMemoryConstraint < os.totalmem() &&
applicableMemoryConstraint < v8.getHeapStatistics().heap_size_limit) {
let dockerArgs = '';
if (cgroupMemoryConstraint) {
dockerArgs += `--memory=${(0, utils_1.toMB)(cgroupMemoryConstraint, 0)}m `;
}
if (cgroupMemoryReservation) {
dockerArgs += `--memory-reservation=${(0, utils_1.toMB)(cgroupMemoryReservation, 0)}m `;
}
const suggestedOldSpaceSizeInMb = (0, utils_1.toMB)(applicableMemoryConstraint * 0.75, 0);
this.logger.warn(`This program is running inside a containerized environment with a memory constraint ` +
`(eg. '${dockerArgs}' or similar). Node itself does not consider this memory constraint ` +
`in how it manages its heap memory. There is consequently a high probability that ` +
`the process will crash due to running out of memory. To increase reliability, we recommend ` +
`adding '--max-old-space-size=${suggestedOldSpaceSizeInMb}' to your node arguments. ` +
`Refer to https://docs.temporal.io/develop/typescript/core-application#run-a-worker-on-docker ` +
`for more advice on tuning your Workers.`, { sdkComponent: common_1.SdkComponent.worker });
}
}
}
tryReadNumberFileSync(file) {
try {
const val = Number(fs.readFileSync(file, { encoding: 'ascii' }));
return isNaN(val) ? undefined : val;
}
catch (_e) {
return undefined;
}
}
}
exports.Runtime = Runtime;
/**
* Default options get overridden when Core is installed and are remembered in case Core is
* re-instantiated after being shut down
*/
Runtime.defaultOptions = {};
//# sourceMappingURL=runtime.js.map