@temporalio/worker
Version:
Temporal.io SDK Worker sub-package
366 lines (333 loc) • 13.6 kB
text/typescript
import * as realFS from 'node:fs';
import { builtinModules } from 'node:module';
import path from 'node:path';
import util from 'node:util';
import * as unionfs from 'unionfs';
import * as memfs from 'memfs';
import { Configuration, webpack } from 'webpack';
import { DefaultLogger, Logger, hasColorSupport } from '../logger';
import { toMB } from '../utils';
export const defaultWorkflowInterceptorModules = [require.resolve('../workflow-log-interceptor')];
export const allowedBuiltinModules = ['assert', 'url', 'util'];
export const disallowedBuiltinModules = builtinModules.filter((module) => !allowedBuiltinModules.includes(module));
export const disallowedModules = [
...disallowedBuiltinModules,
'@temporalio/activity',
'@temporalio/client',
'@temporalio/worker',
'@temporalio/common/lib/internal-non-workflow',
'@temporalio/interceptors-opentelemetry/lib/client',
'@temporalio/interceptors-opentelemetry/lib/worker',
'@temporalio/testing',
'@temporalio/core-bridge',
];
export function moduleMatches(userModule: string, modules: string[]): boolean {
return modules.some((module) => userModule === module || userModule.startsWith(`${module}/`));
}
export interface WorkflowBundleWithSourceMap {
/**
* Source maps are generated inline - this is no longer used
* @deprecated
*/
sourceMap: string;
code: string;
}
/**
* Builds a V8 Isolate by bundling provided Workflows using webpack.
*
* @param workflowsPath all Workflows found in path will be put in the bundle
* @param workflowInterceptorModules list of interceptor modules to register on Workflow creation
*/
export class WorkflowCodeBundler {
private foundProblematicModules = new Set<string>();
public readonly logger: Logger;
public readonly workflowsPath: string;
public readonly workflowInterceptorModules: string[];
protected readonly payloadConverterPath?: string;
protected readonly failureConverterPath?: string;
protected readonly ignoreModules: string[];
protected readonly webpackConfigHook: (config: Configuration) => Configuration;
constructor({
logger,
workflowsPath,
payloadConverterPath,
failureConverterPath,
workflowInterceptorModules,
ignoreModules,
webpackConfigHook,
}: BundleOptions) {
this.logger = logger ?? new DefaultLogger('INFO');
this.workflowsPath = workflowsPath;
this.payloadConverterPath = payloadConverterPath;
this.failureConverterPath = failureConverterPath;
this.workflowInterceptorModules = workflowInterceptorModules ?? [];
this.ignoreModules = ignoreModules ?? [];
this.webpackConfigHook = webpackConfigHook ?? ((config) => config);
}
/**
* @return a {@link WorkflowBundle} containing bundled code, including inlined source map
*/
public async createBundle(): Promise<WorkflowBundleWithSourceMap> {
const vol = new memfs.Volume();
const ufs = new unionfs.Union();
/**
* readdir and exclude sourcemaps and d.ts files
*/
function readdir(...args: Parameters<typeof realFS.readdir>) {
// Help TS a bit because readdir has multiple signatures
const callback: (err: NodeJS.ErrnoException | null, files: string[]) => void = args.pop() as any;
const newArgs: Parameters<typeof realFS.readdir> = [
...args,
(err: Error | null, files: string[]) => {
if (err !== null) {
callback(err, []);
return;
}
callback(
null,
files.filter((f) => /\.[jt]s$/.test(path.extname(f)) && !f.endsWith('.d.ts'))
);
},
] as any;
return realFS.readdir(...newArgs);
}
// Cast because the type definitions are inaccurate
const memoryFs = memfs.createFsFromVolume(vol);
ufs.use(memoryFs as any).use({ ...realFS, readdir: readdir as any });
const distDir = '/dist';
const entrypointPath = this.makeEntrypointPath(ufs, this.workflowsPath);
this.genEntrypoint(vol, entrypointPath);
const bundleFilePath = await this.bundle(ufs, memoryFs, entrypointPath, distDir);
let code = memoryFs.readFileSync(bundleFilePath, 'utf8') as string;
// Replace webpack's module cache with an object injected by the runtime.
// This is the key to reusing a single v8 context.
code = code.replace(
'var __webpack_module_cache__ = {}',
'var __webpack_module_cache__ = globalThis.__webpack_module_cache__'
);
this.logger.info('Workflow bundle created', { size: `${toMB(code.length)}MB` });
// Cast because the type definitions are inaccurate
return {
sourceMap: 'deprecated: this is no longer in use\n',
code,
};
}
protected makeEntrypointPath(fs: typeof unionfs.ufs, workflowsPath: string): string {
const stat = fs.statSync(workflowsPath);
if (stat.isFile()) {
// workflowsPath is a file; make the entrypoint a sibling of that file
const { root, dir, name } = path.parse(workflowsPath);
return path.format({ root, dir, base: `${name}-autogenerated-entrypoint.cjs` });
} else {
// workflowsPath is a directory; make the entrypoint a sibling of that directory
const { root, dir, base } = path.parse(workflowsPath);
return path.format({ root, dir, base: `${base}-autogenerated-entrypoint.cjs` });
}
}
/**
* Creates the main entrypoint for the generated webpack library.
*
* Exports all detected Workflow implementations and some workflow libraries to be used by the Worker.
*/
protected genEntrypoint(vol: typeof memfs.vol, target: string): void {
const interceptorImports = [...new Set(this.workflowInterceptorModules)]
.map((v) => `require(/* webpackMode: "eager" */ ${JSON.stringify(v)})`)
.join(', \n');
const code = `
const api = require('@temporalio/workflow/lib/worker-interface.js');
exports.api = api;
const { overrideGlobals } = require('@temporalio/workflow/lib/global-overrides.js');
overrideGlobals();
exports.importWorkflows = function importWorkflows() {
return require(/* webpackMode: "eager" */ ${JSON.stringify(this.workflowsPath)});
}
exports.importInterceptors = function importInterceptors() {
return [
${interceptorImports}
];
}
`;
try {
vol.mkdirSync(path.dirname(target), { recursive: true });
} catch (err: any) {
if (err.code !== 'EEXIST') throw err;
}
vol.writeFileSync(target, code);
}
/**
* Run webpack
*/
protected async bundle(
inputFilesystem: typeof unionfs.ufs,
outputFilesystem: memfs.IFs,
entry: string,
distDir: string
): Promise<string> {
const captureProblematicModules: Configuration['externals'] = async (data, _callback): Promise<undefined> => {
// Ignore the "node:" prefix if any.
const module: string = data.request?.startsWith('node:')
? data.request.slice('node:'.length)
: data.request ?? '';
if (moduleMatches(module, disallowedModules) && !moduleMatches(module, this.ignoreModules)) {
this.foundProblematicModules.add(module);
}
return undefined;
};
const options: Configuration = {
resolve: {
// https://webpack.js.org/configuration/resolve/#resolvemodules
modules: [path.resolve(__dirname, 'module-overrides'), 'node_modules'],
extensions: ['.ts', '.js'],
extensionAlias: { '.js': ['.ts', '.js'] },
alias: {
__temporal_custom_payload_converter$: this.payloadConverterPath ?? false,
__temporal_custom_failure_converter$: this.failureConverterPath ?? false,
...Object.fromEntries([...this.ignoreModules, ...disallowedModules].map((m) => [m, false])),
},
},
externals: captureProblematicModules,
module: {
rules: [
{
test: /\.js$/,
enforce: 'pre',
use: [require.resolve('source-map-loader')],
},
{
test: /\.ts$/,
exclude: /node_modules/,
use: {
loader: require.resolve('swc-loader'),
options: {
sourceMap: true,
jsc: {
target: 'es2017',
parser: {
syntax: 'typescript',
decorators: true,
},
},
},
},
},
],
},
entry: [entry],
mode: 'development',
devtool: 'inline-source-map',
output: {
path: distDir,
filename: 'workflow-bundle-[fullhash].js',
devtoolModuleFilenameTemplate: '[absolute-resource-path]',
library: '__TEMPORAL__',
},
ignoreWarnings: [/Failed to parse source map/],
};
const compiler = webpack(this.webpackConfigHook(options));
// Cast to any because the type declarations are inaccurate
compiler.inputFileSystem = inputFilesystem as any;
// Don't use ufs due to a strange bug on Windows:
// https://github.com/temporalio/sdk-typescript/pull/554
compiler.outputFileSystem = outputFilesystem as any;
try {
return await new Promise<string>((resolve, reject) => {
compiler.run((err, stats) => {
if (stats !== undefined) {
const hasError = stats.hasErrors();
// To debug webpack build:
// const lines = stats.toString({ preset: 'verbose' }).split('\n');
const webpackOutput = stats.toString({
chunks: false,
colors: hasColorSupport(this.logger),
errorDetails: true,
});
this.logger[hasError ? 'error' : 'info'](webpackOutput);
if (hasError) {
reject(
new Error(
"Webpack finished with errors, if you're unsure what went wrong, visit our troubleshooting page at https://docs.temporal.io/develop/typescript/debugging#webpack-errors"
)
);
}
if (this.foundProblematicModules.size) {
const err = new Error(
`Your Workflow code (or a library used by your Workflow code) is importing the following disallowed modules:\n` +
Array.from(this.foundProblematicModules)
.map((module) => ` - '${module}'\n`)
.join('') +
`These modules can't be used in workflow context as they might break determinism.` +
`HINT: Consider the following options:\n` +
` • Make sure that activity code is not imported from workflow code. Use \`import type\` to import activity function signatures.\n` +
` • Move code that has non-deterministic behaviour to activities.\n` +
` • If you know for sure that a disallowed module will not be used at runtime, add its name to 'WorkerOptions.bundlerOptions.ignoreModules' in order to dismiss this warning.\n` +
`See also: https://typescript.temporal.io/api/namespaces/worker#workflowbundleoption and https://docs.temporal.io/typescript/determinism.`
);
reject(err);
}
const outputFilename = Object.keys(stats.compilation.assets)[0];
if (!err) {
resolve(path.join(distDir, outputFilename));
}
}
reject(err);
});
});
} finally {
await util.promisify(compiler.close).bind(compiler)();
}
}
}
/**
* Options for bundling Workflow code using Webpack
*/
export interface BundleOptions {
/**
* Path to look up workflows in, any function exported in this path will be registered as a Workflows when the bundle is loaded by a Worker.
*/
workflowsPath: string;
/**
* List of modules to import Workflow interceptors from.
*
* Modules should export an `interceptors` variable of type {@link WorkflowInterceptorsFactory}.
*/
workflowInterceptorModules?: string[];
/**
* Optional logger for logging Webpack output
*/
logger?: Logger;
/**
* Path to a module with a `payloadConverter` named export.
* `payloadConverter` should be an instance of a class that implements {@link PayloadConverter}.
*/
payloadConverterPath?: string;
/**
* Path to a module with a `failureConverter` named export.
* `failureConverter` should be an instance of a class that implements {@link FailureConverter}.
*/
failureConverterPath?: string;
/**
* List of modules to be excluded from the Workflows bundle.
*
* Use this option when your Workflow code references an import that cannot be used in isolation,
* e.g. a Node.js built-in module. Modules listed here **MUST** not be used at runtime.
*
* > NOTE: This is an advanced option that should be used with care.
*/
ignoreModules?: string[];
/**
* Before Workflow code is bundled with Webpack, `webpackConfigHook` is called with the Webpack
* {@link https://webpack.js.org/configuration/ | configuration} object so you can modify it.
*/
webpackConfigHook?: (config: Configuration) => Configuration;
}
/**
* Create a bundle to pass to {@link WorkerOptions.workflowBundle}. Helpful for reducing Worker startup time in
* production.
*
* When using with {@link Worker.runReplayHistory}, make sure to pass the same interceptors and payload converter used
* when the history was generated.
*/
export async function bundleWorkflowCode(options: BundleOptions): Promise<WorkflowBundleWithSourceMap> {
const bundler = new WorkflowCodeBundler(options);
return await bundler.createBundle();
}