trigger.dev
Version:
A Command-Line Interface for Trigger.dev projects
490 lines • 20.7 kB
JavaScript
import { makeRe } from "minimatch";
import { access, mkdir, symlink } from "node:fs/promises";
import { dirname, join } from "node:path";
import { readPackageJSON, resolvePackageJSON } from "pkg-types";
import nodeResolve from "resolve";
import { alwaysExternal, } from "@trigger.dev/core/v3/build";
import { logger } from "../utilities/logger.js";
import { resolvePathSync as esmResolveSync } from "mlly";
import braces from "braces";
import { builtinModules } from "node:module";
import { tryCatch } from "@trigger.dev/core/v3";
import { resolveModule } from "./resolveModule.js";
/**
* externals in dev might not be resolvable from the worker directory
* for example, if the external is not an immediate dependency of the project
* and the project is not hoisting the dependency (e.g. pnpm, npm with nested)
*
* This function will create a symbolic link from a place where the external is resolvable
* to the actual resolved external path
*/
async function linkUnresolvableExternals(externals, resolveDir, logger) {
for (const external of externals) {
if (!(await isExternalResolvable(external, resolveDir, logger))) {
await linkExternal(external, resolveDir, logger);
}
}
}
async function linkExternal(external, resolveDir, logger) {
const destinationPath = join(resolveDir, "node_modules");
await mkdir(destinationPath, { recursive: true });
logger.debug("[externals] Make a symbolic link", {
fromPath: external.path,
destinationPath,
external,
});
// For scoped packages, we need to ensure the scope directory exists
if (external.name.startsWith("@")) {
// Get the scope part (e.g., '@huggingface')
const scopeDir = external.name.split("/")[0];
if (scopeDir) {
const scopePath = join(destinationPath, scopeDir);
logger.debug("[externals] Ensure scope directory exists", {
scopeDir,
scopePath,
});
await mkdir(scopePath, { recursive: true });
}
else {
logger.debug("[externals] Unable to get the scope directory", {
external,
});
}
}
const symbolicLinkPath = join(destinationPath, external.name);
// Make sure the symbolic link does not exist
try {
await symlink(external.path, symbolicLinkPath, "dir");
}
catch (e) {
logger.debug("[externals] Unable to create symbolic link", {
error: e,
fromPath: external.path,
destinationPath,
external,
});
}
}
async function isExternalResolvable(external, resolveDir, logger) {
try {
const resolvedPath = resolveSync(external.name, resolveDir);
logger.debug("[externals][isExternalResolvable] Resolved external", {
resolveDir,
external,
resolvedPath,
});
if (!resolvedPath.includes(external.path)) {
logger.debug("[externals][isExternalResolvable] resolvedPath does not match the external.path", {
resolveDir,
external,
resolvedPath,
});
return false;
}
return true;
}
catch (e) {
logger.debug("[externals][isExternalResolvable] Unable to resolve external", {
resolveDir,
external,
error: e,
});
return false;
}
}
function createExternalsCollector(target, resolvedConfig, forcedExternal = []) {
const externals = [];
const maybeExternals = discoverMaybeExternals(target, resolvedConfig, forcedExternal);
// Cache: resolvedPath (dir) -> packageJsonPath (null = failed to resolve)
const packageJsonCache = new Map();
// Cache: packageRoot (dir) -> boolean (true = mark as external)
const isExternalCache = new Map();
return {
externals,
plugin: {
name: "externals",
setup: (build) => {
build.onStart(async () => {
externals.splice(0);
isExternalCache.clear();
});
const autoDetectExternal = resolvedConfig.build?.autoDetectExternal ??
resolvedConfig.build?.experimental_autoDetectExternal ??
true;
build.onEnd(async () => {
logger.debug("[externals][onEnd] Collected externals", {
externals,
maybeExternals,
autoDetectExternal,
packageJsonCache: packageJsonCache.size,
isExternalCache: isExternalCache.size,
});
});
maybeExternals.forEach((external) => {
build.onResolve({ filter: external.filter, namespace: "file" }, async (args) => {
// Check if the external is already in the externals collection
if (externals.find((e) => e.name === external.raw)) {
return {
external: true,
};
}
const packageName = packageNameForImportPath(args.path);
try {
const resolvedPath = resolveSync(packageName, args.resolveDir);
logger.debug("[externals][onResolve] Resolved external", {
external,
resolvedPath,
args,
packageName,
});
const packageJsonPath = await resolvePackageJSON(dirname(resolvedPath));
if (!packageJsonPath) {
return undefined;
}
logger.debug("[externals][onResolve] Found package.json", {
packageJsonPath,
external,
resolvedPath,
args,
packageName,
});
const packageJson = await readPackageJSON(packageJsonPath);
if (!packageJson || !packageJson.name) {
return undefined;
}
if (!external.filter.test(packageJson.name)) {
logger.debug("[externals][onResolve] Package name does not match", {
external,
packageJson,
resolvedPath,
packageName,
});
return undefined;
}
if (!packageJson.version) {
logger.debug("[externals][onResolve] No version found in package.json", {
external,
packageJson,
resolvedPath,
});
return undefined;
}
externals.push({
name: packageName,
path: dirname(packageJsonPath),
version: packageJson.version,
});
logger.debug("[externals][onResolve] adding external to the externals collection", {
external,
resolvedPath,
args,
packageName,
resolvedExternal: {
name: packageJson.name,
path: dirname(packageJsonPath),
version: packageJson.version,
},
});
return {
external: true,
};
}
catch (error) {
logger.debug("[externals][onResolve] Unable to resolve external", {
external,
error,
args,
packageName,
});
return undefined;
}
});
});
if (autoDetectExternal) {
build.onResolve({ filter: /.*/, namespace: "file" }, async (args) => {
if (!isBareModuleImport(args.path)) {
// Not an npm package
return;
}
if (isBuiltinModule(args.path)) {
// Builtin module
return;
}
if (args.path === "_sentry-debug-id-injection-stub") {
// Ignore sentry stub
return;
}
// Try to resolve the actual file path
const [resolveError, resolvedPath] = await tryCatch(resolveModule(args.path, args.resolveDir));
if (resolveError) {
logger.debug("[externals][auto] Resolve module error", {
path: args.path,
resolveError,
});
return;
}
// Find nearest package.json
const packageJsonPath = await findNearestPackageJson(resolvedPath, packageJsonCache);
if (!packageJsonPath) {
logger.debug("[externals][auto] Failed to resolve package.json path", {
path: args.path,
resolvedPath,
});
return;
}
const packageRoot = dirname(packageJsonPath);
// Check cache first
if (isExternalCache.has(packageRoot)) {
const isExternal = isExternalCache.get(packageRoot);
if (isExternal) {
return { path: args.path, external: true };
}
return;
}
const [readError, packageJson] = await tryCatch(readPackageJSON(packageRoot));
if (readError) {
logger.debug("[externals][auto] Unable to read package.json", {
error: readError,
packageRoot,
});
isExternalCache.set(packageRoot, false);
return;
}
const packageName = packageJson.name;
const packageVersion = packageJson.version;
if (!packageName || !packageVersion) {
logger.debug("[externals][auto] No package name or version found in package.json", {
packageRoot,
packageJson,
});
return;
}
const markExternal = (reason) => {
const detectedPackage = {
name: packageName,
path: packageRoot,
version: packageVersion,
};
logger.debug(`[externals][auto] Marking as external - ${reason}`, {
detectedPackage,
});
externals.push(detectedPackage);
// Cache the result
isExternalCache.set(packageRoot, true);
return { path: args.path, external: true };
};
// If the path ends with .wasm or .node, we should mark it as external
if (resolvedPath.endsWith(".wasm") || resolvedPath.endsWith(".node")) {
return markExternal("path ends with .wasm or .node");
}
// Check files, main, module fields for native files
const files = Array.isArray(packageJson.files) ? packageJson.files : [];
const fields = [packageJson.main, packageJson.module, packageJson.browser].filter((f) => typeof f === "string");
const allFiles = files.concat(fields);
// We need to expand any braces in the files array, e.g. ["{js,ts}"] -> ["js", "ts"]
const allFilesExpanded = braces(allFiles, { expand: true });
// Use a regexp to match native-related extensions
const nativeExtRegexp = /\.(wasm|node|gyp|c|cc|cpp|cxx|h|hpp|hxx)$/;
const hasNativeFile = allFilesExpanded.some((file) => nativeExtRegexp.test(file));
if (hasNativeFile) {
return markExternal("has native file");
}
// Check if binding.gyp exists (native addon)
const bindingGypPath = join(packageRoot, "binding.gyp");
// If access succeeds, binding.gyp exists
const [accessError] = await tryCatch(access(bindingGypPath));
if (!accessError) {
return markExternal("binding.gyp exists");
}
// Cache the negative result
isExternalCache.set(packageRoot, false);
return undefined;
});
}
},
},
};
}
function discoverMaybeExternals(target, config, forcedExternal = []) {
const external = [];
for (const externalName of forcedExternal) {
const externalRegex = makeRe(externalName);
if (!externalRegex) {
continue;
}
external.push({
raw: externalName,
filter: new RegExp(`^${externalName}$|${externalRegex.source}`),
});
}
if (config.build?.external) {
for (const externalName of config.build?.external) {
const externalRegex = makeExternalRegexp(externalName);
if (!externalRegex) {
continue;
}
external.push({
raw: externalName,
filter: externalRegex,
});
}
}
for (const externalName of config.instrumentedPackageNames ?? []) {
const externalRegex = makeExternalRegexp(externalName);
if (!externalRegex) {
continue;
}
external.push({
raw: externalName,
filter: externalRegex,
});
}
for (const buildExtension of config.build?.extensions ?? []) {
const moduleExternals = buildExtension.externalsForTarget?.(target);
for (const externalName of moduleExternals ?? []) {
const externalRegex = makeExternalRegexp(externalName);
if (!externalRegex) {
continue;
}
external.push({
raw: externalName,
filter: externalRegex,
});
}
}
return external;
}
export function createExternalsBuildExtension(target, config, forcedExternal = []) {
const { externals, plugin } = createExternalsCollector(target, config, forcedExternal);
return {
name: "externals",
onBuildStart(context) {
context.registerPlugin(plugin, {
target,
// @ts-expect-error
placement: "$head", // cheat to get to the front of the plugins
});
},
onBuildComplete: async (context, manifest) => {
if (context.target === "dev") {
await linkUnresolvableExternals(externals, manifest.outputPath, context.logger);
}
context.addLayer({
id: "externals",
dependencies: externals.reduce((acc, external) => {
acc[external.name] = external.version;
return acc;
}, {}),
});
},
};
}
function makeExternalRegexp(packageName) {
// Escape special regex characters in the package name
const escapedPkg = packageName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
// Create the regex pattern
const pattern = `^${escapedPkg}(?:/[^'"]*)?$`;
return new RegExp(pattern);
}
function packageNameForImportPath(importPath) {
// Remove any leading '@' to handle it separately
const withoutAtSign = importPath.replace(/^@/, "");
// Split the path by '/'
const parts = withoutAtSign.split("/");
// Handle scoped packages
if (importPath.startsWith("@")) {
// Return '@org/package' for scoped packages
return "@" + parts.slice(0, 2).join("/");
}
else {
// Return just the first part for non-scoped packages
return parts[0];
}
}
export async function resolveAlwaysExternal(client) {
try {
const response = await client.retrieveExternals();
if (response.success) {
return response.data.externals;
}
return alwaysExternal;
}
catch (error) {
logger.debug("[externals][resolveAlwaysExternal] Unable to retrieve externals", {
error,
});
return alwaysExternal;
}
}
function resolveSync(id, resolveDir) {
try {
return nodeResolve.sync(id, { basedir: resolveDir });
}
catch (error) {
return esmResolveSync(id, { url: resolveDir });
}
}
function isBareModuleImport(path) {
const excludes = [".", "/", "~", "file:", "data:"];
return !excludes.some((exclude) => path.startsWith(exclude));
}
function isBuiltinModule(path) {
return builtinModules.includes(path.replace("node:", ""));
}
async function isMainPackageJson(filePath) {
try {
const packageJson = await readPackageJSON(filePath);
// Allowlist of non-informative fields that can appear with 'type: module | commonjs' in marker package.json files
const markerFields = new Set([
"type",
"sideEffects",
"browser",
"main",
"module",
"react-native",
"name",
]);
if (!packageJson.type) {
return true;
}
const keys = Object.keys(packageJson);
if (keys.every((k) => markerFields.has(k))) {
return false; // type marker
}
return true;
}
catch (error) {
if (!(error instanceof Error)) {
logger.debug("[externals][containsEsmTypeMarkers] Unknown error", {
error,
});
return false;
}
if ("code" in error && error.code !== "ENOENT") {
logger.debug("[externals][containsEsmTypeMarkers] Error", {
error: error.message,
});
}
return false;
}
}
async function findNearestPackageJson(basePath, cache) {
const baseDir = dirname(basePath);
if (cache.has(baseDir)) {
const resolvedPath = cache.get(baseDir);
if (!resolvedPath) {
return null;
}
return resolvedPath;
}
const [error, packageJsonPath] = await tryCatch(resolvePackageJSON(dirname(basePath), {
test: isMainPackageJson,
}));
if (error) {
cache.set(baseDir, null);
return null;
}
cache.set(baseDir, packageJsonPath);
return packageJsonPath;
}
//# sourceMappingURL=externals.js.map