UNPKG

openclaw

Version:

Multi-channel AI gateway with extensible messaging integrations

460 lines (459 loc) 23.6 kB
import { r as PACKAGE_LIFECYCLE_PENDING_RELATIVE_PATH, t as LEGACY_PACKAGE_INSTALL_GUARD_RELATIVE_PATH } from "./package-lifecycle-marker-DhfhzqrA.js"; import { r as racePromiseWithAbortSignal } from "./abort-signal-D2k14JsD.js"; import { t as runTasksWithConcurrency } from "./run-with-concurrency-Dtu208ef.js"; import { n as collectPackageDistInventory, t as PACKAGE_DIST_INVENTORY_RELATIVE_PATH } from "./package-dist-inventory-Bc5ACNcR.js"; import { a as compareWorkerBundlePaths, o as hashWorkerBundleManifest } from "./worker-bundle-hash-CxAJivBJ.js"; import { t as DEFAULT_WORKER_BUNDLE_ARCHIVE_LIMITS } from "./worker-bundle-limits-CF239rpc.js"; import { n as readWorkerBundleArchiveManifest } from "./worker-bundle-archive-BeoTON-C.js"; import { createRequire } from "node:module"; import { constants, createReadStream, readFileSync } from "node:fs"; import { isDeepStrictEqual } from "node:util"; import path from "node:path"; import fs$1 from "node:fs/promises"; import os from "node:os"; import { createHash } from "node:crypto"; import { satisfies, valid } from "semver"; import * as tar from "tar"; createRequire(import.meta.url); /** Visit static and dynamic module specifiers in a parsed TypeScript source file. */ function visitModuleSpecifiers(ts, sourceFile, visit, options = {}) { function walk(node) { let kind; let specifierNode; if (ts.isImportDeclaration(node) && ts.isStringLiteral(node.moduleSpecifier)) { kind = "import"; specifierNode = node.moduleSpecifier; } else if (ts.isExportDeclaration(node) && node.moduleSpecifier && ts.isStringLiteral(node.moduleSpecifier)) { kind = "export"; specifierNode = node.moduleSpecifier; } else if (ts.isCallExpression(node) && node.expression.kind === ts.SyntaxKind.ImportKeyword && node.arguments.length >= 1 && ts.isStringLiteralLike(node.arguments[0])) { kind = "dynamic-import"; specifierNode = node.arguments[0]; } else if (options.includeImportTypes && ts.isImportTypeNode(node) && ts.isLiteralTypeNode(node.argument) && ts.isStringLiteralLike(node.argument.literal)) { kind = "dynamic-import"; specifierNode = node.argument.literal; } else if (options.includeCommonJs && ts.isCallExpression(node) && ts.isIdentifier(node.expression) && node.expression.text === "require" && node.arguments.length >= 1 && ts.isStringLiteralLike(node.arguments[0])) { kind = "commonjs-require"; specifierNode = node.arguments[0]; } else if (options.includeCommonJs && ts.isImportEqualsDeclaration(node) && ts.isExternalModuleReference(node.moduleReference) && node.moduleReference.expression && ts.isStringLiteralLike(node.moduleReference.expression)) { kind = "commonjs-require"; specifierNode = node.moduleReference.expression; } else if (options.includeImportMetaUrl && ts.isNewExpression(node) && ts.isIdentifier(node.expression) && node.expression.text === "URL" && node.arguments?.length >= 2 && ts.isStringLiteralLike(node.arguments[0]) && ts.isPropertyAccessExpression(node.arguments[1]) && node.arguments[1].name.text === "url" && ts.isMetaProperty(node.arguments[1].expression) && node.arguments[1].expression.keywordToken === ts.SyntaxKind.ImportKeyword && node.arguments[1].expression.name.text === "meta") { kind = "import-meta-url"; specifierNode = node.arguments[0]; } if (specifierNode) visit({ kind, node, specifier: specifierNode.text, specifierNode }); ts.forEachChild(node, walk); } walk(sourceFile); } //#endregion //#region scripts/lib/package-dist-imports.mjs const ts = createRequire(import.meta.url)("typescript"); const JS_DIST_FILE_RE = /^dist\/.*\.(?:cjs|js|mjs)$/u; function normalizePackagePath(value) { return value.replace(/\\/gu, "/").replace(/^package\//u, ""); } function stripSpecifierSuffix(value) { return value.replace(/[?#].*$/u, ""); } function hasJavaScriptFileExtension(value) { return /\.(?:cjs|js|mjs)$/u.test(path.posix.basename(stripSpecifierSuffix(value))); } function resolveDistImportPath(importerPath, specifier) { if (!specifier.startsWith(".")) return null; const stripped = stripSpecifierSuffix(specifier); if (!stripped) return null; return path.posix.normalize(path.posix.join(path.posix.dirname(importerPath), stripped)); } function collectImportSpecifiers(source, importerPath) { const specifiers = []; const sourceFile = ts.createSourceFile(importerPath, source, ts.ScriptTarget.Latest, false, ts.ScriptKind.JS); visitModuleSpecifiers(ts, sourceFile, ({ kind, specifier }) => { if (specifier.startsWith(".") && (kind !== "import-meta-url" || hasJavaScriptFileExtension(specifier) && resolveDistImportPath(importerPath, specifier)?.startsWith("dist/"))) specifiers.push(specifier); }, { includeCommonJs: true, includeImportMetaUrl: true }); return specifiers; } /** Collect missing-file errors for relative imports inside package dist files. */ function collectPackageDistImportErrors(params) { const files = [...new Set(params.files.map(normalizePackagePath))]; const fileSet = new Set(files); const errors = []; const imports = params.imports ?? collectPackageDistImports({ files, readText: params.readText }); for (const { importerPath, importedPath } of imports) if (!fileSet.has(importedPath)) errors.push(`${importerPath} imports missing ${importedPath}`); return errors; } /** Collect relative dist import edges from package JavaScript files. */ function collectPackageDistImports(params) { const files = [...new Set(params.files.map(normalizePackagePath))]; const imports = []; for (const importerPath of files.toSorted((left, right) => left.localeCompare(right))) { if (!JS_DIST_FILE_RE.test(importerPath) || importerPath.includes("/node_modules/")) continue; const source = params.readText(importerPath); for (const specifier of collectImportSpecifiers(source, importerPath)) { const importedPath = resolveDistImportPath(importerPath, specifier); if (!importedPath) continue; imports.push({ importerPath, importedPath }); } } return imports; } //#endregion //#region scripts/package-source-dependencies.mjs const PRIVATE_WORKSPACE_VERSION = "0.0.0-private"; function dependenciesRecord(value, label) { if (value === void 0) return {}; if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${label} dependencies must be an object when present`); return value; } function validateBundledPackageDependencyAlignment({ bundledDependencies, bundledPackageLabel, rootDependencies, rootPackageLabel = "root package.json" }) { const bundled = dependenciesRecord(bundledDependencies, bundledPackageLabel); const root = dependenciesRecord(rootDependencies, rootPackageLabel); const aligned = []; for (const [name, version] of Object.entries(bundled)) { if (typeof version !== "string") throw new Error(`${bundledPackageLabel} dependency ${name} must declare a string version`); if (version === PRIVATE_WORKSPACE_VERSION) continue; const rootVersion = root[name]; if (rootVersion !== void 0 && typeof rootVersion !== "string") throw new Error(`${rootPackageLabel} dependency ${name} must declare a string version`); if (rootVersion !== version && rootVersion !== `workspace:${version}`) throw new Error(`${rootPackageLabel} must declare ${name}@${version} to bundle ${bundledPackageLabel} without duplicate dependencies`); aligned.push([name, version]); } return aligned; } //#endregion //#region src/infra/package-plugin-composition.ts /** Shared dependency ownership for release packages and private node distributions. */ function composePackagePlugins(packageJson, plugins) { const composed = structuredClone(packageJson); for (const { id, packageJson: plugin } of plugins) for (const section of ["dependencies", "optionalDependencies"]) { const dependencies = plugin[section] ?? {}; for (const [name, spec] of Object.entries(dependencies)) if (valid(spec) !== spec) throw new Error(`Selected plugin ${id} requires an exact dependency pin: ${name}@${spec}`); for (const existing of [composed.dependencies, composed.optionalDependencies]) validateBundledPackageDependencyAlignment({ bundledDependencies: dependencies, bundledPackageLabel: `selected plugin ${id}`, rootDependencies: { ...dependencies, ...existing } }); composed[section] = { ...composed[section], ...dependencies }; } for (const name of Object.keys(composed.dependencies ?? {})) delete composed.optionalDependencies?.[name]; for (const { id, packageJson: plugin } of plugins) for (const [name, range] of Object.entries(plugin.peerDependencies ?? {})) { const version = name === composed.name ? composed.version : composed.dependencies?.[name] ?? composed.optionalDependencies?.[name]; if (!version && !plugin.peerDependenciesMeta?.[name]?.optional || version && !satisfies(version, range)) throw new Error(`Selected plugin ${id} requires peer ${name}@${range} in the distribution`); } const exclusions = new Set(plugins.map(({ id }) => `!dist/extensions/${id}/**`)); if (composed.files) composed.files = composed.files.filter((entry) => !exclusions.has(entry)); return composed; } //#endregion //#region src/gateway/worker-environments/node-bootstrap-artifact.ts const BOOTSTRAP_LAUNCHER_FILES = ["openclaw.mjs", "node-version.mjs"]; const BOOTSTRAP_COPY_CONCURRENCY = 16; const IGNORED_PLUGIN_DIRECTORIES = /* @__PURE__ */ new Set([ "node_modules", "src", "test", "tests" ]); const METADATA_KEYS = [ "name", "version", "dependencies", "optionalDependencies", "peerDependencies", "peerDependenciesMeta" ]; function bootstrapPath(value) { if (!value || value.includes("\\") || value.includes("\0") || value.startsWith("/") || value.split("/").some((part) => !part || part === "." || part === "..")) throw new Error(`Unsafe node distribution path: ${value}`); return value; } async function readPackageManifest(root) { const value = JSON.parse(await fs$1.readFile(path.join(root, "package.json"), "utf8")); if (!value.name || valid(value.version) !== value.version) throw new Error("Node distribution requires a named package with an exact version"); return value; } function requireRunningBuild(options, text, version) { const info = JSON.parse(text); if (!options.runningBuildId || info.buildId !== options.runningBuildId || info.version !== version) throw new Error("Cloud bootstrap requires the running Gateway build; run pnpm build and restart the Gateway before provisioning"); return options.runningBuildId; } function assertBuiltImportClosure(root, files, label) { const errors = collectPackageDistImportErrors({ files, readText: (relative) => readFileSync(path.join(root, relative), "utf8") }); if (errors.length > 0) throw new Error(`Node distribution ${label} has an incomplete built import closure; rebuild and restart the Gateway: ${errors.slice(0, 5).join("; ")}`); } async function resolvePlugins(options, packageRoot) { const ids = /* @__PURE__ */ new Set(); return await Promise.all(options.plugins.map(async ({ id, root }) => { if (!/^[a-z0-9][a-z0-9_-]*$/u.test(id) || ids.has(id)) throw new Error(`Invalid or duplicate node bootstrap plugin: ${id}`); ids.add(id); const requestedRoot = await fs$1.realpath(root); const sourceRoot = path.join(packageRoot, "extensions", id); const bundledRoot = path.join(packageRoot, "dist", "extensions", id); const builtRoot = requestedRoot === sourceRoot ? bundledRoot : requestedRoot; const packageJson = await readPackageManifest(builtRoot); if (requestedRoot === sourceRoot) { const sourcePackage = await readPackageManifest(sourceRoot); if (METADATA_KEYS.some((key) => !isDeepStrictEqual(packageJson[key], sourcePackage[key]))) throw new Error(`Built plugin ${id} does not match source metadata; rebuild and restart the Gateway`); } if (JSON.parse(await fs$1.readFile(path.join(builtRoot, "openclaw.plugin.json"), "utf8")).id !== id) throw new Error(`Node bootstrap plugin identity does not match ${id}`); const entries = packageJson.openclaw?.runtimeExtensions ?? packageJson.openclaw?.extensions; if (!entries?.length) throw new Error(`Node bootstrap plugin ${id} has no runtime entry`); for (const entry of entries) { const relative = bootstrapPath(entry.replace(/^\.\//u, "").replace(/\.ts$/u, ".js")); await fs$1.access(path.join(builtRoot, relative)); } return { id, root: builtRoot, packageJson, bundled: builtRoot === bundledRoot }; })); } async function prepareNodeBootstrapArtifact(options, temporaryRoot) { const packageRoot = await fs$1.realpath(options.packageRoot); const sourcePackage = await readPackageManifest(packageRoot); if (sourcePackage.name !== "openclaw") throw new Error("Node bootstrap requires the running OpenClaw package root"); const buildInfoPath = path.join(packageRoot, "dist", "build-info.json"); const buildInfo = await fs$1.readFile(buildInfoPath, "utf8").catch((cause) => { throw new Error("Cloud bootstrap requires a built Gateway; run pnpm build and restart the Gateway before provisioning", { cause }); }); const buildId = requireRunningBuild(options, buildInfo, sourcePackage.version); const plugins = await resolvePlugins(options, packageRoot); const packageJson = composePackagePlugins(sourcePackage, plugins); const stagingRoot = path.join(temporaryRoot, "package"); await fs$1.mkdir(stagingRoot, { mode: 448 }); const staged = /* @__PURE__ */ new Map(); const directories = /* @__PURE__ */ new Map(); let expandedBytes = 0; let reservedEntries = 0; const reserveFile = (relative, bytes) => { bootstrapPath(relative); expandedBytes += bytes; reservedEntries += 1; if (reservedEntries > DEFAULT_WORKER_BUNDLE_ARCHIVE_LIMITS.maxEntries || expandedBytes > DEFAULT_WORKER_BUNDLE_ARCHIVE_LIMITS.maxExpandedBytes) throw new Error("Node bootstrap distribution exceeds its artifact limits"); }; const writeStagedFile = async (relative, contents, executable) => { const target = path.join(stagingRoot, relative); const directory = path.dirname(target); let created = directories.get(directory); if (!created) { created = fs$1.mkdir(directory, { recursive: true, mode: 448 }); directories.set(directory, created); } await created; const mode = executable ? 493 : 420; const entry = { path: `package/${relative}`, size: Buffer.byteLength(contents), sha256: createHash("sha256").update(contents).digest("hex") }; await fs$1.writeFile(target, contents, { mode }); staged.set(relative, { ...entry, mode: process.platform === "win32" ? 448 : (await fs$1.stat(target)).mode & 511 }); }; const writeFile = async (relative, contents) => { reserveFile(relative, Buffer.byteLength(contents)); await writeStagedFile(relative, contents, false); }; const copyFile = async (root, relative, destination = relative) => { bootstrapPath(relative); const source = path.join(root, relative); if (await fs$1.realpath(source) !== source) throw new Error(`Node distribution cannot contain symbolic links: ${relative}`); const handle = await fs$1.open(source, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); try { const before = await handle.stat(); if (!before.isFile()) throw new Error(`Invalid node distribution file: ${relative}`); reserveFile(destination, before.size); const contents = await handle.readFile(); const after = await handle.stat(); const current = await fs$1.lstat(source); if (contents.byteLength !== before.size || before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs || current.isSymbolicLink() || current.dev !== before.dev || current.ino !== before.ino || await fs$1.realpath(source) !== source) throw new Error(`Node distribution changed while packaging: ${relative}`); await writeStagedFile(destination, contents, (before.mode & 73) !== 0); } finally { await handle.close(); } }; const copyFiles = async (root, files, prefix = "") => { const result = await runTasksWithConcurrency({ tasks: [...new Set(files)].map((relative) => () => copyFile(root, relative, `${prefix}${relative}`)), limit: BOOTSTRAP_COPY_CONCURRENCY, errorMode: "stop" }); if (result.hasError) throw result.firstError; }; const externalPluginPrefixes = plugins.filter((plugin) => !plugin.bundled).map(({ id }) => `dist/extensions/${id}/`); const files = (await collectPackageDistInventory(packageRoot, { packageManifest: packageJson })).filter((relative) => !relative.startsWith("dist/worker/") && !externalPluginPrefixes.some((prefix) => relative.startsWith(prefix))); if (!files.includes("dist/entry.js") && !files.includes("dist/entry.mjs")) throw new Error("Cloud bootstrap is missing its built CLI entry; run pnpm build and restart the Gateway"); const scripts = (sourcePackage.files ?? []).filter((relative) => relative.startsWith("scripts/") && !relative.includes("*") && !relative.endsWith("/")); await copyFiles(packageRoot, [ ...BOOTSTRAP_LAUNCHER_FILES, ...files, ...scripts ].filter((relative) => relative !== LEGACY_PACKAGE_INSTALL_GUARD_RELATIVE_PATH)); packageJson.scripts = Object.fromEntries(Object.entries(packageJson.scripts ?? {}).filter(([name]) => [ "preinstall", "install", "postinstall" ].includes(name))); delete packageJson.devDependencies; for (const plugin of plugins) { if (plugin.bundled) continue; const pluginFiles = []; const visit = async (directory, relativeRoot = "") => { for (const child of await fs$1.readdir(directory, { withFileTypes: true })) { if (child.name.startsWith(".") || IGNORED_PLUGIN_DIRECTORIES.has(child.name)) continue; const relative = relativeRoot ? `${relativeRoot}/${child.name}` : child.name; if (relative.split("/").length > 64) throw new Error("Node bootstrap plugin exceeds its directory depth limit"); if (child.isDirectory()) await visit(path.join(directory, child.name), relative); else if (/\.(?:[cm]?js|json|wasm)$/u.test(child.name)) pluginFiles.push(relative); } }; await visit(plugin.root); await copyFiles(plugin.root, pluginFiles, `dist/extensions/${plugin.id}/`); } const bundledNames = /* @__PURE__ */ new Set([...packageJson.bundleDependencies ?? packageJson.bundledDependencies ?? [], ...Object.entries(packageJson.dependencies ?? {}).filter(([, spec]) => spec.startsWith("workspace:")).map(([name]) => name)]); for (const name of [...bundledNames].toSorted()) { bootstrapPath(name); const root = await fs$1.realpath(path.join(packageRoot, "node_modules", name)); const bundled = await readPackageManifest(root); if (bundled.name !== name) throw new Error(`Bundled node distribution dependency identity does not match ${name}`); const dependencies = validateBundledPackageDependencyAlignment({ bundledDependencies: bundled.dependencies, bundledPackageLabel: `bundled ${name}`, rootDependencies: packageJson.dependencies }); for (const [dependency, version] of dependencies) packageJson.dependencies[dependency] = version; const bundledFiles = await collectPackageDistInventory(root); if (bundledFiles.length === 0) throw new Error(`Bundled node dependency ${name} needs its compiled distribution; rebuild the Gateway`); await copyFiles(root, bundledFiles, `node_modules/${name}/`); delete bundled.dependencies; delete bundled.devDependencies; delete bundled.scripts; await writeFile(`node_modules/${name}/package.json`, `${JSON.stringify(bundled, null, 2)}\n`); assertBuiltImportClosure(path.join(stagingRoot, "node_modules", name), ["package.json", ...bundledFiles], name); packageJson.dependencies[name] = bundled.version; } packageJson.bundleDependencies = [...bundledNames].toSorted(); delete packageJson.bundledDependencies; for (const [name, spec] of Object.entries({ ...packageJson.optionalDependencies, ...packageJson.dependencies })) if (valid(spec) !== spec) throw new Error(`Node distribution requires an exact dependency pin: ${name}@${spec}`); await writeFile("package.json", `${JSON.stringify(packageJson, null, 2)}\n`); const inventory = [...staged.keys()].filter((entry) => entry.startsWith("dist/")).toSorted(); await writeFile(PACKAGE_DIST_INVENTORY_RELATIVE_PATH, `${JSON.stringify(inventory)}\n`); await writeFile(PACKAGE_LIFECYCLE_PENDING_RELATIVE_PATH, "pending\n"); assertBuiltImportClosure(stagingRoot, [...staged.keys()], packageJson.name); if (await fs$1.readFile(buildInfoPath, "utf8") !== buildInfo || !isDeepStrictEqual(await readPackageManifest(packageRoot), sourcePackage)) throw new Error("Gateway build changed while preparing cloud bootstrap; restart the Gateway and retry"); const tarballPath = path.join(temporaryRoot, "node-runtime.tgz"); const manifest = [...staged.values()].toSorted((left, right) => compareWorkerBundlePaths(left.path, right.path)); await tar.create({ cwd: temporaryRoot, file: tarballPath, gzip: true, noDirRecurse: true, noMtime: true, portable: true, strict: true }, manifest.map((entry) => entry.path)); const tarballBytes = (await fs$1.stat(tarballPath)).size; if (tarballBytes > 536870912) throw new Error("Node bootstrap archive exceeds the transfer limit"); const archiveManifest = await readWorkerBundleArchiveManifest(tarballPath, DEFAULT_WORKER_BUNDLE_ARCHIVE_LIMITS); if (hashWorkerBundleManifest(manifest) !== hashWorkerBundleManifest(archiveManifest)) throw new Error("Node bootstrap archive does not match the staged distribution"); const hash = createHash("sha256"); for await (const chunk of createReadStream(tarballPath)) hash.update(chunk); await fs$1.rm(stagingRoot, { recursive: true, force: true }); return Object.freeze({ tarballPath, tarballSha256: hash.digest("hex"), tarballBytes, openclawVersion: packageJson.version, buildId, enabledPluginIds: Object.freeze(plugins.map(({ id }) => id).toSorted()) }); } /** Owns one immutable deployment artifact for this Gateway process, never the live installation. */ function createNodeBootstrapArtifactProvider(options) { let prepared; let temporaryRoot; let closed = false; const consumers = /* @__PURE__ */ new Map(); return { async prepare(signal) { signal?.throwIfAborted(); if (closed) throw new Error("Node bootstrap artifact provider is closed"); prepared ??= (async () => { try { temporaryRoot = await fs$1.mkdtemp(path.join(os.tmpdir(), "openclaw-node-runtime-")); if (closed) throw new Error("Node bootstrap artifact provider is closed"); const artifact = await prepareNodeBootstrapArtifact(options, temporaryRoot); if (closed) throw new Error("Node bootstrap artifact provider is closed"); return artifact; } catch (error) { if (temporaryRoot) await fs$1.rm(temporaryRoot, { recursive: true, force: true }); temporaryRoot = void 0; prepared = void 0; throw error; } })(); const artifact = await racePromiseWithAbortSignal(prepared, signal); signal?.throwIfAborted(); if (closed) throw new Error("Node bootstrap artifact provider is closed"); if (signal && !consumers.has(signal)) consumers.set(signal, new Promise((resolve) => { signal.addEventListener("abort", () => { consumers.delete(signal); resolve(); }, { once: true }); })); return artifact; }, async close() { closed = true; await prepared?.catch(() => void 0); await Promise.all(consumers.values()); if (temporaryRoot) { await fs$1.rm(temporaryRoot, { recursive: true, force: true }); temporaryRoot = void 0; } } }; } //#endregion export { createNodeBootstrapArtifactProvider };