UNPKG

@botpress/adk-cli

Version:

Command-line interface for the Botpress Agent Development Kit (ADK)

778 lines (765 loc) 25.4 kB
// @bun import { createDeployedAgentMapSnapshot, uploadDeployedAgentMapSnapshot } from "./chunk-r72adjnh.js"; import { AgentProject, BpDeployCommand, createDeployedAgentManifest, exports_dependencies, tagDeployedAgentManifestBot, toTableSyncFailureDetails, uploadDeployedAgentManifest } from "./chunk-p0hjqn4r.js"; import { EVAL_MANIFEST_SCHEMA_VERSION, EVAL_MANIFEST_TAGS } from "./chunk-t76d8fxx.js"; // src/utils/component-uploader.ts import { createHash } from "crypto"; import fs2 from "fs/promises"; import path from "path"; // src/utils/component-build-plugins.ts import fs from "fs"; import crypto from "crypto"; var REACT_GLOBAL = "(globalThis.__HOST_REACT__ ?? globalThis.React)"; var REACT_IMPORT_BINDING = "__react"; var REACT_MODULE_EXPORTS = [ "Children", "Component", "Fragment", "Profiler", "PureComponent", "StrictMode", "Suspense", "cloneElement", "createContext", "createElement", "createRef", "forwardRef", "isValidElement", "lazy", "memo", "startTransition", "use", "useActionState", "useCallback", "useContext", "useDebugValue", "useDeferredValue", "useEffect", "useId", "useImperativeHandle", "useInsertionEffect", "useLayoutEffect", "useMemo", "useOptimistic", "useReducer", "useRef", "useState", "useSyncExternalStore", "useTransition", "version" ]; function style(options = {}) { return { name: "style", setup({ onResolve, onLoad }) { onLoad({ filter: /\.css$/ }, async (args) => { const cssText = (await fs.promises.readFile(args.path, "utf8")).trimEnd(); const cssFileId = crypto.createHash("md5").update(args.path).digest("hex"); if (options.collect) { options.collect(cssText, cssFileId); return { contents: "", loader: "js" }; } return { contents: `import { inject_style } from "__style_helper__" inject_style(${JSON.stringify(cssText)}, "${cssFileId}")`, loader: "js" }; }); onResolve({ filter: /^__style_helper__$/ }, () => { return { path: "index.js", namespace: "style-helper" }; }); onLoad({ filter: /.*/, namespace: "style-helper" }, () => ({ contents: `export function inject_style(text, id) { ` + ` if (typeof document === 'undefined') { ` + ` return ` + ` } ` + ` if (document.getElementById(id)) { ` + ` return ` + ` } ` + ` var style = document.createElement('style') ` + ` style.setAttribute("id", id) ` + ` var node = document.createTextNode(text) ` + ` style.appendChild(node) ` + ` document.head.appendChild(style) ` + `}`, loader: "js" })); } }; } function globalReactPlugin() { return { name: "global-react-plugin", setup(build) { build.onResolve({ filter: /^react(\/jsx-runtime|\/jsx-dev-runtime)?$/ }, (args) => { return { path: args.path, namespace: "global-react-ns" }; }); build.onLoad({ filter: /.*/, namespace: "global-react-ns" }, (args) => { if (args.path === "react/jsx-runtime" || args.path === "react/jsx-dev-runtime") { return { contents: buildJsxRuntimeShim(), loader: "js" }; } return { contents: buildReactModuleShim(), loader: "js" }; }); build.onLoad({ filter: /\.(jsx?|tsx?)$/ }, async (args) => { let contents = await fs.promises.readFile(args.path, "utf8"); contents = rewriteReactImports(contents); return { contents, loader: args.path.endsWith(".ts") || args.path.endsWith(".tsx") ? "tsx" : "jsx" }; }); } }; } function buildReactModuleShim() { return [ `const R = ${REACT_GLOBAL};`, `if (!R) {`, ` throw new Error('Custom component React shim: host React global is not installed')`, `}`, `export default R;`, ...REACT_MODULE_EXPORTS.map((name) => `export const ${name} = R.${name};`) ].join(` `); } function buildJsxRuntimeShim() { return `const R = ${REACT_GLOBAL}; ` + `const RT = globalThis.__HOST_REACT_JSX_RUNTIME__; ` + `if (!R) { ` + ` throw new Error('Custom component JSX runtime shim: host React global is not installed') ` + `} ` + `function _jsx(type, props, key) { ` + ` if (props == null) return R.createElement(type, key !== undefined ? { key } : null); ` + ` const { children, ...rest } = props; ` + ` if (key !== undefined) rest.key = key; ` + ` return R.createElement(type, rest, children); ` + `} ` + `export const jsx = RT?.jsx ?? _jsx; ` + `export const jsxs = RT?.jsxs ?? _jsx; ` + `export const jsxDEV = RT?.jsxDEV ?? _jsx; ` + `export const Fragment = RT?.Fragment ?? R.Fragment;`; } var reactImportPattern = /import\s+((?:React\s*,\s*)?\{\s*[^}]+\s*\}|React|\*\s+as\s+[A-Za-z0-9_$]+)\s+from\s+['"]react['"];?/g; function buildNamedReactImportDeclarations(imports) { return imports.split(",").map((imp) => imp.trim()).filter((imp) => !imp.startsWith("type ")).map((imp) => { const [name, alias] = imp.split(/\s+as\s+/).map((s) => s.trim()); const varName = alias || name; return `const ${varName} = ${REACT_IMPORT_BINDING}.${name};`; }).join(` `); } function rewriteReactImports(text) { let hasReactImportBinding = false; const buildReactImportBindingDeclaration = () => { if (hasReactImportBinding) { return ""; } hasReactImportBinding = true; return `const ${REACT_IMPORT_BINDING} = ${REACT_GLOBAL}; `; }; return text.replace(reactImportPattern, (_, specifier) => { specifier = specifier.trim(); if (specifier === "React") { return `const React = ${REACT_GLOBAL};`; } if (specifier.startsWith("* as ")) { const name = specifier.slice("* as ".length).trim(); return `const ${name} = ${REACT_GLOBAL};`; } const importsStart = specifier.indexOf("{") + 1; const importsEnd = specifier.lastIndexOf("}"); const imports = specifier.slice(importsStart, importsEnd); const namedDeclarations = buildNamedReactImportDeclarations(imports); const hasDefaultReactImport = /^React\s*,/.test(specifier); if (!namedDeclarations) { return hasDefaultReactImport ? `const React = ${REACT_GLOBAL};` : ""; } const declarations = [buildReactImportBindingDeclaration()]; if (hasDefaultReactImport) { declarations.push(`const React = ${REACT_IMPORT_BINDING}; `); } declarations.push(namedDeclarations); return declarations.join(""); }); } // src/utils/component-uploader.ts var componentTags = (name) => ({ source: "integration", system: "true", integrationName: "webchat", feature: "custom-components", type: "component", name }); async function deleteComponentFiles(client, names) { const staleFiles = []; for (const name of names) { let nextToken; do { const response = await client.listFiles({ tags: componentTags(name), nextToken }); staleFiles.push(...response.files.map((f) => f.id)); nextToken = response.meta.nextToken; } while (nextToken); } if (staleFiles.length > 0) { await Promise.allSettled(staleFiles.map((id) => client.deleteFile({ id }))); } return staleFiles.length; } async function buildComponent(entryPoint) { const result = await Bun.build({ entrypoints: [entryPoint], format: "esm", external: ["react", "react-dom"], plugins: [style({}), globalReactPlugin()] }); if (!result.success) { const errors = result.logs.filter((l) => l.level === "error" || l.level === "warning").map((l) => l.message); if (!errors.length) errors.push("Build failed with no diagnostic output"); return { success: false, code: null, errors }; } const output = result.outputs[0]; if (!output) { return { success: false, code: null, errors: ["Build produced no output"] }; } const code = await output.text(); return { success: true, code, errors: [] }; } async function buildAndUploadComponents({ agentRoot, project, client, log, only }) { const components = only?.length ? project.customComponents.filter((c) => only.includes(c.definition.name)) : project.customComponents; const manifestPath = path.join(agentRoot, ".adk", "components.manifest.json"); let manifest = {}; try { manifest = JSON.parse(await fs2.readFile(manifestPath, "utf-8")); } catch {} const currentNames = new Set(project.customComponents.map((c) => c.definition.name)); const removedNames = Object.keys(manifest).filter((name) => !currentNames.has(name)); if (removedNames.length > 0) { log(`Cleaning up removed component(s): ${removedNames.join(", ")}`); await deleteComponentFiles(client, removedNames); for (const name of removedNames) { delete manifest[name]; } } const groups = new Map; for (const comp of components) { const list = groups.get(comp.source) ?? []; list.push(comp); groups.set(comp.source, list); } for (const [source, group] of groups) { const allNames = group.map((c) => c.definition.name).join(", "); log(`Building ${allNames}...`); let bundleText; try { const { success, code, errors } = await buildComponent(source); if (!success || !code) { log(`\u26A0 Build failed for ${allNames}: ${errors.join("; ")}`); log(`\u26A0 Reverting to previous version if available.`); continue; } bundleText = code; } catch (error) { log(`\u26A0 Failed to build ${allNames}: ${error instanceof Error ? error.message : String(error)}`); continue; } const hash = createHash("sha256").update(bundleText).digest("hex"); for (const comp of group) { const name = comp.definition.name; if (manifest[name]?.hash === hash) { log(`${name} unchanged, skipping upload`); continue; } try { log(`Uploading ${name}...`); const uploaded = await client.uploadFile({ content: bundleText, key: `${name}.js`, accessPolicies: ["public_content"], publicContentImmediatelyAccessible: true, contentType: "application/javascript", tags: componentTags(name) }); manifest[name] = { url: uploaded.file.url, deployedAt: new Date().toISOString(), hash }; log(`\u2705 ${name} deployed`); } catch (error) { log(`\u26A0 Failed to upload ${name}: ${error instanceof Error ? error.message : String(error)}`); } } } await fs2.mkdir(path.dirname(manifestPath), { recursive: true }); await fs2.writeFile(manifestPath, JSON.stringify(manifest, null, 2)); log(`Saved ${Object.keys(manifest).length} component URL(s)`); return manifest; } // src/utils/dependency-snapshot-refresh.ts async function refreshDependencySnapshotOnce(opts) { if (!opts.botId || !opts.client) { if (opts.required) { throw new exports_dependencies.DependencyError({ code: "AUTH_REQUIRED", message: "Could not refresh dependency snapshot from Cloud because project credentials are unavailable." }); } return false; } try { await new exports_dependencies.DependencySnapshotStore({ projectPath: opts.projectPath }).refreshFromCloud({ client: opts.client, botId: opts.botId, env: opts.env, integrationRegistry: new exports_dependencies.IntegrationRegistry, onWarning: (warning) => opts.logger?.warn(warning.message, { event: opts.event ?? "dependency-refresh", code: warning.code, path: warning.path }) }); AgentProject.clearCacheForPath(opts.projectPath); opts.logger?.info(`Dependency snapshot refreshed from Cloud (${opts.env})`, { event: opts.event ?? "dependency-refresh" }); return true; } catch (error) { if (opts.required) throw error; opts.logger?.warn(`Dependency snapshot refresh skipped: ${error instanceof Error ? error.message : String(error)}`, { event: opts.event ?? "dependency-refresh" }); return false; } } // ../evals/dist/loader.js import { readdirSync, existsSync } from "fs"; import { resolve } from "path"; var Eval = class { name; description; tags; type; setup; conversation; outcome; options; constructor(def) { this.name = def.name; this.conversation = def.conversation; if (def.description !== undefined) this.description = def.description; if (def.tags !== undefined) this.tags = def.tags; if (def.type !== undefined) this.type = def.type; if (def.setup !== undefined) this.setup = def.setup; if (def.outcome !== undefined) this.outcome = def.outcome; if (def.options !== undefined) this.options = def.options; } }; var AdkError = class extends Error { static __IS_ADK_BASE_ERROR = true; code; expected; details; suggestion; constructor(opts) { super(opts.message, opts.cause !== undefined ? { cause: opts.cause } : undefined); this.name = this.constructor.name; this.code = opts.code; this.expected = opts.expected ?? false; if (opts.details !== undefined) { this.details = opts.details; } if (opts.suggestion !== undefined) { this.suggestion = opts.suggestion; } } }; var EvalRunnerError = class extends AdkError { }; function isEvalDefinition(value) { return value !== null && typeof value === "object" && typeof value.name === "string" && value.name !== "" && Array.isArray(value.conversation); } async function loadEvalFile(filePath) { const absPath = resolve(filePath); let mod; try { mod = await import(absPath); } catch (err) { const msg = err instanceof Error ? err.message : String(err); throw new EvalRunnerError({ code: "EVAL_LOAD_FAILED", message: `Failed to load eval file ${filePath}: ${msg} Make sure your eval file: - Has no syntax or type errors - Exports one or more \`new Eval({...})\` instances - Has all dependencies installed (\`bun install\`)`, expected: true, cause: err }); } const results = []; for (const [key, value] of Object.entries(mod)) { if (key === "__esModule") continue; if (value instanceof Eval || isEvalDefinition(value)) { results.push(value); } } if (results.length === 0) { throw new EvalRunnerError({ code: "EVAL_FILE_EMPTY", message: `Invalid eval file ${filePath}: no valid evals found. Export one or more \`new Eval({...})\` instances (as default or named exports).`, expected: true }); } return results; } async function loadEvalsFromDir(dirPath) { const absDir = resolve(dirPath); if (!existsSync(absDir)) { return []; } const files = readdirSync(absDir).filter((f) => f.endsWith(".eval.ts")); const evals = []; for (const f of files) { const defs = await loadEvalFile(`${absDir}/${f}`); evals.push(...defs); } const seen = /* @__PURE__ */ new Set; for (const e of evals) { if (seen.has(e.name)) { throw new EvalRunnerError({ code: "EVAL_DUPLICATE_NAME", message: `Duplicate eval name "${e.name}" found in ${dirPath} \u2014 names must be unique across the evals directory.`, expected: true, details: { name: e.name } }); } seen.add(e.name); } return evals; } function filterEvals(evals, filter) { if (!filter) return evals; return evals.filter((e) => { if (filter.names && filter.names.length > 0) { if (!filter.names.includes(e.name)) return false; } if (filter.tags && filter.tags.length > 0) { if (!e.tags || !filter.tags.some((t) => e.tags.includes(t))) return false; } if (filter.type) { if (e.type !== filter.type) return false; } return true; }); } // src/utils/eval-manifest-uploader.ts import { existsSync as existsSync2, readdirSync as readdirSync2 } from "fs"; import { resolve as resolve2 } from "path"; async function computeEvalManifestPlan(agentRoot) { const evals = await loadExportedEvals(`${agentRoot}/evals`); return { evals, evalCount: evals.length }; } async function loadExportedEvals(dirPath) { const absDir = resolve2(dirPath); if (!existsSync2(absDir)) return []; const files = readdirSync2(absDir).filter((f) => f.endsWith(".eval.ts")); const evals = []; for (const file of files) { try { evals.push(...await loadEvalFile(`${absDir}/${file}`)); } catch (error) { if (isEmptyEvalFileError(error)) continue; throw error; } } assertUniqueEvalNames(evals, dirPath); return evals; } function isEmptyEvalFileError(error) { return error !== null && typeof error === "object" && error.code === "EVAL_FILE_EMPTY"; } function assertUniqueEvalNames(evals, dirPath) { const seen = new Set; for (const evalDef of evals) { if (seen.has(evalDef.name)) { throw new Error(`Duplicate eval name "${evalDef.name}" found in ${dirPath}`); } seen.add(evalDef.name); } } async function uploadEvalManifest(opts) { const evalDefs = opts.evals ?? await loadExportedEvals(`${opts.agentRoot}/evals`); let nextToken; const staleIds = []; do { const res = await opts.client.listFiles({ tags: EVAL_MANIFEST_TAGS, nextToken }); staleIds.push(...res.files.map((f) => f.id)); nextToken = res.meta.nextToken; } while (nextToken); if (staleIds.length > 0) { await Promise.allSettled(staleIds.map((id) => opts.client.deleteFile({ id }))); } if (evalDefs.length === 0) { return { uploaded: 0 }; } let chatWebhookId; try { const { bot } = await opts.client.getBot({ id: opts.botId }); const chat = Object.values(bot.integrations || {}).find((int) => int.name === "chat"); chatWebhookId = chat?.webhookId; } catch {} const manifest = { schemaVersion: EVAL_MANIFEST_SCHEMA_VERSION, evals: evalDefs, ...chatWebhookId ? { chatWebhookId } : {} }; await opts.client.uploadFile({ content: JSON.stringify(manifest), key: "eval-manifest.json", contentType: "application/json", tags: EVAL_MANIFEST_TAGS }); opts.log(`Uploaded eval manifest (${evalDefs.length} eval${evalDefs.length === 1 ? "" : "s"})`); return { uploaded: evalDefs.length }; } // src/utils/prod-deploy-pipeline.ts import path2 from "path"; // src/utils/prod-metadata-publisher.ts async function publishProdMetadata(input) { const manifest = createDeployedAgentManifest(input.project, {}); await uploadDeployedAgentManifest(input.client, manifest); await tagDeployedAgentManifestBot(input.client, input.botId); try { const agentMapSnapshot = createDeployedAgentMapSnapshot(input.project); await uploadDeployedAgentMapSnapshot(input.client, agentMapSnapshot); return {}; } catch (error) { return { agentMapSnapshotWarning: error instanceof Error ? error.message : String(error) }; } } // src/utils/prod-deploy-pipeline.ts async function runProdDeployPipeline(opts) { const failures = []; const tableFailures = []; const getClient = createClientResolver(opts); if (opts.applyPlanUpdates && opts.plan.preflight.result.hasChanges) { await runRequiredStage(opts.callbacks, "preflight", async () => { await opts.plan.preflight.apply({ onProgress: (message) => opts.callbacks?.onPreflightProgress?.(message), onSuccess: (message) => opts.callbacks?.onPreflightSuccess?.(message), onError: (message) => opts.callbacks?.onPreflightError?.(message) }); }); } await runRequiredStage(opts.callbacks, "deploy", async () => { const deployCommand = new BpDeployCommand({ botPath: path2.join(opts.project.path, ".adk", "bot"), botId: opts.botId, workspaceId: opts.workspaceId, credentials: opts.credentials, secrets: opts.secrets && Object.keys(opts.secrets).length > 0 ? opts.secrets : undefined }); opts.callbacks?.onDeployCommand?.(deployCommand); await deployCommand.run(); await deployCommand.output(); }); await runStage(opts.callbacks, "manifest", async () => { const client = await getClient(); const result = await publishProdMetadata({ project: opts.project, client, botId: opts.botId }); return { ...result.agentMapSnapshotWarning ? { agentMapSnapshotWarning: result.agentMapSnapshotWarning } : {} }; }, { nonFatal: true }); if (opts.applyPlanUpdates) { await syncKnowledgeBases(opts, failures); tableFailures.push(...await syncTables(opts, failures)); await syncAssets(opts, failures); } if (opts.evalManifestPlan?.evalCount !== 0) { const evalManifestResult = await runStage(opts.callbacks, "eval-manifest", async () => { const client = await getClient(); const result = await uploadEvalManifest({ agentRoot: opts.agentRoot, client, botId: opts.botId, evals: opts.evalManifestPlan?.evals, log: () => {} }); return { uploaded: result.uploaded }; }, {}); if (!evalManifestResult.success) failures.push("eval manifest"); } return { success: failures.length === 0, failures, ...tableFailures.length > 0 ? { tableFailures } : {} }; } function createClientResolver(opts) { let clientPromise; return async () => { if (opts.client) return opts.client; clientPromise ??= Promise.resolve().then(() => { if (!opts.getClient) { throw new Error("Deploy pipeline requires a client or client resolver"); } return opts.getClient(); }); return clientPromise; }; } async function syncKnowledgeBases(opts, failures) { const shouldDeleteOrphans = opts.plan.orphanedKBs.length > 0 && opts.confirmStorageChanges && Boolean(opts.plan.managers.kb); const shouldSyncKnowledgeBases = Boolean(opts.plan.kbPlan?.hasChanges && opts.plan.managers.kb); if (!shouldDeleteOrphans && !shouldSyncKnowledgeBases) return false; const result = await runStage(opts.callbacks, "kb-sync", async () => { const detail = {}; if (shouldDeleteOrphans) { for (const kb of opts.plan.orphanedKBs) { await opts.plan.managers.kb.deleteKnowledgeBase(kb.id, kb.name); } detail.deleted = opts.plan.orphanedKBs.length; } if (shouldSyncKnowledgeBases) { const result2 = await opts.plan.managers.kb.executeSync(opts.plan.kbPlan, { confirmDestructive: opts.confirmStorageChanges }); detail.synced = result2.synced.length; detail.skipped = result2.skipped.length; detail.failed = result2.failed.length; if (result2.failed.length > 0) { throw new StageFailure("Knowledge base sync failed", detail); } } return detail; }, {}); if (!result.success) failures.push("knowledge bases"); return !result.success; } async function syncTables(opts, failures) { if (!opts.plan.tablePlan?.hasChanges || !opts.plan.managers.table) return []; const tableFailures = []; const result = await runStage(opts.callbacks, "tables", async () => { const tableResult = await opts.plan.managers.table.executeSync(opts.plan.tablePlan, { confirmDestructive: opts.confirmStorageChanges }); tableFailures.push(...toTableSyncFailureDetails(tableResult.failed)); if (tableResult.summary.failed > 0) { throw new StageFailure("Table sync failed", { ...tableResult.summary, failures: tableFailures }); } return tableResult.summary; }, {}); if (!result.success) failures.push("tables"); return tableFailures; } async function syncAssets(opts, failures) { if (!opts.plan.assetPlan?.hasChanges || !opts.plan.managers.assets) return; const result = await runStage(opts.callbacks, "assets", async () => { const assetResult = await opts.plan.managers.assets.executeSync(opts.plan.assetPlan, { confirmDestructive: opts.confirmStorageChanges }); if (assetResult.summary.failed > 0) { throw new StageFailure("Asset sync failed", assetResult.summary); } return assetResult.summary; }, {}); if (!result.success) failures.push("assets"); } async function runStage(callbacks, stage, fn, options = {}) { callbacks?.onStageStart?.(stage); try { const detail = await fn(); callbacks?.onStageComplete?.(stage, detail || undefined); return { success: true, detail: detail || undefined }; } catch (error) { const detail = error instanceof StageFailure ? error.detail : undefined; callbacks?.onStageError?.(stage, error, { nonFatal: options.nonFatal, detail }); if (!options.nonFatal) { return { success: false, detail }; } return { success: true, detail }; } } async function runRequiredStage(callbacks, stage, fn) { callbacks?.onStageStart?.(stage); try { const detail = await fn(); callbacks?.onStageComplete?.(stage, detail || undefined); return { detail: detail || undefined }; } catch (error) { callbacks?.onStageError?.(stage, error); if (error instanceof Error) { error.stage = stage; } throw error; } } class StageFailure extends Error { detail; constructor(message, detail) { super(message); this.detail = detail; this.name = "StageFailure"; } } export { style, globalReactPlugin, buildAndUploadComponents, refreshDependencySnapshotOnce, loadEvalsFromDir, filterEvals, computeEvalManifestPlan, runProdDeployPipeline };