everything-dev
Version:
A consolidated product package for building Module Federation apps with oRPC APIs.
697 lines (695 loc) • 23.9 kB
JavaScript
import { mergeAuthoredOverPublished } from "../resolution/merge.mjs";
import { fetchBosConfigFromFastKv, fetchConfigHistory, fetchRegistryKey } from "../registry/fastkv.mjs";
import { findConfigPath, isAppDescriptorPath } from "../resolution/load.mjs";
import { resolveUiRuntimeName } from "../resolution/runtime.mjs";
import { run } from "../../utils/run.mjs";
import { colors, icons } from "../../utils/theme.mjs";
import { openResolution } from "../resolution/session.mjs";
import { pointerFingerprint, slotPins } from "../registry/fingerprint.mjs";
import { applyDeployResults } from "./integrity.mjs";
import { padRight } from "../../utils/string.mjs";
import { generateCodeArtifacts } from "../codegen/artifacts.mjs";
import { BuildTrainService, buildWorkspaceTargets, resolveWorkspaceTarget } from "../workspace/build-train.mjs";
import { composePlatformManifestPublishEntry, platformTruthViolations } from "./platform-manifest.mjs";
import { verifyRollbackSnapshot } from "./rollback.mjs";
import { ensureCustodyBalance, submitRegistryWriteCustody } from "../auth/custody-signer.mjs";
import { ensureDelegateKey, submitRegistryWriteDelegated } from "../auth/delegate-signer.mjs";
import { submitRegistryWrite } from "../auth/near-signer.mjs";
import { SignerService, dispatchSubmit } from "../auth/signer.mjs";
import { BundleStorage, RegistryStore } from "./deps.mjs";
import { pinsFromDeployState, pinsFromPublishedConfig, upsertDeployStatePins } from "./diff.mjs";
import { PublishTrainError } from "./errors.mjs";
import { formatNearError } from "./format.mjs";
import { platformUrlDeployEntries, pluginUiUrlDeployEntries } from "./platform.mjs";
import { preflightPublish } from "./preflight.mjs";
import { DeployTrainRenderService } from "./render.mjs";
import { realStorageLegDeps, sourceLegDirs, uploadWorkspaceBundle } from "./upload-leg.mjs";
import { existsSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { Context, Effect, Layer, Option } from "effect";
import process from "node:process";
import { hasFolderFormUi } from "every-plugin/build/ui";
//#region src/services/deploy/train.ts
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/** Best-effort repo HEAD for the version manifest (`gitSha`): non-git
* contexts (child bundles built outside a repo) omit the field silently —
* it never fails the deploy. */
async function resolveGitSha(cwd) {
try {
const result = await run("git", ["rev-parse", "HEAD"], {
cwd,
capture: true
});
const sha = result?.stdout.trim();
return result && result.exitCode === 0 && sha ? sha : void 0;
} catch {
return;
}
}
async function waitForPublishedConfig(opts) {
const fetchConfig = opts.fetchConfig ?? fetchBosConfigFromFastKv;
const envTimeoutMs = Number(process.env.BOS_PUBLISH_CONFIRMATION_TIMEOUT_MS);
const envIntervalMs = Number(process.env.BOS_PUBLISH_CONFIRMATION_INTERVAL_MS);
const timeoutMs = opts.timeoutMs ?? (Number.isFinite(envTimeoutMs) ? envTimeoutMs : void 0) ?? 12e4;
const intervalMs = opts.intervalMs ?? (Number.isFinite(envIntervalMs) ? envIntervalMs : void 0) ?? 3e3;
const startedAt = Date.now();
let lastError;
let txProbeSkips = 0;
const TX_PROBE_SKIP_LIMIT = 2;
while (Date.now() - startedAt < timeoutMs) {
try {
let trackByTx = opts.txHash !== void 0 && opts.fetchHistory !== void 0;
if (trackByTx && txProbeSkips >= TX_PROBE_SKIP_LIMIT) trackByTx = false;
if (trackByTx) try {
if ((await opts.fetchHistory({
accountId: opts.account,
gatewayId: opts.gateway,
registry: opts.registry,
limit: 5
})).some((entry) => entry.txHash === opts.txHash)) txProbeSkips = 0;
else {
txProbeSkips += 1;
await sleep(intervalMs);
continue;
}
} catch (historyError) {
lastError = historyError;
}
const verifiedConfig = await fetchConfig(`bos://${opts.account}/${opts.gateway}`, opts.registry);
if (JSON.stringify(verifiedConfig) === JSON.stringify(opts.publishConfig)) return;
} catch (error) {
lastError = error;
}
await sleep(intervalMs);
}
const reason = lastError instanceof Error ? ` Last error: ${lastError.message}` : "";
throw new Error(`Timed out waiting for publish confirmation at bos://${opts.account}/${opts.gateway}.${reason}`);
}
async function isConfigAlreadyPublished(opts) {
const fetchConfig = opts.fetchConfig ?? fetchBosConfigFromFastKv;
try {
const current = await fetchConfig(`bos://${opts.account}/${opts.gateway}`, opts.registry);
return JSON.stringify(current) === JSON.stringify(opts.publishConfig);
} catch {
return false;
}
}
/**
* The deploy-train publish core (ADR 0020, as amended): preflight
* (storage/CDN creds + signing) fails fast before the build, then build →
* upload → version-manifest pin → config write-back → FastKV publish +
* read-back. Scoped to one plugin via `packages`, a single-plugin redeploy
* ships real bytes to the storage origin — same as the full train, nothing
* image-native left.
*
* Terminal failures throw `PublishTrainError` (the typed channel, ticket
* #422); callers behind the service map it to wire shapes mechanically.
*/
async function runPublishTrain(input, deps) {
const preflight = await preflightPublish(input);
if (preflight.kind === "error") throw new PublishTrainError({
kind: "preflight",
registryUrl: preflight.registryUrl,
error: preflight.error
});
if (preflight.kind === "dry-run") return {
status: "dry-run",
registryUrl: preflight.registryUrl
};
const plan = preflight.plan;
const { isStaging, account, gateway, network, registryUrl, registryNamespace, targets, useWallet, useCustody, strategy } = plan;
const storageOrigin = plan.storageOrigin;
const storageApiKey = plan.storageApiKey;
const { configDir } = input;
const render = deps.render;
let bosConfig = input.bosConfig;
const runtimeConfig = input.runtimeConfig;
let built;
let skipped;
let deployResults;
const urlOrigin = plan.cdnOrigin ?? `https://${gateway}`;
const platformEntries = [];
const fullUpload = input.fullUpload === true || process.env.BOS_FULL_UPLOAD === "1";
const sourcePublish = bosConfig.publish?.sourcePublish === true;
const gitSha = await resolveGitSha(configDir);
const memoryBackendAbort = () => new PublishTrainError({
kind: "memory-backend",
registryUrl,
built,
skipped,
deployResults,
error: "The receiving instance is serving bundle storage from memory (BOS_STORAGE_* R2 credentials are not configured there) — uploaded bytes would be lost on restart. Aborting before publish."
});
const prevPins = fullUpload ? {} : {
...pinsFromPublishedConfig(plan.publishedConfig ?? {}, account, gateway),
...pinsFromDeployState(configDir, account, gateway)
};
const authoredPath = findConfigPath(configDir);
const tsForm = authoredPath !== null && isAppDescriptorPath(authoredPath);
const openedRaw = await openResolution({ cwd: configDir });
if (!openedRaw?.rawConfig) throw new PublishTrainError({
kind: "reload",
registryUrl,
built,
skipped,
deployResults,
error: "Failed to load the authored config for publish"
});
const authoredRawConfig = openedRaw.rawConfig;
let publishRawConfig = authoredRawConfig;
render?.cdnBanner?.(storageOrigin);
const uploadOneWorkspace = async (key, rawConfig) => {
const startedAt = Date.now();
const ws = resolveWorkspaceTarget(key, bosConfig, runtimeConfig, configDir);
if (!ws) return {
entries: [],
detail: {
base: "",
uploaded: 0,
skipped: 0,
durationMs: 0,
notes: []
},
shared: {}
};
const outcome = await uploadWorkspaceBundle({
account,
gateway,
workspaceKey: key,
distDir: join(ws.path, "dist"),
storageOrigin,
storageApiKey,
urlOrigin,
previousPin: prevPins[key],
...sourcePublish ? { source: sourceLegDirs(ws.path) } : {},
...gitSha ? { gitSha } : {},
deps: deps.storage
});
if (outcome.kind === "error") throw new PublishTrainError({
kind: "upload",
registryUrl,
built,
skipped,
deployResults,
error: outcome.message
});
if (outcome.kind === "memory") throw memoryBackendAbort();
if (outcome.kind === "missing-manifest") throw new PublishTrainError({
kind: "missing-manifest",
registryUrl,
built,
skipped,
deployResults,
error: `bundle upload for ${key}: version manifest missing — the dist predates hashed entry names (rebuild required; atomic-deploys hard break)`
});
if (sourcePublish && outcome.storage === "memory") throw memoryBackendAbort();
const entries = platformUrlDeployEntries({
origin: urlOrigin,
account,
gateway,
key,
kind: ws.kind,
integrity: outcome.entryIntegrity,
ssrIntegrity: outcome.ssrEntryIntegrity,
pin: outcome.pin
});
let uiOutcomeShared;
if (hasFolderFormUi(ws.path)) {
const uiDistDir = join(ws.path, "ui", "dist");
const rawUi = ((ws.kind === "app" ? "app" : "plugins") === "app" ? rawConfig.app : rawConfig.plugins)?.[key]?.ui;
const uiName = resolveUiRuntimeName(rawUi, join(ws.path, "ui"), key);
let uiIntegrity;
let uiSsrIntegrity;
let uiPin;
if (existsSync(uiDistDir)) {
const uiOutcome = await uploadWorkspaceBundle({
account,
gateway,
workspaceKey: `${key}-ui`,
distDir: uiDistDir,
storageOrigin,
storageApiKey,
urlOrigin,
previousPin: prevPins[`${key}-ui`],
deps: deps.storage
});
if (uiOutcome.kind === "error") throw new PublishTrainError({
kind: "upload",
registryUrl,
built,
skipped,
deployResults,
error: uiOutcome.message
});
if (uiOutcome.kind === "memory") throw memoryBackendAbort();
if (uiOutcome.kind === "missing-manifest") throw new PublishTrainError({
kind: "missing-manifest",
registryUrl,
built,
skipped,
deployResults,
error: `bundle upload for ${key}-ui: version manifest missing — the dist predates hashed entry names (rebuild required; atomic-deploys hard break)`
});
uiIntegrity = uiOutcome.entryIntegrity;
uiSsrIntegrity = uiOutcome.ssrEntryIntegrity;
uiPin = uiOutcome.pin;
uiOutcomeShared = uiOutcome.shared;
} else console.log(` ${colors.green(icons.ok)} ${padRight(`${key}-ui`, 28)} → ${urlOrigin}/bundles/${account}/${gateway}/${key}-ui/`);
entries.push(...pluginUiUrlDeployEntries({
origin: urlOrigin,
account,
gateway,
key,
kind: ws.kind,
integrity: uiIntegrity,
ssrIntegrity: uiSsrIntegrity,
name: uiName,
pin: uiPin
}));
}
return {
entries,
detail: {
base: outcome.detail.base,
uploaded: outcome.uploaded,
skipped: outcome.skipped,
durationMs: Date.now() - startedAt,
notes: outcome.detail.notes
},
shared: {
...outcome.shared ? { [key]: outcome.shared } : {},
...uiOutcomeShared ? { [`${key}-ui`]: uiOutcomeShared } : {}
}
};
};
const uploadConcurrency = Math.max(1, Number(process.env.BOS_UPLOAD_CONCURRENCY) || 4);
const uploaded = /* @__PURE__ */ new Map();
const trackUpload = (key, pending) => {
uploaded.set(key, pending);
pending.catch(() => {});
};
let inFlight = 0;
const waiters = [];
const acquireUploadSlot = () => {
if (inFlight < uploadConcurrency) {
inFlight++;
return Promise.resolve();
}
return new Promise((resolve) => waiters.push(resolve));
};
const releaseUploadSlot = () => {
inFlight--;
waiters.shift()?.();
};
const startUpload = (key) => {
if (uploaded.has(key)) return;
trackUpload(key, (async () => {
await acquireUploadSlot();
try {
return await uploadOneWorkspace(key, authoredRawConfig);
} finally {
releaseUploadSlot();
}
})());
};
if (input.build) {
await generateCodeArtifacts(configDir, bosConfig, {
env: "production",
runtimeConfig: runtimeConfig ?? void 0
});
const result = await deps.buildTrain.buildTargets({
configDir,
bosConfig,
runtimeConfig,
targets,
deploy: true,
verbose: input.verbose,
...input.forceBuild ? { forceBuild: true } : {}
}, { onWorkspaceBuilt: (key) => {
if (targets.includes(key)) startUpload(key);
} });
built = result.built;
skipped = result.skipped;
deployResults = result.deployResults;
if (deployResults) {
const failures = deployResults.filter((r) => !r.success);
if (failures.length > 0) {
const total = deployResults.length;
console.log();
console.log(colors.error(` ${icons.err} Deploy failed — ${failures.length} of ${total} workspace${total > 1 ? "s" : ""} failed`));
console.log();
for (const f of failures) {
const errorLine = (f.error ?? "Failed").split("\n")[0];
console.log(` ${colors.error(icons.err)} ${padRight(f.key, 28)} ${errorLine}`);
}
console.log();
if (!input.verbose) {
console.log(colors.dim(" Run with --verbose for full build output."));
console.log();
}
throw new PublishTrainError({
kind: "build",
registryUrl,
built,
skipped,
deployResults,
error: `${failures.length} of ${total} workspaces failed to deploy`
});
}
}
const refreshed = await openResolution({ cwd: configDir });
if (!refreshed?.config) throw new PublishTrainError({
kind: "reload",
registryUrl,
built,
skipped,
deployResults,
error: "Failed to reload the config after build"
});
bosConfig = refreshed.config;
if (refreshed.rawConfig) publishRawConfig = refreshed.rawConfig;
}
if (tsForm && !input.build && input.exactPayload !== true && plan.publishedConfigState === "unreachable") throw new PublishTrainError({
kind: "config-state",
registryUrl,
built,
skipped,
deployResults,
error: "Cannot read the currently published config from the registry — a config-only publish would overwrite live bundle URLs. Retry when the registry is reachable, or run a full deploy."
});
let publishPayload = input.exactPayload === true ? input.bosConfig : tsForm && !input.build ? mergeAuthoredOverPublished(plan.publishedConfig ?? {}, publishRawConfig) : publishRawConfig;
for (const key of built ?? []) if (targets.includes(key)) startUpload(key);
const deployTargets = (built ?? []).filter((key) => targets.includes(key));
const settled = await Promise.allSettled(deployTargets.map((key) => uploaded.get(key) ?? Promise.resolve({
entries: [],
detail: {
base: "",
uploaded: 0,
skipped: 0,
durationMs: 0,
notes: []
},
shared: {}
})));
const sharedByWorkspace = /* @__PURE__ */ new Map();
const uploadFailures = settled.map((result, index) => result.status === "rejected" ? {
key: deployTargets[index],
error: result.reason
} : null).filter((entry) => entry !== null);
for (let index = 0; index < deployTargets.length; index++) {
const result = settled[index];
const key = deployTargets[index];
if (result.status === "fulfilled") {
platformEntries.push(...result.value.entries);
for (const [bundleKey, shared] of Object.entries(result.value.shared)) sharedByWorkspace.set(bundleKey, shared);
render?.uploadOutcome?.({
key,
ok: true,
...input.verbose ? { detailNotes: result.value.detail.notes } : {},
...result.value.detail
});
} else render?.uploadOutcome?.({
key,
ok: false
});
}
if (uploadFailures.length > 0) {
const thrown = uploadFailures[0].error;
if (thrown instanceof PublishTrainError) throw thrown;
throw new PublishTrainError({
kind: "upload",
registryUrl,
built,
skipped,
deployResults,
error: thrown instanceof Error ? thrown.message : String(thrown)
});
}
if (platformEntries.length > 0) {
const merged = applyDeployResults(publishRawConfig, platformEntries);
if (!tsForm) try {
writeFileSync(join(configDir, "bos.config.json"), `${JSON.stringify(merged, null, 2)}\n`);
} catch (error) {
throw new PublishTrainError({
kind: "config-write",
registryUrl,
built,
skipped,
deployResults,
error: `Failed to write bundle URLs to bos.config.json: ${error instanceof Error ? error.message : String(error)}`
});
}
publishPayload = merged;
}
if (isStaging) publishPayload = {
...publishPayload,
domain: gateway
};
const pinVerification = await verifyRollbackSnapshot(publishPayload);
if (pinVerification.verifiable && !pinVerification.ok) {
const failures = pinVerification.slots.filter((check) => !check.ok).map((check) => ` ${check.slot}: ${check.reason}`).join("\n");
console.error(` ${colors.error(icons.err)} Publish blocked — a pinned slot's bytes do not serve at its production URL:\n${failures}`);
throw new PublishTrainError({
kind: "pin-verification",
registryUrl,
built,
skipped,
deployResults,
error: `Publish blocked — pinned slot bytes missing or SRI mismatched:\n${failures}`
});
}
const registryKey = `apps/${account}/${gateway}/bos.config.json`;
const publishedAt = (/* @__PURE__ */ new Date()).toISOString();
const fingerprint = pointerFingerprint(publishPayload);
const publishSlotPins = slotPins(publishPayload);
const registryEntries = { [registryKey]: JSON.stringify(publishPayload) };
const manifestKey = `apps/${account}/${gateway}/manifests/${publishedAt.replace(/[:.]/g, "-")}.json`;
let platformManifest = null;
if (!isStaging && input.build) {
try {
const previous = await fetchRegistryKey(account, gateway, "platform.json");
platformManifest = composePlatformManifestPublishEntry({
configDir,
account,
gateway,
previousFrameworkVersion: previous?.frameworkVersion,
gitSha
});
} catch (error) {
console.warn(` platform manifest skipped this deploy — the previous release could not be read: ${error instanceof Error ? error.message : String(error)}`);
}
if (platformManifest) {
const violations = platformTruthViolations(platformManifest.manifest, sharedByWorkspace);
if (violations.length > 0) throw new PublishTrainError({
kind: "platform-truth",
registryUrl,
built,
skipped,
deployResults,
error: `platform manifest does not match the uploaded bundles:\n` + violations.map((line) => ` ${line}`).join("\n")
});
}
}
registryEntries[manifestKey] = JSON.stringify({
account,
gateway,
network,
publishedAt,
registryUrl,
...platformManifest ? { platformManifest: {
file: platformManifest.key,
integrity: platformManifest.integrity
} } : {}
});
if (platformManifest) registryEntries[platformManifest.key] = platformManifest.value;
render?.publish?.({
registryUrl,
fingerprint,
slotPins: Object.entries(publishSlotPins),
auditManifestFile: manifestKey.split("/").pop() ?? manifestKey
});
const persistDeployState = () => {
try {
const pins = pinsFromPublishedConfig(publishPayload, account, gateway);
if (Object.keys(pins).length > 0) upsertDeployStatePins(configDir, account, gateway, pins);
} catch (error) {
console.log(colors.dim(` Note: could not update .bos/deploy-state.json — ${error instanceof Error ? error.message : String(error)}`));
}
};
try {
if (await isConfigAlreadyPublished({
account,
gateway,
publishConfig: publishPayload,
registry: input.registry,
fetchConfig: deps.registry.fetchConfig
})) {
console.log(" Already up to date — skipping transaction");
persistDeployState();
return {
status: "published",
registryUrl,
built,
skipped,
deployResults,
publishConfig: publishPayload
};
}
console.log(` Submitting transaction on ${network}...`);
const result = await (async () => {
if (useCustody) {
if (!plan.custodyWallet) throw new PublishTrainError({
kind: "submit",
registryUrl,
error: "custody publish requires the preflight custody wallet"
});
return deps.signer.submit({
mode: "custody",
account,
contract: registryNamespace,
args: registryEntries,
custodyWallet: plan.custodyWallet,
idempotencyKey: `bos-publish-${fingerprint}`
});
}
if (useWallet) {
const credential = plan.credential;
if (!credential || credential.accountId !== account) throw new PublishTrainError({
kind: "submit",
registryUrl,
error: `--wallet requires a CLI session under ${account}. Run bos login first.`
});
return deps.signer.submit({
mode: "wallet",
account,
contract: registryNamespace,
network,
args: registryEntries,
credential,
configDir
});
}
if (!strategy) throw new PublishTrainError({
kind: "submit",
registryUrl,
error: "non-wallet publish requires a resolved signing strategy"
});
return deps.signer.submit({
mode: "key",
account,
contract: registryNamespace,
network,
args: registryEntries,
privateKey: input.privateKey,
strategy
});
})();
if (result.txHash) console.log(` Transaction submitted: ${colors.dim(result.txHash)}`);
console.log(" Waiting for publish confirmation...");
try {
await waitForPublishedConfig({
account,
gateway,
publishConfig: publishPayload,
registry: input.registry,
fetchConfig: deps.registry.fetchConfig,
...result.txHash ? {
txHash: result.txHash,
fetchHistory: deps.registry.fetchHistory
} : {}
});
} catch (error) {
throw new PublishTrainError({
kind: "confirm",
registryUrl,
error: formatNearError(error),
built,
skipped,
deployResults
});
}
persistDeployState();
return {
status: "published",
registryUrl,
txHash: result.txHash,
fingerprint,
slotPins: publishSlotPins,
built,
skipped,
deployResults,
publishConfig: publishPayload
};
} catch (error) {
if (error instanceof PublishTrainError) throw error;
throw new PublishTrainError({
kind: "sign",
registryUrl,
error: formatNearError(error),
built,
skipped,
deployResults
});
}
}
/** Procedural entry point: the train behind a plain promise — legacy callers
* and tests see the string-error result shape, the Effect service maps the
* typed channel. */
async function publishToFastKv(input) {
try {
return await runPublishTrain(input, realTrainDeps);
} catch (error) {
if (error instanceof PublishTrainError) return error.toResult();
throw error;
}
}
const defaultBuildTrain = { buildTargets: (input, deps) => buildWorkspaceTargets(input, deps) };
const realTrainDeps = {
buildTrain: defaultBuildTrain,
storage: realStorageLegDeps,
registry: {
fetchConfig: fetchBosConfigFromFastKv,
fetchHistory: fetchConfigHistory
},
signer: {
submitRegistryWrite,
submitRegistryWriteDelegated,
submitRegistryWriteCustody,
ensureDelegateKey,
ensureCustodyBalance,
submit: dispatchSubmit
}
};
/**
* The train as a Context.Service (ticket #422): `publish` fails with the
* typed `PublishTrainError`; handlers yield the service and map failures
* mechanically.
*/
var DeployTrainService = class DeployTrainService extends Context.Service()("everything-dev/deploy/train") {
static Live = Layer.effect(this, Effect.gen(function* () {
const storage = yield* BundleStorage;
const registry = yield* RegistryStore;
const signer = yield* SignerService;
const buildTrainOption = yield* Effect.serviceOption(BuildTrainService);
const renderOption = yield* Effect.serviceOption(DeployTrainRenderService);
return DeployTrainService.of({ publish: (input) => Effect.tryPromise({
try: () => runPublishTrain(input, {
storage,
registry,
signer,
buildTrain: Option.getOrUndefined(buildTrainOption) ?? defaultBuildTrain,
render: Option.getOrUndefined(renderOption)
}),
catch: (error) => error instanceof PublishTrainError ? error : new PublishTrainError({
kind: "submit",
registryUrl: "",
error: String(error)
})
}) });
}));
};
//#endregion
export { DeployTrainService, isConfigAlreadyPublished, publishToFastKv, realTrainDeps, runPublishTrain, waitForPublishedConfig };
//# sourceMappingURL=train.mjs.map