UNPKG

everything-dev

Version:

A consolidated product package for building Module Federation apps with oRPC APIs.

github.com/NEARBuilders/everything-dev

NEARBuilders/everything-dev

1 lines • 18.6 kB
1
{"version":3,"file":"upload-leg.mjs","names":[],"sources":["../../../src/services/deploy/upload-leg.ts"],"sourcesContent":["import { readFileSync, rmSync } from \"node:fs\";\nimport { basename, dirname, join } from \"node:path\";\nimport type { SourceRef } from \"every-plugin/version-manifest\";\nimport { diffWorkspaceFiles, localSriMap, mergedFileIntegrity, type PreviousPin } from \"./diff\";\nimport { buildSourceArchive, makeSourceArchiveTempDir } from \"./source-archive\";\nimport {\n type BundleUploadResult,\n collectDistFiles,\n type DistFile,\n uploadBundle,\n uploadWorkspaceDist,\n} from \"./storage\";\nimport {\n composeWorkspaceVersionManifest,\n readBuildReport,\n resolveSlotVersion,\n} from \"./version-manifest\";\n\n/**\n * The ONE workspace upload leg (ticket #422): dist collection → diff upload\n * → version-manifest pin → memory-backend guard. Every workspace bundle in\n * the train — the workspace itself and a folder-form plugin's `-ui` sub-bundle —\n * goes through this single implementation; the previous near line-for-line ui\n * copy is gone.\n */\n\nexport interface VersionManifestPin {\n file: string;\n integrity: string;\n storage?: \"s3\" | \"memory\";\n}\n\n/** The storage surface the upload leg needs — injectable for tests, provided\n * by the real modules in production (BundleStorage service in T3). */\nexport interface StorageLegDeps {\n collectDistFiles: typeof collectDistFiles;\n uploadWorkspaceDist: typeof uploadWorkspaceDist;\n uploadBundle: typeof uploadBundle;\n resolveSlotVersion: typeof resolveSlotVersion;\n readBuildReport: typeof readBuildReport;\n composeWorkspaceVersionManifest: typeof composeWorkspaceVersionManifest;\n buildSourceArchive: typeof buildSourceArchive;\n}\n\nexport const realStorageLegDeps: StorageLegDeps = {\n collectDistFiles,\n uploadWorkspaceDist,\n uploadBundle,\n resolveSlotVersion,\n readBuildReport,\n composeWorkspaceVersionManifest,\n buildSourceArchive,\n};\n\nfunction formatBundleMb(files: Array<{ bytes: Uint8Array }>): string {\n const total = files.reduce((sum, file) => sum + file.bytes.byteLength, 0);\n return `${(total / 1024 / 1024).toFixed(1)} MB`;\n}\n\ninterface DiffUploadOutcome {\n result: BundleUploadResult;\n /** per-file SRI map covering the whole dist (uploaded → server SRI, skipped → previous manifest SRI) */\n fileIntegrity: Record<string, string>;\n uploaded: number;\n skipped: number;\n total: number;\n}\n\n/** One workspace's upload with skip-unchanged diffing; any failure to resolve\n * the previous manifest's file map degrades to a full upload. */\nasync function diffUploadWorkspace(\n deps: StorageLegDeps,\n input: {\n origin: string;\n apiKey?: string;\n account: string;\n gateway: string;\n workspace: string;\n files: DistFile[];\n base: string;\n previousPin: PreviousPin | undefined;\n notes: string[];\n },\n): Promise<DiffUploadOutcome> {\n const localSri = localSriMap(input.files);\n let uploadFiles = input.files;\n let prevFiles: Record<string, string> | undefined;\n if (input.previousPin) {\n try {\n const resolved = await deps.resolveSlotVersion({ base: input.base, pin: input.previousPin });\n if (resolved.files) {\n const diff = diffWorkspaceFiles(input.files, localSri, resolved.files);\n prevFiles = resolved.files;\n uploadFiles = diff.upload;\n if (diff.skipped > 0) {\n input.notes.push(`${diff.skipped} unchanged file(s) — skipping (manifest diff)`);\n }\n }\n } catch {\n input.notes.push(\"previous manifest unusable — uploading everything\");\n }\n }\n\n input.notes.push(\n `uploading ${uploadFiles.length} of ${input.files.length} files (${formatBundleMb(uploadFiles)}) → ${input.base}`,\n );\n const result =\n uploadFiles.length > 0\n ? await deps.uploadWorkspaceDist({\n origin: input.origin,\n apiKey: input.apiKey,\n account: input.account,\n gateway: input.gateway,\n workspace: input.workspace,\n files: uploadFiles,\n })\n : { stored: 0, totalBytes: 0, integrity: {} };\n return {\n result,\n fileIntegrity: mergedFileIntegrity(input.files, result.integrity, prevFiles),\n uploaded: result.stored,\n skipped: input.files.length - uploadFiles.length,\n total: input.files.length,\n };\n}\n\n/** Compose + upload the workspace's version manifest — server-computed SRI of\n * the just-finished upload, uploaded additively at `versions/<version>.json`.\n * Returns undefined when the dist predates hashed entry names. The manifest's\n * `shared` record rides back for the publish train's truth gate (#546). */\nasync function pinWorkspaceVersionManifest(\n deps: StorageLegDeps,\n input: {\n origin: string;\n apiKey?: string;\n account: string;\n gateway: string;\n workspace: string;\n report: ReturnType<typeof readBuildReport>;\n ssrReport: ReturnType<typeof readBuildReport>;\n integrityMap: Record<string, string>;\n files?: Record<string, string>;\n source?: SourceRef;\n gitSha?: string;\n notes: string[];\n },\n): Promise<{ pin: VersionManifestPin; shared?: Record<string, string> } | undefined> {\n if (!input.report) return undefined;\n const versionManifest = deps.composeWorkspaceVersionManifest({\n report: input.report,\n ssrReport: input.ssrReport,\n integrityMap: input.integrityMap,\n ...(input.files ? { files: input.files } : {}),\n ...(input.source ? { source: input.source } : {}),\n ...(input.gitSha ? { gitSha: input.gitSha } : {}),\n });\n if (!versionManifest) return undefined;\n\n const manifestFile = `versions/${versionManifest.version}.json`;\n const manifestUpload = await deps.uploadBundle({\n origin: input.origin,\n apiKey: input.apiKey,\n account: input.account,\n gateway: input.gateway,\n workspace: input.workspace,\n files: [\n {\n path: manifestFile,\n bytes: new TextEncoder().encode(`${JSON.stringify(versionManifest, null, 2)}\\n`),\n },\n ],\n });\n const manifestIntegrity = manifestUpload.integrity[manifestFile];\n if (!manifestIntegrity) return undefined;\n input.notes.push(`version manifest ${manifestFile} (SRI pinned)`);\n return {\n pin: { file: manifestFile, integrity: manifestIntegrity, storage: manifestUpload.storage },\n ...(versionManifest.shared ? { shared: versionManifest.shared } : {}),\n };\n}\n\n/** Source-archive leg (publish.sourcePublish, #406): tar the workspace\n * subtree, dedupe against the previous pin by archive SRI, otherwise upload\n * and take the server-computed SRI. The temp dir is cleaned on every path. */\nasync function resolveSourceReference(\n deps: StorageLegDeps,\n input: {\n origin: string;\n apiKey?: string;\n account: string;\n gateway: string;\n workspace: string;\n base: string;\n previousPin: PreviousPin | undefined;\n repoRoot: string;\n workspaceDir: string;\n notes: string[];\n },\n): Promise<{ source: SourceRef } | { memory: true }> {\n const notes = input.notes;\n const outDir = makeSourceArchiveTempDir();\n try {\n const archive = await deps.buildSourceArchive({\n repoRoot: input.repoRoot,\n workspace: input.workspaceDir,\n outDir,\n });\n if (input.previousPin) {\n try {\n const resolved = await deps.resolveSlotVersion({\n base: input.base,\n pin: input.previousPin,\n });\n if (resolved.source?.integrity === archive.integrity) {\n notes.push(`source unchanged — reusing ${archive.file}`);\n return {\n source: { file: resolved.source.file, integrity: resolved.source.integrity },\n };\n }\n } catch {\n // previous manifest unusable — fall through to a full upload\n }\n }\n const upload = await deps.uploadBundle({\n origin: input.origin,\n apiKey: input.apiKey,\n account: input.account,\n gateway: input.gateway,\n workspace: input.workspace,\n files: [{ path: archive.file, bytes: new Uint8Array(readFileSync(archive.path)) }],\n });\n if (upload.storage === \"memory\") return { memory: true };\n const integrity = upload.integrity[archive.file];\n if (!integrity) {\n throw new Error(\n `[publish] source archive upload for ${input.workspace}: missing server SRI for ${archive.file}`,\n );\n }\n notes.push(`source archive ${archive.file}`);\n return { source: { file: archive.file, integrity } };\n } finally {\n rmSync(outDir, { recursive: true, force: true });\n }\n}\n\nexport interface UploadWorkspaceBundleInput {\n account: string;\n gateway: string;\n /** bundle key: the workspace key itself, or `<key>-ui` for the folder-form ui sub-bundle */\n workspaceKey: string;\n distDir: string;\n storageOrigin: string;\n storageApiKey?: string;\n /** the public CDN origin the bundle URL is printed + pinned against */\n urlOrigin: string;\n previousPin: PreviousPin | undefined;\n /** enable the source-archive leg (publish.sourcePublish) — main workspaces only */\n source?: { repoRoot: string; workspaceDir: string };\n gitSha?: string;\n deps?: StorageLegDeps;\n}\n\nexport type WorkspaceBundleUpload =\n | {\n kind: \"ok\";\n pin: VersionManifestPin;\n /** the serving backend of the diff upload — memory aborts the train */\n storage?: \"s3\" | \"memory\";\n /** entry SRI keyed by the build report's hashed name */\n entryIntegrity?: string;\n ssrEntryIntegrity?: string;\n /** Shared-singleton `name → version` the pinned build embedded (#546) —\n * the publish train's truth gate cross-checks it against the platform\n * release manifest before FastKV. */\n shared?: Record<string, string>;\n uploaded: number;\n skipped: number;\n durationMs: number;\n /** attributed upload detail (#431) — the renderer decides how it shows;\n * the leg itself stays silent */\n detail: {\n base: string;\n manifestFile: string | undefined;\n notes: string[];\n };\n }\n | { kind: \"memory\" }\n /** the dist predates hashed entry names — the train aborts (atomic-deploys hard break) */\n | { kind: \"missing-manifest\" }\n | { kind: \"error\"; message: string };\n\nexport async function uploadWorkspaceBundle(\n input: UploadWorkspaceBundleInput,\n): Promise<WorkspaceBundleUpload> {\n const deps = input.deps ?? realStorageLegDeps;\n const base = `${input.urlOrigin}/bundles/${input.account}/${input.gateway}/${input.workspaceKey}/`;\n const startedAt = Date.now();\n let phase: \"upload\" | \"source\" = \"upload\";\n try {\n const distFiles = await deps.collectDistFiles(input.distDir);\n const notes: string[] = [];\n const outcome = await diffUploadWorkspace(deps, {\n origin: input.storageOrigin,\n apiKey: input.storageApiKey,\n account: input.account,\n gateway: input.gateway,\n workspace: input.workspaceKey,\n files: distFiles,\n base,\n previousPin: input.previousPin,\n notes,\n });\n const report = deps.readBuildReport(input.distDir);\n const ssrReport = deps.readBuildReport(join(input.distDir, \"ssr\"));\n const entryIntegrity = report ? outcome.fileIntegrity[report.entry] : undefined;\n const ssrEntryIntegrity = ssrReport?.entry\n ? outcome.fileIntegrity[`ssr/${ssrReport.entry}`]\n : undefined;\n\n let sourceRef: SourceRef | undefined;\n if (input.source) {\n if (outcome.result.storage === \"memory\") {\n return { kind: \"memory\" };\n }\n phase = \"source\";\n const sourceOutcome = await resolveSourceReference(deps, {\n origin: input.storageOrigin,\n apiKey: input.storageApiKey,\n account: input.account,\n gateway: input.gateway,\n workspace: input.workspaceKey,\n base,\n previousPin: input.previousPin,\n repoRoot: input.source.repoRoot,\n workspaceDir: input.source.workspaceDir,\n notes,\n });\n if (\"memory\" in sourceOutcome) return { kind: \"memory\" };\n sourceRef = sourceOutcome.source;\n phase = \"upload\";\n }\n\n const pinned = await pinWorkspaceVersionManifest(deps, {\n origin: input.storageOrigin,\n apiKey: input.storageApiKey,\n account: input.account,\n gateway: input.gateway,\n workspace: input.workspaceKey,\n report,\n ssrReport,\n integrityMap: outcome.fileIntegrity,\n files: outcome.fileIntegrity,\n ...(sourceRef ? { source: sourceRef } : {}),\n ...(input.gitSha ? { gitSha: input.gitSha } : {}),\n notes,\n });\n if (!pinned) return { kind: \"missing-manifest\" };\n const pin = pinned.pin;\n if (outcome.result.storage === \"memory\" || pin.storage === \"memory\") {\n return { kind: \"memory\" };\n }\n\n return {\n kind: \"ok\",\n pin,\n storage: outcome.result.storage,\n entryIntegrity,\n ssrEntryIntegrity,\n ...(pinned.shared ? { shared: pinned.shared } : {}),\n uploaded: outcome.uploaded,\n skipped: outcome.skipped,\n durationMs: Date.now() - startedAt,\n detail: {\n base,\n manifestFile: pin.file,\n notes,\n },\n };\n } catch (error) {\n return {\n kind: \"error\",\n message:\n error instanceof Error\n ? error.message\n : phase === \"source\"\n ? `[publish] source archive for ${input.workspaceKey} failed`\n : `[publish] bundle upload for ${input.workspaceKey} failed`,\n };\n }\n}\n\n/** Directory names the source leg needs (main workspaces only). */\nexport function sourceLegDirs(workspacePath: string): { repoRoot: string; workspaceDir: string } {\n return { repoRoot: dirname(workspacePath), workspaceDir: basename(workspacePath) };\n}\n"],"mappings":";;;;;;;;AA4CA,MAAa,qBAAqC;CAChD;CACA;CACA;CACA;CACA;CACA;CACA;AACF;AAEA,SAAS,eAAe,OAA6C;CAEnE,OAAO,IADO,MAAM,QAAQ,KAAK,SAAS,MAAM,KAAK,MAAM,YAAY,CACxD,IAAI,OAAO,KAAI,CAAE,QAAQ,CAAC,EAAE;AAC7C;;;AAaA,eAAe,oBACb,MACA,OAW4B;CAC5B,MAAM,WAAW,YAAY,MAAM,KAAK;CACxC,IAAI,cAAc,MAAM;CACxB,IAAI;CACJ,IAAI,MAAM,aACR,IAAI;EACF,MAAM,WAAW,MAAM,KAAK,mBAAmB;GAAE,MAAM,MAAM;GAAM,KAAK,MAAM;EAAY,CAAC;EAC3F,IAAI,SAAS,OAAO;GAClB,MAAM,OAAO,mBAAmB,MAAM,OAAO,UAAU,SAAS,KAAK;GACrE,YAAY,SAAS;GACrB,cAAc,KAAK;GACnB,IAAI,KAAK,UAAU,GACjB,MAAM,MAAM,KAAK,GAAG,KAAK,QAAQ,8CAA8C;EAEnF;CACF,QAAQ;EACN,MAAM,MAAM,KAAK,mDAAmD;CACtE;CAGF,MAAM,MAAM,KACV,aAAa,YAAY,OAAO,MAAM,MAAM,MAAM,OAAO,UAAU,eAAe,WAAW,EAAE,MAAM,MAAM,MAC7G;CACA,MAAM,SACJ,YAAY,SAAS,IACjB,MAAM,KAAK,oBAAoB;EAC7B,QAAQ,MAAM;EACd,QAAQ,MAAM;EACd,SAAS,MAAM;EACf,SAAS,MAAM;EACf,WAAW,MAAM;EACjB,OAAO;CACT,CAAC,IACD;EAAE,QAAQ;EAAG,YAAY;EAAG,WAAW,CAAC;CAAE;CAChD,OAAO;EACL;EACA,eAAe,oBAAoB,MAAM,OAAO,OAAO,WAAW,SAAS;EAC3E,UAAU,OAAO;EACjB,SAAS,MAAM,MAAM,SAAS,YAAY;EAC1C,OAAO,MAAM,MAAM;CACrB;AACF;;;;;AAMA,eAAe,4BACb,MACA,OAcmF;CACnF,IAAI,CAAC,MAAM,QAAQ,OAAO;CAC1B,MAAM,kBAAkB,KAAK,gCAAgC;EAC3D,QAAQ,MAAM;EACd,WAAW,MAAM;EACjB,cAAc,MAAM;EACpB,GAAI,MAAM,QAAQ,EAAE,OAAO,MAAM,MAAM,IAAI,CAAC;EAC5C,GAAI,MAAM,SAAS,EAAE,QAAQ,MAAM,OAAO,IAAI,CAAC;EAC/C,GAAI,MAAM,SAAS,EAAE,QAAQ,MAAM,OAAO,IAAI,CAAC;CACjD,CAAC;CACD,IAAI,CAAC,iBAAiB,OAAO;CAE7B,MAAM,eAAe,YAAY,gBAAgB,QAAQ;CACzD,MAAM,iBAAiB,MAAM,KAAK,aAAa;EAC7C,QAAQ,MAAM;EACd,QAAQ,MAAM;EACd,SAAS,MAAM;EACf,SAAS,MAAM;EACf,WAAW,MAAM;EACjB,OAAO,CACL;GACE,MAAM;GACN,OAAO,IAAI,YAAY,CAAC,CAAC,OAAO,GAAG,KAAK,UAAU,iBAAiB,MAAM,CAAC,EAAE,GAAG;EACjF,CACF;CACF,CAAC;CACD,MAAM,oBAAoB,eAAe,UAAU;CACnD,IAAI,CAAC,mBAAmB,OAAO;CAC/B,MAAM,MAAM,KAAK,oBAAoB,aAAa,cAAc;CAChE,OAAO;EACL,KAAK;GAAE,MAAM;GAAc,WAAW;GAAmB,SAAS,eAAe;EAAQ;EACzF,GAAI,gBAAgB,SAAS,EAAE,QAAQ,gBAAgB,OAAO,IAAI,CAAC;CACrE;AACF;;;;AAKA,eAAe,uBACb,MACA,OAYmD;CACnD,MAAM,QAAQ,MAAM;CACpB,MAAM,SAAS,yBAAyB;CACxC,IAAI;EACF,MAAM,UAAU,MAAM,KAAK,mBAAmB;GAC5C,UAAU,MAAM;GAChB,WAAW,MAAM;GACjB;EACF,CAAC;EACD,IAAI,MAAM,aACR,IAAI;GACF,MAAM,WAAW,MAAM,KAAK,mBAAmB;IAC7C,MAAM,MAAM;IACZ,KAAK,MAAM;GACb,CAAC;GACD,IAAI,SAAS,QAAQ,cAAc,QAAQ,WAAW;IACpD,MAAM,KAAK,8BAA8B,QAAQ,MAAM;IACvD,OAAO,EACL,QAAQ;KAAE,MAAM,SAAS,OAAO;KAAM,WAAW,SAAS,OAAO;IAAU,EAC7E;GACF;EACF,QAAQ,CAER;EAEF,MAAM,SAAS,MAAM,KAAK,aAAa;GACrC,QAAQ,MAAM;GACd,QAAQ,MAAM;GACd,SAAS,MAAM;GACf,SAAS,MAAM;GACf,WAAW,MAAM;GACjB,OAAO,CAAC;IAAE,MAAM,QAAQ;IAAM,OAAO,IAAI,WAAW,aAAa,QAAQ,IAAI,CAAC;GAAE,CAAC;EACnF,CAAC;EACD,IAAI,OAAO,YAAY,UAAU,OAAO,EAAE,QAAQ,KAAK;EACvD,MAAM,YAAY,OAAO,UAAU,QAAQ;EAC3C,IAAI,CAAC,WACH,MAAM,IAAI,MACR,uCAAuC,MAAM,UAAU,2BAA2B,QAAQ,MAC5F;EAEF,MAAM,KAAK,kBAAkB,QAAQ,MAAM;EAC3C,OAAO,EAAE,QAAQ;GAAE,MAAM,QAAQ;GAAM;EAAU,EAAE;CACrD,UAAU;EACR,OAAO,QAAQ;GAAE,WAAW;GAAM,OAAO;EAAK,CAAC;CACjD;AACF;AAgDA,eAAsB,sBACpB,OACgC;CAChC,MAAM,OAAO,MAAM,QAAQ;CAC3B,MAAM,OAAO,GAAG,MAAM,UAAU,WAAW,MAAM,QAAQ,GAAG,MAAM,QAAQ,GAAG,MAAM,aAAa;CAChG,MAAM,YAAY,KAAK,IAAI;CAC3B,IAAI,QAA6B;CACjC,IAAI;EACF,MAAM,YAAY,MAAM,KAAK,iBAAiB,MAAM,OAAO;EAC3D,MAAM,QAAkB,CAAC;EACzB,MAAM,UAAU,MAAM,oBAAoB,MAAM;GAC9C,QAAQ,MAAM;GACd,QAAQ,MAAM;GACd,SAAS,MAAM;GACf,SAAS,MAAM;GACf,WAAW,MAAM;GACjB,OAAO;GACP;GACA,aAAa,MAAM;GACnB;EACF,CAAC;EACD,MAAM,SAAS,KAAK,gBAAgB,MAAM,OAAO;EACjD,MAAM,YAAY,KAAK,gBAAgB,KAAK,MAAM,SAAS,KAAK,CAAC;EACjE,MAAM,iBAAiB,SAAS,QAAQ,cAAc,OAAO,SAAS;EACtE,MAAM,oBAAoB,WAAW,QACjC,QAAQ,cAAc,OAAO,UAAU,WACvC;EAEJ,IAAI;EACJ,IAAI,MAAM,QAAQ;GAChB,IAAI,QAAQ,OAAO,YAAY,UAC7B,OAAO,EAAE,MAAM,SAAS;GAE1B,QAAQ;GACR,MAAM,gBAAgB,MAAM,uBAAuB,MAAM;IACvD,QAAQ,MAAM;IACd,QAAQ,MAAM;IACd,SAAS,MAAM;IACf,SAAS,MAAM;IACf,WAAW,MAAM;IACjB;IACA,aAAa,MAAM;IACnB,UAAU,MAAM,OAAO;IACvB,cAAc,MAAM,OAAO;IAC3B;GACF,CAAC;GACD,IAAI,YAAY,eAAe,OAAO,EAAE,MAAM,SAAS;GACvD,YAAY,cAAc;GAC1B,QAAQ;EACV;EAEA,MAAM,SAAS,MAAM,4BAA4B,MAAM;GACrD,QAAQ,MAAM;GACd,QAAQ,MAAM;GACd,SAAS,MAAM;GACf,SAAS,MAAM;GACf,WAAW,MAAM;GACjB;GACA;GACA,cAAc,QAAQ;GACtB,OAAO,QAAQ;GACf,GAAI,YAAY,EAAE,QAAQ,UAAU,IAAI,CAAC;GACzC,GAAI,MAAM,SAAS,EAAE,QAAQ,MAAM,OAAO,IAAI,CAAC;GAC/C;EACF,CAAC;EACD,IAAI,CAAC,QAAQ,OAAO,EAAE,MAAM,mBAAmB;EAC/C,MAAM,MAAM,OAAO;EACnB,IAAI,QAAQ,OAAO,YAAY,YAAY,IAAI,YAAY,UACzD,OAAO,EAAE,MAAM,SAAS;EAG1B,OAAO;GACL,MAAM;GACN;GACA,SAAS,QAAQ,OAAO;GACxB;GACA;GACA,GAAI,OAAO,SAAS,EAAE,QAAQ,OAAO,OAAO,IAAI,CAAC;GACjD,UAAU,QAAQ;GAClB,SAAS,QAAQ;GACjB,YAAY,KAAK,IAAI,IAAI;GACzB,QAAQ;IACN;IACA,cAAc,IAAI;IAClB;GACF;EACF;CACF,SAAS,OAAO;EACd,OAAO;GACL,MAAM;GACN,SACE,iBAAiB,QACb,MAAM,UACN,UAAU,WACR,gCAAgC,MAAM,aAAa,WACnD,+BAA+B,MAAM,aAAa;EAC5D;CACF;AACF;;AAGA,SAAgB,cAAc,eAAmE;CAC/F,OAAO;EAAE,UAAU,QAAQ,aAAa;EAAG,cAAc,SAAS,aAAa;CAAE;AACnF"}