UNPKG

@embeddable.com/sdk-core

Version:

Core Embeddable SDK module responsible for web-components bundling and publishing.

919 lines (809 loc) 27.1 kB
import buildTypes, { EMB_OPTIONS_FILE_REGEX, EMB_TYPE_FILE_REGEX, } from "./buildTypes"; import prepare, { removeIfExists } from "./prepare"; import generate, {generateDTS, triggerWebComponentRebuild} from "./generate"; import open from "open"; import provideConfig from "./provideConfig"; import { CompilerBuildResults, CompilerWatcher } from "@stencil/core/compiler"; import { CompilerSystem, createNodeLogger, createNodeSys, } from "@stencil/core/sys/node"; import { RollupWatcher } from "rollup"; import * as http from "node:http"; import { IncomingMessage, Server, ServerResponse } from "http"; import { ChildProcess } from "node:child_process"; import { WebSocketServer } from "ws"; import * as chokidar from "chokidar"; import * as path from "path"; import { getToken, default as login } from "./login"; import axios from "axios"; import { findFiles } from "@embeddable.com/sdk-utils"; import { archive, CUBE_FILES, sendBuild, SECURITY_CONTEXT_FILES, CLIENT_CONTEXT_FILES, EMBEDDABLE_FILES, } from "./push"; import validate, { embeddableValidation, formatIssue } from "./validate"; import { checkNodeVersion, shouldSkipModelCheck } from "./utils"; import { createManifest } from "./cleanup"; import { selectWorkspace } from "./workspaceUtils"; import * as fs from "node:fs/promises"; import minimist from "minimist"; import { initLogger, logError } from "./logger"; import devLogger from "./devLogger"; import fg from "fast-glob"; import * as dotenv from "dotenv"; import ora, { Ora } from "ora"; import finalhandler from "finalhandler"; import serveStatic from "serve-static"; import { ResolvedEmbeddableConfig } from "./defineConfig"; import buildGlobalHooks from "./buildGlobalHooks"; import { createWatcherLock, delay, preventContentLength, waitUntilFileStable } from "./utils/dev.utils"; import { BuildResultsComponentGraph } from "@stencil/core/internal"; type FSWatcher = chokidar.FSWatcher; dotenv.config(); let wss: WebSocketServer; let changedFiles: string[] = []; let browserWindow: ChildProcess | null = null; let lastEmbeddableError: string | null = null; let previewWorkspace: string; // Build coordination to prevent duplicate plugin builds let pluginBuildInProgress = false; let pendingPluginBuilds: (() => Promise<void>)[] = []; const SERVER_PORT = 8926; const BUILD_DEV_DIR = ".embeddable-dev-build"; // NOTE: for backward compatibility, keep the file name as global.css const CUSTOM_CANVAS_CSS = "/global.css"; let stencilWatcher: CompilerWatcher | undefined; let isActiveBundleBuild = false; /** We use two steps compilation for embeddable components. * 1. Compile *emb.ts files using plugin complier (sdk-react) * 2. Compile the web component using Stencil compiler. * These compilations can happen in parallel, but we need to ensure that * the first step is not started until the second step is finished (if recompilation is needed). * We use this lock to lock it before the second step starts and unlock it after the second step is finished. * */ const lock = createWatcherLock(); export const buildWebComponent = async (config: any) => { // if there is no watcher, then this is the first build. We need to create a watcher // otherwise we can just trigger a rebuild if (!stencilWatcher) { stencilWatcher = (await generate(config, "sdk-react")) as CompilerWatcher; stencilWatcher.on("buildFinish", (e) => onWebComponentBuildFinish(e, config), ); stencilWatcher.start(); } else { await triggerWebComponentRebuild(config); } }; const executePluginBuilds = async ( config: ResolvedEmbeddableConfig, watchers: Array<RollupWatcher | FSWatcher>, ) => { if (pluginBuildInProgress) { // If a plugin build is already in progress, queue this one return new Promise<void>((resolve) => { pendingPluginBuilds.push(async () => { await doPluginBuilds(config, watchers); resolve(); }); }); } else { // Start the plugin build immediately await doPluginBuilds(config, watchers); } }; const doPluginBuilds = async ( config: ResolvedEmbeddableConfig, watchers: Array<RollupWatcher | FSWatcher>, ) => { pluginBuildInProgress = true; try { for (const getPlugin of config.plugins) { const plugin = getPlugin(); await plugin.validate(config); const watcher = await plugin.build(config); await configureWatcher(watcher as RollupWatcher, config); watchers.push(watcher as RollupWatcher); } } finally { pluginBuildInProgress = false; // Process any pending builds if (pendingPluginBuilds.length > 0) { const nextBuild = pendingPluginBuilds.shift(); if (nextBuild) { await nextBuild(); } } } }; const addToGitingore = async () => { try { const gitignorePath = path.resolve(process.cwd(), ".gitignore"); const gitignoreContent = await fs.readFile(gitignorePath, "utf8"); if (!gitignoreContent.includes(BUILD_DEV_DIR)) { await fs.appendFile(gitignorePath, `\n${BUILD_DEV_DIR}\n`); } } catch (e) { // ignore } }; const chokidarWatchOptions = { ignoreInitial: true, usePolling: false, // Ensure polling is disabled awaitWriteFinish: { stabilityThreshold: 200, pollInterval: 100, }, }; export default async () => { await initLogger("dev"); const breadcrumbs: string[] = []; try { const cliArgs = minimist(process.argv.slice(2)); await devLogger.init({ logFile: cliArgs["log-file"], eventsFile: cliArgs["events-file"], }); breadcrumbs.push("run dev"); checkNodeVersion(); addToGitingore(); process.on("warning", (e) => console.warn(e.stack)); const logger = createNodeLogger(); const sys = createNodeSys({ process }); const defaultConfig = await provideConfig(); const buildDir = path.resolve(defaultConfig.client.rootDir, BUILD_DEV_DIR); const config = { ...defaultConfig, dev: { watch: true, logger, sys, }, client: { ...defaultConfig.client, buildDir, webComponentRoot: path.resolve(buildDir, "web-component"), componentDir: path.resolve(buildDir, "web-component", "component"), stencilBuild: path.resolve(buildDir, "dist", "embeddable-wrapper"), tmpDir: path.resolve( defaultConfig.client.rootDir, ".embeddable-dev-tmp", ), }, }; breadcrumbs.push("prepare config"); await prepare(config); const serve = serveStatic(config.client.buildDir, { setHeaders: (res, path) => { if (path.includes("/dist/embeddable-wrapper/")) { // Prevent content length for HMR files preventContentLength(res); } }, }); let workspacePreparation = ora("Preparing workspace...").start(); breadcrumbs.push("get preview workspace"); try { previewWorkspace = await getPreviewWorkspace( workspacePreparation, config, ); } catch (e: any) { if (e.response?.status === 401) { // login and retry await login(); workspacePreparation = ora("Preparing workspace...").start(); previewWorkspace = await getPreviewWorkspace( workspacePreparation, config, ); } else { workspacePreparation.fail( e.response?.data?.errorMessage || "Unknown error: " + e.message, ); process.exit(1); } } workspacePreparation.succeed("Workspace is ready"); const server = http.createServer( async (request: IncomingMessage, res: ServerResponse) => { res.setHeader("Access-Control-Allow-Origin", "*"); res.setHeader( "Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS", ); res.setHeader( "Access-Control-Allow-Headers", "Content-Type, Authorization", ); if (request.method === "OPTIONS") { // Respond to OPTIONS requests with just the CORS headers and a 200 status code res.writeHead(200); res.end(); return; } const done = finalhandler(request, res); try { if (request.url?.endsWith(CUSTOM_CANVAS_CSS)) { res.writeHead(200, { "Content-Type": "text/css" }); res.end(await fs.readFile(config.client.customCanvasCss)); return; } } catch {} // Last line of defence: wait for the file to be fully written before // handing it to serve-static. This catches any race condition between // the WS "build success" notification and the actual HTTP request — // e.g. when buildFinish fires slightly before Stencil flushes files. const urlPath = (request.url ?? "").split("?")[0]; if ( urlPath.includes("/dist/embeddable-wrapper/") && urlPath.endsWith(".js") ) { const filePath = path.resolve( config.client.buildDir, urlPath.slice(1), ); await waitUntilFileStable(filePath, "sourceMappingURL", { maxAttempts: 40, // up to ~2 s; fast in the happy path requiredStableCount: 2, }).catch(() => { // If the check times out we still serve — better a partial file // warning in the console than a hung request. }); } serve(request, res, done); }, ); const { themeWatcher, lifecycleWatcher } = await buildGlobalHooks(config); const dtsOra = ora("Generating component type files...").start(); await generateDTS(config) dtsOra.succeed("Component type files generated"); wss = new WebSocketServer({ server }); wss.on("connection", (ws) => { if (lastEmbeddableError) { ws.send( JSON.stringify({ type: "embeddablesUpdateError", error: lastEmbeddableError, }), ); } }); server.listen(SERVER_PORT, async () => { const watchers: Array<RollupWatcher | FSWatcher> = []; if (sys?.onProcessInterrupt) { sys.onProcessInterrupt( async () => await onClose(server, sys, watchers, config), ); } // Build plugins first to populate componentsWithPreview if (config.pushComponents) { breadcrumbs.push("build plugins with coordination"); await executePluginBuilds(config, watchers); } breadcrumbs.push("create manifest"); await createManifest({ ctx: { ...config, client: { ...config.client, tmpDir: buildDir, }, }, typesFileName: "embeddable-types.js", stencilWrapperFileName: "embeddable-wrapper.js", metaFileName: "embeddable-components-meta.js", editorsMetaFileName: "embeddable-editors-meta.js", }); await sendBuildChanges(config); if (config.pushComponents) { const customCanvasCssWatch = globalCustomCanvasWatcher(config); watchers.push(customCanvasCssWatch); if (themeWatcher) { await globalHookWatcher(themeWatcher, "themeProvider"); watchers.push(themeWatcher); } if (lifecycleWatcher) { await globalHookWatcher(lifecycleWatcher, "lifecycleHook"); watchers.push(lifecycleWatcher); } } else { await openDevWorkspacePage(config.previewBaseUrl, previewWorkspace); } const cubeSecurityContextAndClientContextWatch = await cubeSecurityContextAndClientContextWatcher(config); watchers.push(cubeSecurityContextAndClientContextWatch); if (config.pushEmbeddables) { await sendEmbeddableChanges(config, { isInitialSync: true }); const embeddableWatchers = await embeddableWatcher(config); watchers.push(...embeddableWatchers); } }); } catch (error: any) { try { await devLogger.close(); } catch { // never let logger cleanup hide the original error } await logError({ command: "dev", breadcrumbs, error }); console.log(error); process.exit(1); } }; export const configureWatcher = async ( watcher: RollupWatcher, ctx: ResolvedEmbeddableConfig, ) => { watcher.on("change", (path) => { changedFiles.push(path); }); watcher.on("event", async (e) => { if (e.code === "START") { await lock.waitUntilFree(); } if (e.code === "BUNDLE_START") { isActiveBundleBuild = true; await onBuildStart(ctx); } if (e.code === "BUNDLE_END") { lock.lock(); isActiveBundleBuild = false; if (stencilWatcher && shouldRebuildWebComponent()) { try { await fs.rm( path.resolve(ctx.client.buildDir, "dist", "embeddable-wrapper"), { recursive: true }, ); } catch (error) { console.error("Error cleaning up build directory:", error); } } await onBundleBuildEnd(ctx); changedFiles = []; } if (e.code === "ERROR") { lock.unlock(); isActiveBundleBuild = false; sendMessage("componentsBuildError", { error: e.error?.message }); changedFiles = []; } }); }; export const globalHookWatcher = async ( watcher: RollupWatcher, key: string, ) => { watcher.on("change", (path) => { changedFiles.push(path); }); watcher.on("event", async (e) => { if (e.code === "BUNDLE_START") { sendMessage(`${key}BuildStart`, { changedFiles }); } if (e.code === "BUNDLE_END") { sendMessage(`${key}BuildSuccess`, { version: new Date().getTime() }); changedFiles = []; } if (e.code === "ERROR") { sendMessage("componentsBuildError", { error: e.error?.message }); changedFiles = []; } }); }; const sendMessage = (type: string, meta = {}) => { wss?.clients?.forEach((ws) => { ws.send(JSON.stringify({ type, ...meta })); }); }; const typeFilesFilter = (f: string) => EMB_OPTIONS_FILE_REGEX.test(f) || EMB_TYPE_FILE_REGEX.test(f); const onlyTypesChanged = () => changedFiles.length !== 0 && changedFiles.filter(typeFilesFilter).length === changedFiles.length; const isTypeFileChanged = () => changedFiles.findIndex(typeFilesFilter) >= 0; const onBuildStart = async (ctx: ResolvedEmbeddableConfig) => { if (changedFiles.length === 0 || isTypeFileChanged()) { await buildTypes(ctx); } sendMessage("componentsBuildStart", { changedFiles }); }; export const openDevWorkspacePage = async ( previewBaseUrl: string, workspaceId: string, ) => { ora( `Preview workspace is available at ${previewBaseUrl}/workspace/${workspaceId}`, ).info(); return await open(`${previewBaseUrl}/workspace/${workspaceId}`); }; function shouldRebuildWebComponent() { return !onlyTypesChanged() || changedFiles.length === 0; } const onBundleBuildEnd = async (ctx: ResolvedEmbeddableConfig) => { if (shouldRebuildWebComponent()) { await buildWebComponent(ctx); } else { lock.unlock(); sendMessage("componentsBuildSuccess"); } }; const cubeSecurityContextAndClientContextWatcher = async ( ctx: ResolvedEmbeddableConfig, ): Promise<FSWatcher> => { let filesToWatch: string[] = []; if (ctx.pushComponents) { const clientContextFiles = await fg("**/*.cc.{yaml,yml}", { cwd: ctx.client.presetsSrc, absolute: true, }); filesToWatch = [...filesToWatch, ...clientContextFiles]; } if (ctx.pushModels) { const [cubeFiles, securityContextFiles] = await Promise.all([ fg("**/*.cube.{yaml,yml,js}", { cwd: ctx.client.modelsSrc, absolute: true, }), fg("**/*.sc.{yaml,yml}", { cwd: ctx.client.presetsSrc, absolute: true, }), ]); filesToWatch = [...filesToWatch, ...cubeFiles, ...securityContextFiles]; } const fsWatcher = chokidar.watch(filesToWatch, chokidarWatchOptions); fsWatcher.on("all", () => sendBuildChanges(ctx)); return fsWatcher; }; const embeddableWatcher = async ( ctx: ResolvedEmbeddableConfig, ): Promise<FSWatcher[]> => { const embeddableFiles = await fg("**/*.embeddable.{yaml,yml}", { cwd: ctx.client.srcDir, absolute: true, }); const knownFiles = new Set(embeddableFiles); const fsWatcher = chokidar.watch(embeddableFiles, chokidarWatchOptions); const allWatchers: FSWatcher[] = [fsWatcher]; // Watch the directory for newly created .embeddable.yml files only. // Existing files are already tracked by fsWatcher above. const dirWatcher = chokidar.watch(ctx.client.srcDir, { ...chokidarWatchOptions, ignoreInitial: true, }); const onEmbeddableEvent = (change: string, filePath: string): void => { devLogger.marker("change_detected", { scope: "embeddable", change, file: filePath, }); void sendEmbeddableChanges(ctx).catch((error) => { console.error( `Failed to sync embeddable change (${change} ${filePath}):`, error, ); }); }; dirWatcher.on("add", (filePath) => { if ( /\.embeddable\.(yaml|yml)$/.test(filePath) && !knownFiles.has(filePath) ) { knownFiles.add(filePath); fsWatcher.add(filePath); onEmbeddableEvent("add", filePath); } }); allWatchers.push(dirWatcher); fsWatcher.on("all", onEmbeddableEvent); // When a watched embeddable file is removed, forget it so the dirWatcher above // re-registers it if a file with the same name is recreated later. The // dirWatcher ignores paths still present in knownFiles, so without this a // deleted-then-restored embeddable would stop syncing until the next // dev-server restart. fsWatcher.on("unlink", (filePath) => { if (/\.embeddable\.(yaml|yml)$/.test(filePath)) { knownFiles.delete(filePath); fsWatcher.unwatch(filePath); } }); return allWatchers; }; const globalCustomCanvasWatcher = ( ctx: ResolvedEmbeddableConfig, ): FSWatcher => { const fsWatcher = chokidar.watch( ctx.client.customCanvasCss, chokidarWatchOptions, ); fsWatcher.on("all", async () => { sendMessage("globalCssUpdateSuccess"); }); return fsWatcher; }; export const sendBuildChanges = async (ctx: ResolvedEmbeddableConfig) => { const isValid = await validate({ ...ctx, pushEmbeddables: false }); if (!isValid) { return sendMessage("dataModelsAndOrSecurityContextUpdateError"); } // NOTE: This event name is kept for backward compatibility. Despite the name, // it tracks sync of data models, security contexts, and client contexts. sendMessage("dataModelsAndOrSecurityContextUpdateStart"); const sending = ora( "Synchronising data models and/or security contexts...", ).start(); let filesList: [string, string][] = []; if (ctx.pushComponents) { const clientContextFilesList = await findFiles( ctx.client.presetsSrc, CLIENT_CONTEXT_FILES, ); // Map the files to include their full filenames const clientContextFileList = [...clientContextFilesList].map( (entry): [string, string] => [path.basename(entry[1]), entry[1]], ); filesList = [ ...clientContextFileList, // add manifest to the archive [ "embeddable-manifest.json", path.resolve(ctx.client.buildDir, "embeddable-manifest.json"), ], ]; } if (ctx.pushModels) { const cubeFilesList = await findFiles(ctx.client.modelsSrc, CUBE_FILES); const securityContextFilesList = await findFiles( ctx.client.presetsSrc, SECURITY_CONTEXT_FILES, ); // Map the files to include their full filenames const cubeAndSecurityContextFileList = [ ...cubeFilesList, ...securityContextFilesList, ].map((entry): [string, string] => [path.basename(entry[1]), entry[1]]); filesList = [...filesList, ...cubeAndSecurityContextFileList]; } try { const token = await getToken(); await archive({ ctx, filesList, isDev: true, }); await sendBuild({ ...ctx, pushEmbeddables: false }, { workspaceId: previewWorkspace, token, skipModelCheck: shouldSkipModelCheck() }); } catch (e: any) { const errorMessage = e.response?.data?.errorMessage ?? e.message ?? "Unknown error"; sending.fail( `Data models and/or security context synchronization failed with error: ${errorMessage}`, ); return sendMessage("dataModelsAndOrSecurityContextUpdateError", { error: errorMessage }); } sending.succeed(`Data models and/or security context synchronized`); sendMessage("dataModelsAndOrSecurityContextUpdateSuccess"); }; type ErrorMetadata = { errors?: Record<string, string>; message?: string }; const extractDetailLines = (errorMetadata?: ErrorMetadata): string[] => { if (!errorMetadata) return []; const lines: string[] = []; if (errorMetadata.message) lines.push(errorMetadata.message); if (errorMetadata.errors) lines.push(...Object.values(errorMetadata.errors)); return lines; }; const buildErrorMessage = (e: any): string => { const base = e.response?.data?.errorMessage ?? e.message ?? "Unknown error"; const metadata: ErrorMetadata | undefined = e.response?.data?.errorMetadata; const lines = extractDetailLines(metadata); const bullets = lines.map((l) => ` • ${l}`).join("\n"); return lines.length ? `${base}\n${bullets}` : base; }; export const sendEmbeddableChanges = async ( ctx: ResolvedEmbeddableConfig, { isInitialSync = false }: { isInitialSync?: boolean } = {}, ) => { const embeddableFilesList = await findFiles( ctx.client.srcDir, EMBEDDABLE_FILES, ); if (!embeddableFilesList.length && isInitialSync) { lastEmbeddableError = null; return; } const cycleId = devLogger.startCycle("embeddable", { files: embeddableFilesList.map(([, p]) => p), }); const issues = await embeddableValidation(embeddableFilesList); if (issues.length) { const spinnerValidate = ora("Embeddable validation").start(); spinnerValidate.fail("One or more embeddable.yml files are invalid:"); issues.forEach((issue) => spinnerValidate.info(formatIssue(issue))); issues.forEach((issue) => devLogger.issue({ scope: "embeddable", stage: "validate", filePath: issue.filePath, message: issue.message, line: issue.line, column: issue.column, path: issue.path, }), ); lastEmbeddableError = issues.map(formatIssue).join("; "); devLogger.endCycle(cycleId, "embeddable", "error", { stage: "validate", errorCount: issues.length, }); return sendMessage("embeddablesUpdateError", { error: lastEmbeddableError, }); } sendMessage("embeddablesUpdateStart"); const sending = ora("Synchronising embeddables...").start(); const filesList: [string, string][] = embeddableFilesList.map( (entry): [string, string] => [path.basename(entry[1]), entry[1]], ); try { const token = await getToken(); await archive({ ctx, filesList, isDev: true, }); await sendBuild( { ...ctx, pushComponents: false, pushModels: false }, { workspaceId: previewWorkspace, token, skipModelCheck: shouldSkipModelCheck() }, ); } catch (e: any) { const errorMessage = buildErrorMessage(e); sending.fail(`Embeddables synchronization failed: ${errorMessage}`); lastEmbeddableError = errorMessage; devLogger.issue({ scope: "embeddable", stage: "sync", filePath: embeddableFilesList[0]?.[1] ?? "<unknown>", message: errorMessage, }); devLogger.endCycle(cycleId, "embeddable", "error", { stage: "sync" }); return sendMessage("embeddablesUpdateError", { error: errorMessage }); } lastEmbeddableError = null; sending.succeed(`Embeddables synchronized`); sendMessage("embeddablesUpdateSuccess"); devLogger.endCycle(cycleId, "embeddable", "ok"); }; const onClose = async ( server: Server, sys: CompilerSystem, watchers: Array<RollupWatcher | FSWatcher>, config: ResolvedEmbeddableConfig, ) => { server.close(); wss.close(); browserWindow?.unref(); await stencilWatcher?.close(); for (const watcher of watchers) { if (watcher.close) { await watcher.close(); } } for (const getPlugin of config.plugins) { const plugin = getPlugin(); await plugin.cleanup(config); } await removeIfExists(config); await sys.destroy(); await devLogger.close(); process.exit(0); }; const getPreviewWorkspace = async ( startedOra: Ora, ctx: ResolvedEmbeddableConfig, ): Promise<string> => { const token = await getToken(); const params = minimist(process.argv.slice(2)); let primaryWorkspaceId = params.w || params.workspace; if (!primaryWorkspaceId) { startedOra.stop(); // Stop current Ora, otherwise the last option will get hidden by it. const { workspaceId } = await selectWorkspace(ora, ctx, token); primaryWorkspaceId = workspaceId; startedOra.start(); } try { const instanceUrl = process.env.CUBE_CLOUD_ENDPOINT; const response = await axios.post( `${ctx.pushBaseUrl}/workspace/dev-workspace`, { primaryWorkspaceId, instanceUrl, pushModels: ctx.pushModels, pushComponents: ctx.pushComponents, pushEmbeddables: ctx.pushEmbeddables, }, { headers: { Authorization: `Bearer ${token}`, }, }, ); return response.data; } catch (e: any) { if (e.response.status === 401) { // login and retry await login(); return await getPreviewWorkspace(startedOra, ctx); } else { throw e; } } }; export async function onWebComponentBuildFinish( e: CompilerBuildResults, config: ResolvedEmbeddableConfig, ) { lock.unlock(); if (!browserWindow) { browserWindow = await openDevWorkspacePage( config.previewBaseUrl, previewWorkspace, ); return; } await delay(50); if (isActiveBundleBuild) { return; } if ( e.hasSuccessfulBuild && e.hmr?.componentsUpdated && e.hmr.reloadStrategy === "hmr" ) { try { await waitForStableHmrFiles(e.componentGraph, config); } finally { sendMessage("componentsBuildSuccessHmr", e.hmr); } } else { sendMessage("componentsBuildSuccess"); } } export async function waitForStableHmrFiles( componentGraph: BuildResultsComponentGraph | undefined, config: ResolvedEmbeddableConfig, ) { const promises = []; for (const files of Object.values(componentGraph ?? {})) { for (const file of files) { if (file.startsWith("embeddable-component")) { const fullPath = path.resolve( config.client.buildDir, "dist", "embeddable-wrapper", file, ); promises.push(waitUntilFileStable(fullPath, "sourceMappingURL")); } } } await Promise.all(promises); } export function resetStateForTesting() { stencilWatcher = undefined; isActiveBundleBuild = false; pluginBuildInProgress = false; pendingPluginBuilds = []; browserWindow = null; }