@embeddable.com/sdk-core/src/dev.ts

Core Embeddable SDK module responsible for web-components bundling and publishing.

import buildTypes, {
  EMB_OPTIONS_FILE_REGEX,
  EMB_TYPE_FILE_REGEX,
} from "./buildTypes";
import prepare, { removeIfExists } from "./prepare";
import generate from "./generate";
import provideConfig from "./provideConfig";
import {
  CompilerSystem,
  createNodeLogger,
  createNodeSys,
} from "@stencil/core/sys/node";
import { RollupWatcher } from "rollup";
import * as http from "node:http";
import { IncomingMessage, Server, ServerResponse } from "http";
import { ChildProcess } from "node:child_process";
import { WebSocketServer } from "ws";
import * as chokidar from "chokidar";
import * as path from "path";
import { getToken, default as login } from "./login";
import axios from "axios";
import { findFiles } from "@embeddable.com/sdk-utils";
import {
  archive,
  CUBE_FILES,
  sendBuild,
  SECURITY_CONTEXT_FILES,
  CLIENT_CONTEXT_FILES,
} from "./push";
import validate from "./validate";
import { checkNodeVersion } from "./utils";
import { createManifest } from "./cleanup";
import { selectWorkspace } from "./workspaceUtils";
import * as fs from "node:fs/promises";
import minimist from "minimist";
import { initLogger, logError } from "./logger";
import fg from "fast-glob";
import * as dotenv from "dotenv";
import ora, { Ora } from "ora";
import finalhandler from "finalhandler";
import serveStatic from "serve-static";
import { ResolvedEmbeddableConfig } from "./defineConfig";
import buildGlobalHooks from "./buildGlobalHooks";

type FSWatcher = chokidar.FSWatcher;

dotenv.config();

let wss: WebSocketServer;
let changedFiles: string[] = [];
let browserWindow: ChildProcess | null = null;

let previewWorkspace: string;

const SERVER_PORT = 8926;
const BUILD_DEV_DIR = ".embeddable-dev-build";
const GLOBAL_CSS = "/global.css";

const buildWebComponent = async (config: any) => {
  await generate(config, "sdk-react");
};

const addToGitingore = async () => {
  try {
    const gitignorePath = path.resolve(process.cwd(), ".gitignore");
    const gitignoreContent = await fs.readFile(gitignorePath, "utf8");

    if (!gitignoreContent.includes(BUILD_DEV_DIR)) {
      await fs.appendFile(gitignorePath, `\n${BUILD_DEV_DIR}\n`);
    }
  } catch (e) {
    // ignore
  }
};

const chokidarWatchOptions = {
  ignoreInitial: true,
  usePolling: false, // Ensure polling is disabled
  awaitWriteFinish: {
    stabilityThreshold: 200,
    pollInterval: 100,
  },
};

export default async () => {
  await initLogger("dev");
  const breadcrumbs: string[] = [];

  try {
    breadcrumbs.push("run dev");
    checkNodeVersion();
    addToGitingore();

    process.on("warning", (e) => console.warn(e.stack));

    const logger = createNodeLogger();
    const sys = createNodeSys({ process });

    const defaultConfig = await provideConfig();

    const buildDir = path.resolve(defaultConfig.client.rootDir, BUILD_DEV_DIR);

    const config = {
      ...defaultConfig,
      dev: {
        watch: true,
        logger,
        sys,
      },
      client: {
        ...defaultConfig.client,
        buildDir,
        componentDir: path.resolve(buildDir, "component"),
        stencilBuild: path.resolve(buildDir, "dist", "embeddable-wrapper"),
        tmpDir: path.resolve(
          defaultConfig.client.rootDir,
          ".embeddable-dev-tmp"
        ),
      },
    };

    breadcrumbs.push("prepare config");
    await prepare(config);

    const serve = serveStatic(config.client.buildDir);

    let workspacePreparation = ora("Preparing workspace...").start();

    breadcrumbs.push("get preview workspace");
    try {
      previewWorkspace = await getPreviewWorkspace(
        workspacePreparation,
        config
      );
    } catch (e: any) {
      if (e.response?.status === 401) {
        // login and retry
        await login();
        workspacePreparation = ora("Preparing workspace...").start();
        previewWorkspace = await getPreviewWorkspace(
          workspacePreparation,
          config
        );
      } else {
        workspacePreparation.fail(
          e.response?.data?.errorMessage || "Unknown error: " + e.message
        );
        process.exit(1);
      }
    }

    workspacePreparation.succeed("Workspace is ready");

    const server = http.createServer(
      async (request: IncomingMessage, res: ServerResponse) => {
        res.setHeader("Access-Control-Allow-Origin", "*");
        res.setHeader(
          "Access-Control-Allow-Methods",
          "GET, POST, PUT, DELETE, OPTIONS"
        );
        res.setHeader(
          "Access-Control-Allow-Headers",
          "Content-Type, Authorization"
        );

        if (request.method === "OPTIONS") {
          // Respond to OPTIONS requests with just the CORS headers and a 200 status code
          res.writeHead(200);
          res.end();
          return;
        }

        const done = finalhandler(request, res);

        try {
          if (request.url?.endsWith(GLOBAL_CSS)) {
            res.writeHead(200, { "Content-Type": "text/css" });
            res.end(await fs.readFile(config.client.globalCss));
            return;
          }
        } catch {}

        serve(request, res, done);
      }
    );

    const { themeWatcher, lifecycleWatcher } = await buildGlobalHooks(config);

    wss = new WebSocketServer({ server });
    server.listen(SERVER_PORT, async () => {
      const watchers: Array<RollupWatcher | FSWatcher> = [];
      if (sys?.onProcessInterrupt) {
        sys.onProcessInterrupt(
          async () => await onClose(server, sys, watchers, config)
        );
      }

      breadcrumbs.push("create manifest");
      await createManifest({
        ctx: {
          ...config,
          client: {
            ...config.client,
            tmpDir: buildDir,
          },
        },
        typesFileName: "embeddable-types.js",
        stencilWrapperFileName: "embeddable-wrapper.js",
        metaFileName: "embeddable-components-meta.js",
        editorsMetaFileName: "embeddable-editors-meta.js",
      });

      await sendBuildChanges(config);

      if (config.pushComponents) {
        for (const getPlugin of config.plugins) {
          const plugin = getPlugin();

          breadcrumbs.push("validate plugin");
          await plugin.validate(config);
          breadcrumbs.push("build plugin");
          const watcher = await plugin.build(config);
          breadcrumbs.push("configure watcher");
          await configureWatcher(watcher as RollupWatcher, config);
          watchers.push(watcher as RollupWatcher);
        }

        const customGlobalCssWatch = globalCssWatcher(config);
        watchers.push(customGlobalCssWatch);

        if (themeWatcher) {
          await globalHookWatcher(themeWatcher, config);
          watchers.push(themeWatcher);
        }
        if (lifecycleWatcher) {
          await globalHookWatcher(lifecycleWatcher, config);
          watchers.push(lifecycleWatcher);
        }
      } else {
        await openDevWorkspacePage(config.previewBaseUrl);
      }

      const cubeSecurityContextAndClientContextWatch =
        await cubeSecurityContextAndClientContextWatcher(config);
      watchers.push(cubeSecurityContextAndClientContextWatch);
    });
  } catch (error: any) {
    await logError({ command: "dev", breadcrumbs, error });
    console.log(error);
    process.exit(1);
  }
};

const configureWatcher = async (
  watcher: RollupWatcher,
  ctx: ResolvedEmbeddableConfig
) => {
  watcher.on("change", (path) => {
    changedFiles.push(path);
  });

  watcher.on("event", async (e) => {
    if (e.code === "BUNDLE_START") {
      await onBuildStart(ctx);
    }
    if (e.code === "BUNDLE_END") {
      await onBundleBuildEnd(ctx);
      changedFiles = [];
    }
    if (e.code === "ERROR") {
      sendMessage("componentsBuildError", { error: e.error?.message });
      changedFiles = [];
    }
  });
};

const globalHookWatcher = async (watcher: RollupWatcher, ctx: any) => {
  watcher.on("change", (path) => {
    changedFiles.push(path);
  });

  watcher.on("event", async (e) => {
    if (e.code === "BUNDLE_START") {
      sendMessage("componentsBuildStart", { changedFiles });
    }
    if (e.code === "BUNDLE_END") {
      sendMessage("componentsBuildSuccess");
      changedFiles = [];
    }
    if (e.code === "ERROR") {
      sendMessage("componentsBuildError", { error: e.error?.message });
      changedFiles = [];
    }
  });
};

const sendMessage = (type: string, meta = {}) => {
  wss?.clients?.forEach((ws) => {
    ws.send(JSON.stringify({ type, ...meta }));
  });
};

const typeFilesFilter = (f: string) =>
  EMB_OPTIONS_FILE_REGEX.test(f) || EMB_TYPE_FILE_REGEX.test(f);
const onlyTypesChanged = () =>
  changedFiles.length !== 0 &&
  changedFiles.filter(typeFilesFilter).length === changedFiles.length;
const isTypeFileChanged = () => changedFiles.findIndex(typeFilesFilter) >= 0;

const onBuildStart = async (ctx: ResolvedEmbeddableConfig) => {
  if (changedFiles.length === 0 || isTypeFileChanged()) {
    await buildTypes(ctx);
  }
  sendMessage("componentsBuildStart", { changedFiles });
};

const openDevWorkspacePage = async (previewBaseUrl: string) => {
  const open = (await import("open")).default;
  return await open(`${previewBaseUrl}/workspace/${previewWorkspace}`);
};

const onBundleBuildEnd = async (ctx: ResolvedEmbeddableConfig) => {
  if (!onlyTypesChanged() || changedFiles.length === 0) {
    await buildWebComponent(ctx);
  }
  if (browserWindow == null) {
    browserWindow = await openDevWorkspacePage(ctx.previewBaseUrl);
  } else {
    sendMessage("componentsBuildSuccess");
  }
};

const cubeSecurityContextAndClientContextWatcher = async (
  ctx: ResolvedEmbeddableConfig
): Promise<FSWatcher> => {
  let filesToWatch: any = [];

  if (ctx.pushComponents) {
    const clientContextFiles = await fg("**/*.cc.{yaml,yml}", {
      cwd: ctx.client.presetsSrc,
      absolute: true,
    });
    filesToWatch = [...filesToWatch, ...clientContextFiles];
  }

  if (ctx.pushModels) {
    const [cubeFiles, securityContextFiles] = await Promise.all([
      fg("**/*.cube.{yaml,yml,js}", {
        cwd: ctx.client.modelsSrc,
        absolute: true,
      }),
      fg("**/*.sc.{yaml,yml}", {
        cwd: ctx.client.presetsSrc,
        absolute: true,
      }),
    ]);
    filesToWatch = [...filesToWatch, ...cubeFiles, ...securityContextFiles];
  }

  const fsWatcher = chokidar.watch(filesToWatch, chokidarWatchOptions);

  fsWatcher.on("all", () => sendBuildChanges(ctx));

  return fsWatcher;
};

const globalCssWatcher = (ctx: ResolvedEmbeddableConfig): FSWatcher => {
  const fsWatcher = chokidar.watch(ctx.client.globalCss, chokidarWatchOptions);

  fsWatcher.on("all", async () => {
    sendMessage("globalCssUpdateSuccess");
  });

  return fsWatcher;
};

const sendBuildChanges = async (ctx: ResolvedEmbeddableConfig) => {
  const isValid = await validate(ctx);

  if (!isValid) {
    return sendMessage("dataModelsAndOrSecurityContextUpdateError");
  }

  const sending = ora(
    "Synchronising data models and/or security contexts..."
  ).start();

  let filesList: [string, string][] = [];
  if (ctx.pushComponents) {
    const clientContextFilesList = await findFiles(
      ctx.client.presetsSrc,
      CLIENT_CONTEXT_FILES
    );

    // Map the files to include their full filenames
    const clientContextFileList = [...clientContextFilesList].map(
      (entry): [string, string] => [path.basename(entry[1]), entry[1]]
    );

    filesList = [...clientContextFileList];
  }

  if (ctx.pushModels) {
    const cubeFilesList = await findFiles(ctx.client.modelsSrc, CUBE_FILES);
    const securityContextFilesList = await findFiles(
      ctx.client.presetsSrc,
      SECURITY_CONTEXT_FILES
    );

    // Map the files to include their full filenames
    const cubeAndSecurityContextFileList = [
      ...cubeFilesList,
      ...securityContextFilesList,
    ].map((entry): [string, string] => [path.basename(entry[1]), entry[1]]);

    filesList = [
      ...filesList,
      ...cubeAndSecurityContextFileList,
      // add manifest to the archive
      [
        "embeddable-manifest.json",
        path.resolve(ctx.client.buildDir, "embeddable-manifest.json"),
      ],
    ];
  }

  const token = await getToken();
  await archive({
    ctx,
    filesList,
    isDev: true,
  });
  await sendBuild(ctx, { workspaceId: previewWorkspace, token });
  sending.succeed(`Data models and/or security context synchronized`);
  sendMessage("dataModelsAndOrSecurityContextUpdateSuccess");
};

const onClose = async (
  server: Server,
  sys: CompilerSystem,
  watchers: Array<RollupWatcher | FSWatcher>,
  config: ResolvedEmbeddableConfig
) => {
  server.close();
  wss.close();
  browserWindow?.unref();
  for (const watcher of watchers) {
    if (watcher.close) {
      await watcher.close();
    }
  }
  for (const getPlugin of config.plugins) {
    const plugin = getPlugin();
    await plugin.cleanup(config);
  }
  await removeIfExists(config);
  await sys.destroy();
  process.exit(0);
};

const getPreviewWorkspace = async (
  startedOra: Ora,
  ctx: ResolvedEmbeddableConfig
): Promise<string> => {
  const token = await getToken();

  const params = minimist(process.argv.slice(2));
  let primaryWorkspaceId = params.w || params.workspace;

  if (!primaryWorkspaceId) {
    startedOra.stop(); // Stop current Ora, otherwise the last option will get hidden by it.
    const { workspaceId } = await selectWorkspace(ora, ctx, token);
    primaryWorkspaceId = workspaceId;
    startedOra.start();
  }

  try {
    const instanceUrl = process.env.CUBE_CLOUD_ENDPOINT;

    const response = await axios.post(
      `${ctx.pushBaseUrl}/workspace/dev-workspace`,
      {
        primaryWorkspaceId,
        instanceUrl,
        pushModels: ctx.pushModels,
        pushComponents: ctx.pushComponents,
      },
      {
        headers: {
          Authorization: `Bearer ${token}`,
        },
      }
    );
    return response.data;
  } catch (e: any) {
    if (e.response.status === 401) {
      // login and retry
      await login();
      return await getPreviewWorkspace(startedOra, ctx);
    } else {
      throw e;
    }
  }
};