@embeddable.com/sdk-core/src/push.ts

Core Embeddable SDK module responsible for web-components bundling and publishing.

import * as fs from "node:fs/promises";
import * as fsSync from "node:fs";
import yazl from "yazl";
import axios, {AxiosResponse} from "axios";
import ora, { Ora } from "ora";
import { initLogger, logError } from "./logger";
import * as path from "path";

import provideConfig from "./provideConfig";
// @ts-ignore
import reportErrorToRollbar from "./rollbar.mjs";

import { findFiles } from "@embeddable.com/sdk-utils";
import { getToken } from "./login";
import { checkBuildSuccess, checkNodeVersion, getArgumentByKey, shouldSkipModelCheck } from "./utils";
import { selectWorkspace } from "./workspaceUtils";
import { ResolvedEmbeddableConfig } from "./defineConfig";

// grab cube files
export const CUBE_FILES = /^(.*)\.cube\.(ya?ml|js)$/;

export const CLIENT_CONTEXT_FILES = /^(.*)\.cc\.ya?ml$/;
export const SECURITY_CONTEXT_FILES = /^(.*)\.sc\.ya?ml$/;
export const EMBEDDABLE_FILES = /^(.*)\.embeddable\.ya?ml$/;

export default async () => {
  await initLogger("push");
  const breadcrumbs: string[] = [];
  let spinnerPushing;

  try {
    checkNodeVersion();
    breadcrumbs.push("checkNodeVersion");
    const isBuildSuccess = await checkBuildSuccess();
    const config = await provideConfig();
    const cubeVersion = getArgumentByKey(["--cube-version"]);
    const skipModelCheck = shouldSkipModelCheck();

    if (!isBuildSuccess && config.pushComponents) {
      console.error(
        "Build failed or not completed. Please run `embeddable:build` first.",
      );
      process.exit(1);
    }

    if (process.argv.includes("--api-key") || process.argv.includes("-k")) {
      spinnerPushing = ora("Using API key...").start();
      breadcrumbs.push("push by api key");
      try {
        await pushByApiKey(config, spinnerPushing, cubeVersion, skipModelCheck);
      } catch (error: any) {
        if (error.response?.data?.errorCode === "BUILDER-998") {
          spinnerPushing.fail(
            `Authentication failure. Server responded with: "${error.response?.data?.errorMessage}".
Ensure that your API key is valid for the region specified in the embeddable.config.ts|js file.
You are using the following region: ${config.region.replace("legacy-", "")} (${config.previewBaseUrl.replace("https://", "")} via ${config.pushBaseUrl})
Read more about deployment regions at https://docs.embeddable.com/deployment/deployment-regions`,
          );
          process.exit(1);
        }
        spinnerPushing.fail("Publishing failed");
        console.log(error.response?.data || error);
        process.exit(1);
      }
      publishedSectionFeedback(config, spinnerPushing);
      spinnerPushing.succeed("Published using API key");

      return;
    }

    breadcrumbs.push("push by standard login");
    const token = await verify(config);
    spinnerPushing = ora()
      .start()
      .info("No API Key provided. Standard login will be used.");

    breadcrumbs.push("select workspace");
    const { workspaceId, name: workspaceName } = await selectWorkspace(
      ora,
      config,
      token,
    );

    const workspacePreviewUrl = `${config.previewBaseUrl}/workspace/${workspaceId}`;
    const message = getArgumentByKey(["--message", "-m"]);

    breadcrumbs.push("build archive");
    await buildArchive(config);
    spinnerPushing.info(
      `Publishing to ${workspaceName} using ${workspacePreviewUrl}...`,
    );

    breadcrumbs.push("send build");
    await sendBuild(config, { workspaceId, token, message, cubeVersion, skipModelCheck });

    publishedSectionFeedback(config, spinnerPushing);
    spinnerPushing.succeed(
      `Published to ${workspaceName} using ${workspacePreviewUrl}`,
    );
  } catch (error: any) {
    spinnerPushing?.fail("Publishing failed");
    await logError({ command: "push", breadcrumbs, error });
    await reportErrorToRollbar(error);
    console.log(error.response?.data || error);
    process.exit(1);
  }
};

const publishedSectionFeedback = (
  config: ResolvedEmbeddableConfig,
  spinnerPushing: Ora,
) => {
  config.pushModels && spinnerPushing.succeed("Models published");
  config.pushComponents && spinnerPushing.succeed("Components published");
  config.pushEmbeddables && spinnerPushing.succeed("Embeddables published");
};

async function pushByApiKey(
  config: ResolvedEmbeddableConfig,
  spinner: any,
  cubeVersion?: string,
  skipModelCheck?: boolean,
) {
  const apiKey = getArgumentByKey(["--api-key", "-k"]);

  if (!apiKey) {
    spinner.fail("No API key provided");
    process.exit(1);
  }

  const email = getArgumentByKey(["--email", "-e"]);

  if (!email || !/\S+@\S+\.\S+/.test(email)) {
    spinner.fail(
      "Invalid email provided. Please provide a valid email using --email (-e) flag",
    );
    process.exit(1);
  }

  // message is optional
  const message = getArgumentByKey(["--message", "-m"]);

  await buildArchive(config);

  return sendBuildByApiKey(config, {
    apiKey,
    email,
    message,
    cubeVersion,
    skipModelCheck,
  });
}

async function verify(ctx: ResolvedEmbeddableConfig) {
  if (ctx.pushComponents) {
    try {
      await fs.access(ctx.client.buildDir);
    } catch (_e) {
      console.error("No embeddable build was produced.");
      process.exit(1);
    }
  }

  // TODO: initiate login if no/invalid token.
  const token = await getToken();

  if (!token) {
    console.error('Unauthorized. Please login using "npm run embeddable:login"');
    process.exit(1);
  }

  return token;
}

export async function buildArchive(config: ResolvedEmbeddableConfig) {
  const spinnerArchive = ora("Building...").start();

  if (!config.pushModels && !config.pushComponents && !config.pushEmbeddables) {
    spinnerArchive.fail(
      "Cannot push: pushModels, pushComponents, and pushEmbeddables are all disabled",
    );
    process.exit(1);
  }

  const filesList: [string, string][] = [];

  if (config.pushModels) {
    const cubeFilesList = await findFiles(
      config.client.modelsSrc || config.client.srcDir,
      CUBE_FILES,
    );

    const securityContextFilesList = await findFiles(
      config.client.presetsSrc || config.client.srcDir,
      SECURITY_CONTEXT_FILES,
    );

    filesList.push(
      ...cubeFilesList.map((entry): [string, string] => [
        path.basename(entry[1]),
        entry[1],
      ]),

      ...securityContextFilesList.map((entry): [string, string] => [
        path.basename(entry[1]),
        entry[1],
      ]),
    );
  }

  if (config.pushComponents) {
    const clientContextFilesList = await findFiles(
      config.client.presetsSrc || config.client.srcDir,
      CLIENT_CONTEXT_FILES,
    );

    filesList.push(
      ...clientContextFilesList.map((entry): [string, string] => [
        path.basename(entry[1]),
        entry[1],
      ]),
    );
  }

  if (config.pushEmbeddables) {
    const embeddableFilesList = await findFiles(
      config.client.srcDir,
      EMBEDDABLE_FILES,
    );

    filesList.push(
      ...embeddableFilesList.map((entry): [string, string] => [
        path.basename(entry[1]),
        entry[1],
      ]),
    );
  }

  await archive({
    ctx: config,
    filesList,
    isDev: false,
  });
  return spinnerArchive.succeed("Bundling completed");
}

export async function archive(args: {
  ctx: ResolvedEmbeddableConfig;
  filesList: [string, string][];
  isDev: boolean;
}) {
  const { ctx, filesList, isDev } = args;
  const zip = new yazl.ZipFile();

  if (!isDev) {
    if (ctx.pushComponents) {
      addDirectoryToZip(zip, ctx.client.buildDir);
    }
    // NOTE: for backward compatibility, keep the file name as global.css
    if (fsSync.existsSync(ctx.client.customCanvasCss)) {
      zip.addFile(ctx.client.customCanvasCss, "global.css", { compress: true });
    }
  }

  for (const [name, filePath] of filesList) {
    zip.addFile(filePath, name, { compress: true });
  }

  zip.end();

  return new Promise<void>((resolve, reject) => {
    const output = fsSync.createWriteStream(ctx.client.archiveFile);
    zip.outputStream.pipe(output);
    output.on("close", resolve);
    output.on("error", reject);
  });
}

function addDirectoryToZip(zip: yazl.ZipFile, dir: string) {
  if (!fsSync.existsSync(dir)) {
    return;
  }
  const entries = fsSync.readdirSync(dir, { recursive: true });
  for (const entry of entries) {
    const relativePath = String(entry);
    const fullPath = path.join(dir, relativePath);
    if (fsSync.statSync(fullPath).isFile()) {
      zip.addFile(fullPath, relativePath, { compress: true });
    }
  }
}

export async function createFormData(
  filePath: string,
  metadata: Record<string, any>,
) {
  const { FormData, Blob } = await import("formdata-node");
  const { fileFromPath } = await import("formdata-node/file-from-path");

  const file = await fileFromPath(filePath, "embeddable-build.zip");
  const form = new FormData();
  form.set("file", file, "embeddable-build.zip");

  const metadataBlob = new Blob([JSON.stringify(metadata)], {
    type: "application/json",
  });
  form.set("request", metadataBlob, "request.json");

  return form;
}

export async function sendBuildByApiKey(
  ctx: ResolvedEmbeddableConfig,
  {
    apiKey,
    email,
    message,
    cubeVersion,
    skipModelCheck,
  }: { apiKey: string; email: string; message?: string; cubeVersion?: string; skipModelCheck?: boolean },
) {
  const form = await createFormData(ctx.client.archiveFile, {
    pushModels: ctx.pushModels,
    pushComponents: ctx.pushComponents,
    pushEmbeddables: ctx.pushEmbeddables,
    starterEmbeddableIds: ctx.starterEmbeddables?.[ctx.region],
    authorEmail: email,
    description: message,
    ...(cubeVersion ? { cubeVersion } : {}),
    ...(skipModelCheck ? { skipModelCheck } : {}),
  });

  const response = await uploadFile(
    form,
    `${ctx.pushBaseUrl}/api/v1/bundle/upload`,
    apiKey,
  );
  await fs.rm(ctx.client.archiveFile);
  checkAndLogWarnings(response);

  return { ...response.data, message, cubeVersion };
}

export async function sendBuild(
  ctx: ResolvedEmbeddableConfig,
  {
    workspaceId,
    token,
    message,
    cubeVersion,
    skipModelCheck,
  }: {
    workspaceId: string;
    token: string;
    message?: string;
    cubeVersion?: string;
    skipModelCheck?: boolean;
  },
) {
  const form = await createFormData(ctx.client.archiveFile, {
    pushModels: ctx.pushModels,
    pushComponents: ctx.pushComponents,
    pushEmbeddables: ctx.pushEmbeddables,
    starterEmbeddableIds: ctx.starterEmbeddables?.[ctx.region],
    authorEmail: "",
    description: message,
    ...(cubeVersion ? { cubeVersion } : {}),
    ...(skipModelCheck ? { skipModelCheck } : {}),
  });

  const response = await uploadFile(
    form,
    `${ctx.pushBaseUrl}/bundle/${workspaceId}/upload`,
    token,
  );

  const suppressedCodes = [
    !ctx.pushModels && "WARN-001",
    !ctx.pushComponents && "WARN-003",
    !ctx.pushEmbeddables && "WARN-005",
  ].filter(Boolean) as string[];

  checkAndLogWarnings(response, suppressedCodes);

  await fs.rm(ctx.client.archiveFile);
}

async function uploadFile(formData: any, url: string, token: string) {
  return axios.post(url, formData, {
    headers: {
      "Content-Type": "multipart/form-data",
      Authorization: `Bearer ${token}`,
    },
    maxContentLength: Infinity,
    maxBodyLength: Infinity,
  });
}

function checkAndLogWarnings(response: AxiosResponse, suppressedCodes: string[] = []) {
  const suppressed = new Set(suppressedCodes);
  const warnings = (response.data.warnings || []) as string[];
  const visible = warnings.filter((w) => {
    const warningCode = /^([^:\s]+)/.exec(w)?.[1] ?? "";
    return !suppressed.has(warningCode);
  });
  if (visible.length > 0) {
    ora().warn(visible.join("\n"));
  }
}