@embeddable.com/sdk-core/src/validate.ts

Core Embeddable SDK module responsible for web-components bundling and publishing.

import * as fs from "node:fs/promises";
import * as YAML from "yaml";
import {
  errorFormatter,
  findFiles,
  formatErrorPath,
  resolveDeepestIssue,
} from "@embeddable.com/sdk-utils";
import { DIMENSION_TYPES, MEASURE_TYPES } from "@embeddable.com/core";
import { z } from "zod";
import ora from "ora";
import { checkNodeVersion } from "./utils";
import { ResolvedEmbeddableConfig } from "./defineConfig";

export type ValidationIssue = {
  filePath: string;
  message: string;
  line?: number;
  column?: number;
  path?: string;
};

export function formatIssue(issue: ValidationIssue): string {
  return `${issue.filePath}: ${issue.message}`;
}
const CUBE_YAML_FILE_REGEX = /^(.*)\.cube\.ya?ml$/;
const SECURITY_CONTEXT_FILE_REGEX = /^(.*)\.sc\.ya?ml$/;
const CLIENT_CONTEXT_FILE_REGEX = /^(.*)\.cc\.ya?ml$/;
const EMBEDDABLE_FILE_REGEX = /^(.*)\.embeddable\.ya?ml$/;

export default async (ctx: ResolvedEmbeddableConfig) => {
  checkNodeVersion();

  const spinnerValidate = ora("Data model validation...").start();

  const cubeFilesList = await findFiles(
    ctx.client.modelsSrc || ctx.client.srcDir,
    CUBE_YAML_FILE_REGEX,
  );
  const securityContextFilesList = await findFiles(
    ctx.client.presetsSrc || ctx.client.srcDir,
    SECURITY_CONTEXT_FILE_REGEX,
  );
  const clientContextFilesList = await findFiles(
    ctx.client.presetsSrc || ctx.client.srcDir,
    CLIENT_CONTEXT_FILE_REGEX,
  );

  const dataModelErrors = await dataModelsValidation(cubeFilesList);

  if (dataModelErrors.length) {
    spinnerValidate.fail("One or more cube.yaml files are invalid:");

    dataModelErrors.forEach((errorMessage) =>
      spinnerValidate.info(errorMessage),
    );

    process.exit(1);
  }

  spinnerValidate.succeed("Data model validation completed");

  const securityContextErrors = await securityContextValidation(
    securityContextFilesList,
  );

  const clientContextErrors = await clientContextValidation(
    clientContextFilesList,
  );

  if (securityContextErrors.length) {
    spinnerValidate.fail("One or more security context files are invalid:");

    securityContextErrors.forEach((errorMessage) =>
      spinnerValidate.info(errorMessage),
    );

    process.exit(1);
  }

  if (clientContextErrors.length) {
    spinnerValidate.fail("One or more client context files are invalid:");

    clientContextErrors.forEach((errorMessage) =>
      spinnerValidate.info(errorMessage),
    );

    process.exit(1);
  }

  let embeddableIssues: ValidationIssue[] = [];

  if (ctx.pushEmbeddables) {
    const embeddableFilesList = await findFiles(
      ctx.client.srcDir,
      EMBEDDABLE_FILE_REGEX,
    );
    embeddableIssues = await embeddableValidation(embeddableFilesList);

    if (embeddableIssues.length) {
      spinnerValidate.fail("One or more embeddable.yml files are invalid:");

      embeddableIssues.forEach((issue) =>
        spinnerValidate.info(formatIssue(issue)),
      );

      process.exit(1);
    }
  }

  return (
    dataModelErrors.length === 0 &&
    securityContextErrors.length === 0 &&
    clientContextErrors.length === 0 &&
    embeddableIssues.length === 0
  );
};

export async function dataModelsValidation(filesList: [string, string][]) {
  const errors: string[] = [];

  for (const [_, filePath] of filesList) {
    const fileContentRaw = await fs.readFile(filePath, "utf8");

    try {
      const cube = YAML.parse(fileContentRaw);
      if (!cube?.cubes && !cube?.views) {
        return [`${filePath}: At least one cubes or views must be defined`];
      }

      const cubeModelSafeParse = cubeModelSchema.safeParse(cube);
      const viewModelSafeParse = viewModelSchema.safeParse(cube);

      if (cube.cubes && !cubeModelSafeParse.success) {
        errorFormatter(cubeModelSafeParse.error.issues).forEach((error) => {
          errors.push(`${filePath}: ${error}`);
        });
      }

      if (cube.views && !viewModelSafeParse.success) {
        errorFormatter(viewModelSafeParse.error.issues).forEach((error) => {
          errors.push(`${filePath}: ${error}`);
        });
      }
    } catch (e: any) {
      errors.push(`${filePath}: ${e.message}`);
    }
  }

  return errors;
}

export async function securityContextValidation(filesList: [string, string][]) {
  const errors: string[] = [];

  const nameSet = new Set<string>();
  for (const [_, filePath] of filesList) {
    const fileContentRaw = await fs.readFile(filePath, "utf8");

    const cube = YAML.parse(fileContentRaw);

    cube.forEach((item: { name: string }) => {
      if (nameSet.has(item.name)) {
        errors.push(
          `${filePath}: security context with name "${item.name}" already exists`,
        );
      } else {
        nameSet.add(item.name);
      }
    });

    const safeParse = securityContextSchema.safeParse(cube);
    if (!safeParse.success) {
      errorFormatter(safeParse.error.issues).forEach((error) => {
        errors.push(`${filePath}: ${error}`);
      });
    }
  }

  return errors;
}

export async function clientContextValidation(filesList: [string, string][]) {
  const errors: string[] = [];

  const nameSet = new Set<string>();
  for (const [_, filePath] of filesList) {
    const fileContentRaw = await fs.readFile(filePath, "utf8");

    const cube = YAML.parse(fileContentRaw);

    cube.forEach((item: { name: string }) => {
      if (nameSet.has(item.name)) {
        errors.push(
          `${filePath}: client context with name "${item.name}" already exists`,
        );
      } else {
        nameSet.add(item.name);
      }
    });

    const safeParse = clientContextSchema.safeParse(cube);
    if (!safeParse.success) {
      errorFormatter(safeParse.error.issues).forEach((error) => {
        errors.push(`${filePath}: ${error}`);
      });
    }
  }

  return errors;
}

const cubeModelSchema = z
  .object({
    cubes: z
      .object({
        name: z.string(),
        dimensions: z
          .object({
            name: z.string(),
            type: z.enum(DIMENSION_TYPES),
          })
          .array()
          .optional(),
        measures: z
          .object({
            name: z.string(),
            type: z.enum(MEASURE_TYPES),
          })
          .array()
          .optional(),
      })
      .array()
      .min(1),
  })
  .refine(
    (data) =>
      data.cubes.every(
        (cube) => cube.dimensions?.length || cube.measures?.length,
      ),
    {
      message: "At least one measure or dimension must be defined",
      path: ["cubes"],
    },
  );

const viewModelSchema = z.object({
  views: z
    .object({
      name: z.string(),
      cubes: z
        .object({
          join_path: z.string(),
        })
        .array(),
    })
    .array()
    .min(1),
});

const securityContextSchema = z.array(
  z
    .object({
      name: z.string(),
      securityContext: z.object({}), // can be any object
      environment: z.string().optional(),
      useQueryRewrite: z.boolean().optional(),
      dataProvider: z.string().optional(),
      roles: z.array(z.string()).optional(),
      filters: z.array(z.object({})).optional(),
    })
    .strict(),
);

const clientContextSchema = z.array(
  z
    .object({
      name: z.string(),
      clientContext: z.object({}), // can be any object
      variables: z.record(z.any()).optional(), // variables key-value pairs
      canvas: z.object({}).optional(),
    })
    .strict(),
);

// --- Embeddable YAML schemas ---

const valueTypeEnum = z.enum(["VALUE", "VARIABLE"]);
const sourceTypeEnum = z.enum(["EVENT_PROPERTY", "VARIABLE"]);

const positionSchema = z
  .object({
    x: z.number(),
    y: z.number(),
  })
  .strict();

const dimensionsSchema = z
  .object({
    width: z.number(),
    height: z.number(),
  })
  .strict();

const nestedInputSchema = z
  .object({
    input: z.string(),
    value: z.any(),
    valueType: valueTypeEnum,
    parentValue: z.any().optional(),
  })
  .strict();

const filterSchema = z
  .object({
    member: z.string(),
    operator: z.string(),
    value: z.any().optional(),
    valueType: valueTypeEnum.optional(),
  })
  .strict();

const inputConfigSchema = z
  .object({
    dataset: z.string().optional(),
    filters: z.array(filterSchema).optional(),
    limit: z.number().optional(),
    order: z
      .array(
        z
          .object({
            member: z.string(),
            direction: z.enum(["asc", "desc"]),
          })
          .strict(),
      )
      .optional(),
    inputs: z.array(nestedInputSchema).optional(),
  })
  .strict();

const inputSchema = z
  .object({
    input: z.string(),
    inputType: z.string(),
    value: z.any().optional(),
    valueType: valueTypeEnum,
    array: z.boolean().optional(),
    config: inputConfigSchema.optional(),
  })
  .strict();

const setVariableConfigSchema = z
  .object({
    variable: z.string(),
    sourceType: sourceTypeEnum,
    sourceValue: z.string(),
  })
  .strict();

const drilldownVariableOverrideSchema = z
  .object({
    variable: z.string(),
    sourceType: sourceTypeEnum,
    sourceValue: z.string(),
  })
  .strict();

const drilldownConfigSchema = z
  .object({
    embeddable: z.string(),
    variableOverrides: z.array(drilldownVariableOverrideSchema).optional(),
  })
  .strict();

const eventSchema = z.discriminatedUnion("action", [
  z
    .object({
      event: z.string(),
      action: z.literal("SET_VARIABLE"),
      config: setVariableConfigSchema,
    })
    .strict(),
  z
    .object({
      event: z.string(),
      action: z.literal("DRILLDOWN"),
      config: drilldownConfigSchema,
    })
    .strict(),
]);

const widgetSchema = z
  .object({
    component: z.string(),
    position: positionSchema,
    dimensions: dimensionsSchema,
    inputs: z.array(inputSchema).optional(),
    events: z.array(eventSchema).optional(),
  })
  .strict();

const variableSchema = z
  .object({
    name: z.string(),
    type: z.string(),
    array: z.boolean().optional(),
    defaultValue: z.any().optional(),
    operation: z.enum(["NO_FILTER", "VALUE"]).optional(),
  })
  .strict();

const datasetSchema = z
  .object({
    name: z.string(),
    model: z.string(),
    filters: z.array(filterSchema).optional(),
  })
  .strict();

const templateInputSchema = z
  .object({
    input: z.string(),
    inputType: z.string().optional(),
    value: z.any().optional(),
    array: z.boolean().optional(),
    valueType: valueTypeEnum.optional(),
    config: inputConfigSchema.optional(),
    visible: z.boolean().optional(),
  })
  .strict();

const templateSchema = z
  .object({
    key: z.string(),
    name: z.string(),
    component: z.string(),
    description: z.string().optional(),
    icon: z.string().optional(),
    inputs: z.array(templateInputSchema).optional(),
  })
  .strict();

const starterCanvasWidgetSchema = z
  .object({
    template: z.string(),
    position: positionSchema.optional(),
    dimensions: dimensionsSchema.optional(),
    inputs: z.array(templateInputSchema).optional(),
  })
  .strict();

const customCanvasSchema = z
  .object({
    datasets: z
      .array(
        z
          .object({
            dataset: z.string(),
          })
          .strict(),
      )
      .optional(),
    templates: z.array(templateSchema).optional(),
    starterCanvas: z
      .object({
        widgets: z.array(starterCanvasWidgetSchema).optional(),
      })
      .strict()
      .optional(),
  })
  .strict();

const embeddableItemSchema = z
  .object({
    name: z.string(),
    title: z.string().optional(),
    description: z.string().optional(),
    variables: z.array(variableSchema).optional(),
    datasets: z.array(datasetSchema).optional(),
    widgets: z.array(widgetSchema).optional(),
    customCanvas: customCanvasSchema.optional(),
  })
  .strict();

const embeddableFileSchema = z
  .object({
    embeddables: z.array(embeddableItemSchema).min(1),
  })
  .strict();

type EmbeddableItem = z.infer<typeof embeddableItemSchema>;
type WidgetItem = z.infer<typeof widgetSchema>;

type NamedSegmentConfig = {
  arrayKey: string;
  displayLabel: string;
  nameField: string;
};

const NAMED_SEGMENT_CONFIGS: NamedSegmentConfig[] = [
  { arrayKey: "embeddables", displayLabel: "Embeddable", nameField: "name" },
  { arrayKey: "widgets", displayLabel: "widget", nameField: "component" },
  { arrayKey: "inputs", displayLabel: "input", nameField: "input" },
];

function resolveIndexedSegment(
  arrayKey: string,
  index: number,
  parentContext: unknown,
): { displayText: string; childContext: unknown } | null {
  const config = NAMED_SEGMENT_CONFIGS.find((c) => c.arrayKey === arrayKey);
  if (!config) return null;

  const array = (parentContext as Record<string, unknown>)?.[arrayKey];
  const item = Array.isArray(array)
    ? (array[index] as Record<string, unknown>)
    : undefined;
  const resolvedName = item?.[config.nameField];

  const displayText =
    typeof resolvedName === "string"
      ? `${config.displayLabel} '${resolvedName}'`
      : `${arrayKey}[${index}]`;

  return { displayText, childContext: item };
}

function formatEmbeddableIssuePath(
  issuePath: (string | number)[],
  rootData: unknown,
): string {
  const pathSegments: string[] = [];
  let currentContext: unknown = rootData;

  for (let position = 0; position < issuePath.length; position++) {
    const currentKey = issuePath[position];
    const nextElement = issuePath[position + 1];

    if (typeof currentKey === "string" && typeof nextElement === "number") {
      const resolved = resolveIndexedSegment(
        currentKey,
        nextElement,
        currentContext,
      );
      if (resolved) {
        pathSegments.push(resolved.displayText);
        currentContext = resolved.childContext;
        position++;
        continue;
      }
    }

    pathSegments.push(String(currentKey));
    currentContext = (currentContext as Record<string, unknown>)?.[
      String(currentKey)
    ];
  }

  return pathSegments.join(", ");
}

type RefContext = {
  filePath: string;
  variableNames: Set<string>;
  datasetNames: Set<string>;
  templateNames: Set<string>;
};

function validateDatasetRefs(
  emb: EmbeddableItem,
  ctx: RefContext,
): ValidationIssue[] {
  const issues: ValidationIssue[] = [];
  for (const ds of emb.datasets ?? []) {
    for (const f of ds.filters ?? []) {
      if (f.valueType === "VARIABLE" && !ctx.variableNames.has(f.value)) {
        issues.push({
          filePath: ctx.filePath,
          message: `dataset "${ds.name}" references undefined variable "${f.value}"`,
        });
      }
    }
  }
  return issues;
}

function validateWidgetInputRefs(
  widget: WidgetItem,
  ctx: RefContext,
): ValidationIssue[] {
  const issues: ValidationIssue[] = [];
  for (const inp of widget.inputs ?? []) {
    if (inp.valueType === "VARIABLE" && !ctx.variableNames.has(inp.value)) {
      issues.push({
        filePath: ctx.filePath,
        message: `widget "${widget.component}" input "${inp.input}" references undefined variable "${inp.value}"`,
      });
    }
    if (
      inp.inputType === "dataset" &&
      inp.valueType === "VALUE" &&
      inp.value &&
      !ctx.datasetNames.has(inp.value)
    ) {
      issues.push({
        filePath: ctx.filePath,
        message: `widget "${widget.component}" input "${inp.input}" references undefined dataset "${inp.value}"`,
      });
    }
  }
  return issues;
}

function validateWidgetEventRefs(
  widget: WidgetItem,
  ctx: RefContext,
): ValidationIssue[] {
  const issues: ValidationIssue[] = [];
  for (const evt of widget.events ?? []) {
    if (evt.action !== "SET_VARIABLE") continue;
    const config = evt.config as z.infer<typeof setVariableConfigSchema>;
    if (!ctx.variableNames.has(config.variable)) {
      issues.push({
        filePath: ctx.filePath,
        message: `widget "${widget.component}" event "${evt.event}" references undefined variable "${config.variable}"`,
      });
    }
  }
  return issues;
}

function widgetsOverlap(a: WidgetItem, b: WidgetItem): boolean {
  if (
    !Number.isFinite(a.dimensions.width) ||
    !Number.isFinite(a.dimensions.height) ||
    !Number.isFinite(b.dimensions.width) ||
    !Number.isFinite(b.dimensions.height) ||
    a.dimensions.width <= 0 ||
    a.dimensions.height <= 0 ||
    b.dimensions.width <= 0 ||
    b.dimensions.height <= 0
  ) {
    return false;
  }

  const aRight = a.position.x + a.dimensions.width;
  const bRight = b.position.x + b.dimensions.width;
  const aBottom = a.position.y + a.dimensions.height;
  const bBottom = b.position.y + b.dimensions.height;

  return (
    a.position.x < bRight &&
    aRight > b.position.x &&
    a.position.y < bBottom &&
    aBottom > b.position.y
  );
}

function validateWidgetOverlaps(
  emb: EmbeddableItem,
  filePath: string,
): ValidationIssue[] {
  const issues: ValidationIssue[] = [];
  const widgets = emb.widgets ?? [];

  for (let i = 0; i < widgets.length; i++) {
    for (let j = i + 1; j < widgets.length; j++) {
      const first = widgets[i];
      const second = widgets[j];

      if (!widgetsOverlap(first, second)) {
        continue;
      }

      issues.push({
        filePath,
        message:
          `embeddable "${emb.name}" widgets "${first.component}" and "${second.component}" overlap. ` +
          `"${first.component}" occupies x ${first.position.x}-${first.position.x + first.dimensions.width}, y ${first.position.y}-${first.position.y + first.dimensions.height}; ` +
          `"${second.component}" occupies x ${second.position.x}-${second.position.x + second.dimensions.width}, y ${second.position.y}-${second.position.y + second.dimensions.height}.`,
      });
    }
  }

  return issues;
}

function validateCustomCanvasRefs(
  emb: EmbeddableItem,
  ctx: RefContext,
): ValidationIssue[] {
  const issues: ValidationIssue[] = [];
  for (const ref of emb.customCanvas?.datasets ?? []) {
    if (!ctx.datasetNames.has(ref.dataset)) {
      issues.push({
        filePath: ctx.filePath,
        message: `customCanvas references undefined dataset "${ref.dataset}"`,
      });
    }
  }
  for (const sw of emb.customCanvas?.starterCanvas?.widgets ?? []) {
    if (!ctx.templateNames.has(sw.template)) {
      issues.push({
        filePath: ctx.filePath,
        message: `starterCanvas references undefined template "${sw.template}"`,
      });
    }
  }
  return issues;
}

function validateTemplateKeys(
  emb: EmbeddableItem,
  filePath: string,
): ValidationIssue[] {
  const issues: ValidationIssue[] = [];
  const seen = new Set<string>();
  for (const t of emb.customCanvas?.templates ?? []) {
    if (seen.has(t.key)) {
      issues.push({
        filePath,
        message: `customCanvas has duplicate template key "${t.key}"`,
      });
    } else {
      seen.add(t.key);
    }
  }
  return issues;
}

function validateEmbeddableRefs(
  emb: EmbeddableItem,
  filePath: string,
): ValidationIssue[] {
  const templateKeyIssues = validateTemplateKeys(emb, filePath);
  const ctx: RefContext = {
    filePath,
    variableNames: new Set((emb.variables ?? []).map((v) => v.name)),
    datasetNames: new Set((emb.datasets ?? []).map((d) => d.name)),
    templateNames: new Set(
      (emb.customCanvas?.templates ?? []).map((t) => t.key),
    ),
  };

  return [
    ...templateKeyIssues,
    ...validateDatasetRefs(emb, ctx),
    ...(emb.widgets ?? []).flatMap((widget) => [
      ...validateWidgetInputRefs(widget, ctx),
      ...validateWidgetEventRefs(widget, ctx),
    ]),
    ...validateWidgetOverlaps(emb, filePath),
    ...validateCustomCanvasRefs(emb, ctx),
  ];
}

function yamlParseIssues(
  doc: YAML.Document.Parsed,
  filePath: string,
): ValidationIssue[] {
  return doc.errors.map((err) => {
    const linePos = err.linePos?.[0];
    return {
      filePath,
      message: err.message,
      line: linePos?.line,
      column: linePos?.col,
    };
  });
}

function locateZodIssue(
  doc: YAML.Document.Parsed,
  lineCounter: YAML.LineCounter,
  zodPath: (string | number)[],
): { line?: number; column?: number } {
  if (zodPath.length === 0) return {};
  const node = doc.getIn(zodPath, true) as
    | { range?: [number, number, number] }
    | undefined;
  if (!node?.range || typeof node.range[0] !== "number") return {};
  const pos = lineCounter.linePos(node.range[0]);
  return { line: pos.line, column: pos.col };
}

function schemaIssues(
  zError: z.ZodError,
  doc: YAML.Document.Parsed,
  lineCounter: YAML.LineCounter,
  parsed: unknown,
  filePath: string,
): ValidationIssue[] {
  return zError.issues.map((zIssue) => {
    const unwrapped = resolveDeepestIssue(zIssue);
    const formattedPath = formatEmbeddableIssuePath(unwrapped.path, parsed);
    const message = formattedPath
      ? `${formattedPath}: ${unwrapped.message}`
      : unwrapped.message;
    return {
      filePath,
      message,
      path: formatErrorPath(unwrapped.path) || undefined,
      ...locateZodIssue(doc, lineCounter, unwrapped.path),
    };
  });
}

function duplicateNameAndRefIssues(
  embeddables: z.infer<typeof embeddableFileSchema>["embeddables"],
  filePath: string,
  nameSet: Set<string>,
): ValidationIssue[] {
  const issues: ValidationIssue[] = [];
  for (const emb of embeddables) {
    if (nameSet.has(emb.name)) {
      issues.push({
        filePath,
        message: `embeddable with name "${emb.name}" already exists`,
      });
    } else {
      nameSet.add(emb.name);
    }
    issues.push(...validateEmbeddableRefs(emb, filePath));
  }
  return issues;
}

async function validateEmbeddableFile(
  filePath: string,
  nameSet: Set<string>,
): Promise<ValidationIssue[]> {
  try {
    const fileContentRaw = await fs.readFile(filePath, "utf8");
    const lineCounter = new YAML.LineCounter();
    const doc = YAML.parseDocument(fileContentRaw, { lineCounter });

    if (doc.errors.length) return yamlParseIssues(doc, filePath);

    const parsed = doc.toJS();
    const safeParse = embeddableFileSchema.safeParse(parsed);
    if (!safeParse.success) {
      return schemaIssues(safeParse.error, doc, lineCounter, parsed, filePath);
    }

    return duplicateNameAndRefIssues(
      safeParse.data.embeddables,
      filePath,
      nameSet,
    );
  } catch (e: any) {
    return [{ filePath, message: e.message }];
  }
}

export async function embeddableValidation(
  filesList: [string, string][],
): Promise<ValidationIssue[]> {
  const nameSet = new Set<string>();
  const all: ValidationIssue[] = [];
  for (const [, filePath] of filesList) {
    all.push(...(await validateEmbeddableFile(filePath, nameSet)));
  }
  return all;
}