@puls-atlas/cli/dist/cmd/sync/apply.js

The Puls Atlas CLI tool for managing Atlas projects

import fs from 'fs';
import { createHash } from 'node:crypto';
import * as features from '../../utils/feature.js';
import { spreadIf } from '../../utils/value.js';
import { uploadSyncRuntimeConfig } from './runtimeArtifacts.js';
import { resolveSyncCloudRunDeployConfig } from './deploymentConfig.js';
import { compileSyncPlan, resolveSyncConfigLocation } from './config/syncConfig.js';
import { logger, getAtlasGeneratedFeatureConfigPath, loadAtlasFeatureCache, writeAtlasFeatureCache, writeAtlasGeneratedFeatureConfig } from '../../utils/index.js';
import { createSyncConfigFingerprint, promoteSyncReleaseState, resolveSyncReleaseTarget, resolveSyncRolloutPlan } from './release.js';
import { createSyncReadAliasEntryRows, createSyncReadAliasSummaryRows, createSyncSchemaPlanSummaryRows, inspectSyncManagedSchemas, inspectSyncReadAliases, reconcileSyncReadAliases } from './schemaPlan.js';
const SYNC_RUNTIME_CONFIG_VERSION = 3;
const POSTGRES_IDENTIFIER_MAX_LENGTH = 63;
const BIGQUERY_IDENTIFIER_MAX_LENGTH = 1024;
const UNSUPPORTED_SYNC_SOURCE_TYPES = new Set(['http', 'sql']);
export const assertNoUnsupportedEnabledSources = syncPlan => {
  const unsupported = syncPlan.pipelines.filter(pipeline => UNSUPPORTED_SYNC_SOURCE_TYPES.has(pipeline.source.type));
  if (unsupported.length === 0) {
    return;
  }
  const workloadList = unsupported.map(p => `  - ${p.workloadKey} (source: ${p.source.type})`).join('\n');
  throw new Error(`The following enabled workloads use source types that are not yet production-ready:\n${workloadList}\n` + 'The "http" and "sql" source types are scaffold-only and cannot be deployed. ' + 'Set workload.enabled = false or switch to a supported source type (firestore).');
};
const STABLE_SORT_EXCLUDED_ARRAY_PATHS = new Set(['destination.bigquery.primaryKey', 'destination.postgres.primaryKey']);
const normalizeIdentifierBase = (value, fallbackValue) => {
  const normalizedValue = value?.trim()?.toLowerCase().replace(/[^a-z0-9_]+/g, '_').replace(/^_+|_+$/g, '') ?? null;
  return normalizedValue || fallbackValue;
};
const createReleaseSuffix = releaseId => createHash('sha1').update(releaseId).digest('hex').slice(0, 12);
const createManagedIdentifier = (baseName, releaseId, maxLength, fallbackName) => {
  const normalizedBaseName = normalizeIdentifierBase(baseName, fallbackName);
  const suffix = createReleaseSuffix(releaseId);
  const baseLength = Math.max(1, maxLength - suffix.length - 2);
  return `${normalizedBaseName.slice(0, baseLength)}__${suffix}`;
};
const createBigQueryPhysicalTargets = (destinationConfig, release) => {
  const baseDataset = destinationConfig.bigquery?.dataset ?? null;
  const baseTable = destinationConfig.bigquery?.table ?? null;
  return {
    active: release.active ? {
      dataset: baseDataset,
      releaseId: release.active,
      table: createManagedIdentifier(baseTable, release.active, BIGQUERY_IDENTIFIER_MAX_LENGTH, destinationConfig.name),
      target: 'active'
    } : null,
    candidate: release.candidate ? {
      dataset: baseDataset,
      releaseId: release.candidate,
      table: createManagedIdentifier(baseTable, release.candidate, BIGQUERY_IDENTIFIER_MAX_LENGTH, destinationConfig.name),
      target: 'candidate'
    } : null,
    direct: {
      dataset: baseDataset,
      releaseId: null,
      table: baseTable,
      target: 'direct'
    }
  };
};
const createPostgresPhysicalTargets = (destinationConfig, release) => {
  const baseSchema = destinationConfig.postgres?.schema ?? null;
  const baseTable = destinationConfig.postgres?.table ?? null;
  return {
    active: release.active ? {
      releaseId: release.active,
      schema: baseSchema,
      table: createManagedIdentifier(baseTable, release.active, POSTGRES_IDENTIFIER_MAX_LENGTH, destinationConfig.name),
      target: 'active'
    } : null,
    candidate: release.candidate ? {
      releaseId: release.candidate,
      schema: baseSchema,
      table: createManagedIdentifier(baseTable, release.candidate, POSTGRES_IDENTIFIER_MAX_LENGTH, destinationConfig.name),
      target: 'candidate'
    } : null,
    direct: {
      releaseId: null,
      schema: baseSchema,
      table: baseTable,
      target: 'direct'
    }
  };
};
const DESTINATION_PHYSICAL_TARGET_FACTORIES = {
  bigquery: createBigQueryPhysicalTargets,
  postgres: createPostgresPhysicalTargets
};
const createDestinationPhysicalTargets = (destinationConfig, release) => {
  const destinationType = destinationConfig.type?.trim().toLowerCase();
  const createPhysicalTargets = DESTINATION_PHYSICAL_TARGET_FACTORIES[destinationType];
  if (!createPhysicalTargets) {
    throw new Error(`Atlas sync destination type "${destinationType}" is not supported for runtime target generation.`);
  }
  return createPhysicalTargets(destinationConfig, release);
};
const createStableFingerprintValue = (value, path = '') => {
  if (Array.isArray(value)) {
    const normalizedEntries = value.map((entry, index) => createStableFingerprintValue(entry, `${path}[${index}]`));
    if (STABLE_SORT_EXCLUDED_ARRAY_PATHS.has(path)) {
      return normalizedEntries;
    }
    if (path === 'pipelines') {
      return [...normalizedEntries].sort((left, right) => String(left?.workloadKey ?? '').localeCompare(String(right?.workloadKey ?? '')));
    }
    return normalizedEntries;
  }
  if (value && typeof value === 'object') {
    return Object.fromEntries(Object.entries(value).sort(([leftKey], [rightKey]) => leftKey.localeCompare(rightKey)).map(([entryKey, entryValue]) => [entryKey, createStableFingerprintValue(entryValue, path ? `${path}.${entryKey}` : entryKey)]));
  }
  return value;
};
export const createSyncRuntimeFingerprintPayload = syncPlan => createStableFingerprintValue({
  pipelines: syncPlan.pipelines,
  projectId: syncPlan.projectId,
  version: syncPlan.version
});
const createPhysicalTargetSignature = target => {
  if (!target) {
    return null;
  }
  if (target.dataset && target.table) {
    return `${target.dataset}.${target.table}`;
  }
  if (target.schema && target.table) {
    return `${target.schema}.${target.table}`;
  }
  return target.target ?? null;
};
const resolveRoutingTarget = (physicalTargets, targetName) => {
  if (!targetName) {
    return null;
  }
  return physicalTargets[targetName] ?? null;
};
const resolveRoutingTargets = (physicalTargets, targetNames = []) => {
  const dedupedTargets = new Map();
  for (const targetName of targetNames) {
    const target = resolveRoutingTarget(physicalTargets, targetName);
    if (!target) {
      continue;
    }
    const signature = createPhysicalTargetSignature(target);
    if (!signature || dedupedTargets.has(signature)) {
      continue;
    }
    dedupedTargets.set(signature, target);
  }
  return [...dedupedTargets.values()];
};
const createWriteTargetNames = rollout => {
  const targetNames = [rollout.writeTarget];
  if (rollout.mode === 'candidate-backfill' && rollout.backfillTarget && rollout.backfillTarget !== rollout.writeTarget) {
    targetNames.push(rollout.backfillTarget);
  }
  return [...new Set(targetNames.filter(Boolean))];
};
const createRuntimePipeline = (pipeline, releasePlan) => {
  const physicalTargets = createDestinationPhysicalTargets(pipeline.destination, releasePlan.release);
  const writeTargets = resolveRoutingTargets(physicalTargets, createWriteTargetNames(releasePlan.rollout));
  return {
    ...pipeline,
    destination: {
      ...pipeline.destination,
      physicalTargets,
      routing: {
        backfill: resolveRoutingTarget(physicalTargets, releasePlan.rollout.backfillTarget),
        read: resolveRoutingTarget(physicalTargets, releasePlan.rollout.readTarget),
        write: writeTargets[0] ?? resolveRoutingTarget(physicalTargets, releasePlan.rollout.writeTarget),
        writeTargets
      }
    }
  };
};
export const buildSyncRuntimeConfig = (context, options = {}) => {
  const syncPlan = options.syncPlan ?? context.syncPlan ?? compileSyncPlan(context.config);
  const configFingerprint = createSyncConfigFingerprint(createSyncRuntimeFingerprintPayload(syncPlan));
  const releasePlan = options.releasePlan ?? resolveSyncRolloutPlan(configFingerprint, options.cacheArtifact?.cache ?? null, {
    now: options.now
  });
  const runtimePipelines = syncPlan.pipelines.map(pipeline => createRuntimePipeline(pipeline, releasePlan));
  return {
    destinations: Object.fromEntries(runtimePipelines.map(pipeline => [pipeline.destination.name, pipeline.destination])),
    pipelines: runtimePipelines,
    projectId: syncPlan.projectId,
    release: releasePlan.release,
    releaseTarget: releasePlan.releaseTarget ?? resolveSyncReleaseTarget(releasePlan.release),
    rollout: releasePlan.rollout,
    ...spreadIf(options.readAliasPlan, {
      readAliasPlan: options.readAliasPlan
    }),
    ...spreadIf(options.schemaPlan, {
      schemaPlan: options.schemaPlan
    }),
    version: Math.max(syncPlan.version ?? 1, SYNC_RUNTIME_CONFIG_VERSION)
  };
};
const createSyncDeployCachePayload = ({
  cacheArtifact,
  context,
  runtimeConfig,
  runtimeConfigArtifact
}) => ({
  ...(cacheArtifact?.cache ?? {}),
  cachedAt: new Date().toISOString(),
  config: context.config,
  configFingerprint: runtimeConfig.rollout?.configFingerprint ?? null,
  configPath: context.configPath,
  environment: context.environment,
  feature: context.featureName,
  projectId: context.projectId,
  release: runtimeConfig.release,
  releaseTarget: runtimeConfig.releaseTarget,
  rollout: runtimeConfig.rollout,
  readAliasPlan: runtimeConfig.readAliasPlan ?? null,
  schemaPlan: runtimeConfig.schemaPlan ?? null,
  runtimeConfigPath: runtimeConfigArtifact.filePath,
  version: 1
});
const createSyncApplySummaryRows = (context, generationResult) => [{
  label: 'Project',
  value: context.projectId
}, context.environment ? {
  label: 'Environment',
  value: context.environment
} : null, {
  label: 'Version',
  value: generationResult.runtimeConfig.version
}, {
  label: 'Enabled pipelines',
  value: generationResult.enabledPipelineCount
}, {
  label: 'Release strategy',
  value: `${generationResult.release.strategy}` + `${generationResult.release.active ? ` [active=${generationResult.release.active}]` : ''}` + `${generationResult.release.candidate ? ` [candidate=${generationResult.release.candidate}]` : ''}`
}, {
  label: 'Read target',
  value: generationResult.rollout.readTarget ?? 'not set'
}, {
  label: 'Write target',
  value: generationResult.rollout.writeTarget ?? 'not set'
}, {
  label: 'Backfill target',
  value: generationResult.rollout.backfillTarget ?? 'not required'
}, ...createSyncReadAliasSummaryRows(generationResult.runtimeConfig.readAliasPlan), ...createSyncSchemaPlanSummaryRows(generationResult.runtimeConfig.schemaPlan)];
export const generateSyncRuntimeConfig = (context, options = {}, cwd = process.cwd()) => {
  const {
    cacheArtifact = null,
    dryRun = false,
    now,
    readAliasPlan,
    runtimeConfig: prebuiltRuntimeConfig,
    schemaPlan,
    writeGeneratedFeatureConfig = writeAtlasGeneratedFeatureConfig
  } = options;
  const syncPlan = context.syncPlan ?? compileSyncPlan(context.config);
  const runtimeConfig = prebuiltRuntimeConfig ?? buildSyncRuntimeConfig({
    ...context,
    syncPlan
  }, {
    cacheArtifact,
    now,
    readAliasPlan,
    schemaPlan
  });
  const runtimeConfigArtifact = dryRun === true ? {
    filePath: getAtlasGeneratedFeatureConfigPath('sync', context.projectId, cwd)
  } : writeGeneratedFeatureConfig('sync', context.projectId, runtimeConfig, cwd);
  return {
    enabledPipelineCount: runtimeConfig.pipelines.length,
    projectId: context.projectId,
    release: runtimeConfig.release,
    releaseTarget: runtimeConfig.releaseTarget,
    rollout: runtimeConfig.rollout,
    runtimeConfig,
    runtimeConfigArtifact,
    status: dryRun === true ? 'dry-run' : 'updated'
  };
};
export const runSyncApply = async (options = {}, dependencies = {}, workingDirectory = process.cwd()) => {
  const {
    existsSyncImpl = fs.existsSync,
    generateSyncRuntimeConfigImpl = generateSyncRuntimeConfig,
    inspectSyncManagedSchemasImpl = inspectSyncManagedSchemas,
    inspectSyncReadAliasesImpl = inspectSyncReadAliases,
    loadFeatureContextImpl = features.loadFeatureContext,
    loadFeatureCacheImpl = loadAtlasFeatureCache,
    loggerImpl = logger,
    resolveProjectSelectionImpl = features.resolveProjectSelection,
    writeFeatureCacheImpl = writeAtlasFeatureCache
  } = dependencies;
  let spinner;
  try {
    const projectSelection = await resolveProjectSelectionImpl(options);
    const configLocation = resolveSyncConfigLocation(workingDirectory, {
      existsSyncImpl
    });
    if (!configLocation.hasPreferred) {
      loggerImpl.info(`Atlas sync config is not configured for project ${projectSelection.projectId}. Run "atlas sync init" first.`);
      return {
        projectId: projectSelection.projectId,
        reason: 'missing-config',
        status: 'skipped'
      };
    }
    const context = await loadFeatureContextImpl('sync', options, {
      cwd: workingDirectory,
      resolveProjectSelectionImpl: async () => projectSelection
    });
    assertNoUnsupportedEnabledSources(compileSyncPlan(context.config));
    const cacheArtifact = loadFeatureCacheImpl('sync', context.projectId, workingDirectory);
    const previewRuntimeConfig = buildSyncRuntimeConfig(context, {
      cacheArtifact
    });
    const schemaPlan = await inspectSyncManagedSchemasImpl(context, previewRuntimeConfig, {
      strict: false
    });
    const readAliasPlan = await inspectSyncReadAliasesImpl(context, previewRuntimeConfig, {
      strict: false
    });
    spinner = loggerImpl.spinner('Generating Atlas sync runtime config...');
    const generationResult = generateSyncRuntimeConfigImpl(context, {
      cacheArtifact,
      dryRun: options.dryRun === true,
      runtimeConfig: {
        ...previewRuntimeConfig,
        ...(schemaPlan ? {
          schemaPlan
        } : {}),
        ...(readAliasPlan ? {
          readAliasPlan
        } : {})
      },
      readAliasPlan,
      schemaPlan
    }, workingDirectory);
    const writtenCacheArtifact = options.dryRun === true ? null : writeFeatureCacheImpl('sync', context.projectId, createSyncDeployCachePayload({
      cacheArtifact,
      context,
      runtimeConfig: generationResult.runtimeConfig,
      runtimeConfigArtifact: generationResult.runtimeConfigArtifact
    }), workingDirectory);
    spinner.succeed(options.dryRun === true ? 'Atlas sync runtime config dry run is ready.' : 'Atlas sync runtime config is ready.');
    loggerImpl.summary('Apply summary', createSyncApplySummaryRows(context, generationResult));
    loggerImpl.summary('Local files', [{
      label: 'Config',
      value: context.configPath
    }, {
      label: 'Runtime config',
      value: generationResult.runtimeConfigArtifact.filePath
    }, writtenCacheArtifact ? {
      label: 'Cache',
      value: writtenCacheArtifact.filePath
    } : null]);
    if (context.configPaths.length > 0) {
      loggerImpl.section('Workload files', context.configPaths, {
        detailOnly: true
      });
    }
    if (schemaPlan?.entries?.length > 0) {
      loggerImpl.summary('Managed schema targets', schemaPlan.entries.map(entry => ({
        label: `${entry.destinationName} (${entry.target}: ${entry.dataset}.${entry.table})`,
        value: entry.schemaStatus
      })));
    }
    if (readAliasPlan?.entries?.length > 0) {
      loggerImpl.summary('BigQuery read aliases', createSyncReadAliasEntryRows(readAliasPlan));
    }
    for (const warning of schemaPlan?.warnings ?? []) {
      loggerImpl.warning(warning);
    }
    for (const warning of readAliasPlan?.warnings ?? []) {
      loggerImpl.warning(warning);
    }
    if (options.dryRun === true) {
      loggerImpl.info('Dry run requested: Atlas did not write the generated runtime config.');
      loggerImpl.info('Dry run requested: Atlas did not update the local sync rollout cache.');
    }
    if (generationResult.enabledPipelineCount === 0) {
      loggerImpl.warning('Atlas sync config is valid, but no workloads are enabled for the selected project.');
    }
    return {
      ...generationResult,
      cacheArtifact: writtenCacheArtifact
    };
  } catch (error) {
    spinner?.fail('Failed to generate Atlas sync runtime config.');
    loggerImpl.error(error.message);
    return null;
  }
};
export default async options => runSyncApply(options);
export const runSyncPromote = async (options = {}, dependencies = {}, workingDirectory = process.cwd()) => {
  const {
    generateSyncRuntimeConfigImpl = generateSyncRuntimeConfig,
    inspectSyncManagedSchemasImpl = inspectSyncManagedSchemas,
    inspectSyncReadAliasesImpl = inspectSyncReadAliases,
    loadFeatureContextImpl = features.loadFeatureContext,
    loadFeatureCacheImpl = loadAtlasFeatureCache,
    loggerImpl = logger,
    reconcileSyncReadAliasesImpl = reconcileSyncReadAliases,
    resolveProjectSelectionImpl = features.resolveProjectSelection,
    resolveSyncCloudRunDeployConfigImpl = resolveSyncCloudRunDeployConfig,
    uploadSyncRuntimeConfigImpl = uploadSyncRuntimeConfig,
    writeFeatureCacheImpl = writeAtlasFeatureCache
  } = dependencies;
  let spinner;
  try {
    const projectSelection = await resolveProjectSelectionImpl(options);
    const context = await loadFeatureContextImpl('sync', options, {
      cwd: workingDirectory,
      resolveProjectSelectionImpl: async () => projectSelection
    });
    const cacheArtifact = loadFeatureCacheImpl('sync', context.projectId, workingDirectory);
    if (!cacheArtifact?.cache) {
      throw new Error('Atlas sync release promotion requires an existing local sync rollout cache. Run "atlas sync apply" first.');
    }
    spinner = loggerImpl.spinner('Promoting Atlas sync release...');
    const promotedReleasePlan = promoteSyncReleaseState(cacheArtifact.cache ?? null);
    const previewRuntimeConfig = buildSyncRuntimeConfig(context, {
      cacheArtifact,
      releasePlan: promotedReleasePlan
    });
    const schemaPlan = await inspectSyncManagedSchemasImpl(context, previewRuntimeConfig, {
      strict: false
    });
    const readAliasPlan = await inspectSyncReadAliasesImpl(context, previewRuntimeConfig, {
      strict: false
    });
    const generationResult = generateSyncRuntimeConfigImpl(context, {
      cacheArtifact,
      releasePlan: promotedReleasePlan,
      runtimeConfig: {
        ...previewRuntimeConfig,
        ...(schemaPlan ? {
          schemaPlan
        } : {}),
        ...(readAliasPlan ? {
          readAliasPlan
        } : {})
      },
      readAliasPlan,
      schemaPlan
    }, workingDirectory);
    const deployConfig = resolveSyncCloudRunDeployConfigImpl(context);
    uploadSyncRuntimeConfigImpl(generationResult.runtimeConfigArtifact.filePath, deployConfig.runtimeConfigUri);
    const reconciledReadAliasPlan = await reconcileSyncReadAliasesImpl(context, generationResult.runtimeConfig, {
      strict: true
    });
    if (reconciledReadAliasPlan.blockingIssues.length > 0) {
      throw new Error(reconciledReadAliasPlan.blockingIssues.join(' '));
    }
    const writtenCacheArtifact = writeFeatureCacheImpl('sync', context.projectId, createSyncDeployCachePayload({
      cacheArtifact,
      context,
      runtimeConfig: {
        ...generationResult.runtimeConfig,
        readAliasPlan: reconciledReadAliasPlan
      },
      runtimeConfigArtifact: generationResult.runtimeConfigArtifact
    }), workingDirectory);
    spinner.succeed('Atlas sync release promoted.');
    loggerImpl.summary('Promotion summary', [{
      label: 'Project',
      value: context.projectId
    }, context.environment ? {
      label: 'Environment',
      value: context.environment
    } : null, {
      label: 'Active release',
      value: generationResult.release.active
    }, {
      label: 'Candidate release',
      value: generationResult.release.candidate ?? 'cleared'
    }, {
      label: 'Runtime config',
      value: generationResult.runtimeConfigArtifact.filePath
    }, {
      label: 'Runtime config URI',
      value: deployConfig.runtimeConfigUri
    }, {
      label: 'Cache',
      value: writtenCacheArtifact.filePath
    }, ...createSyncReadAliasSummaryRows(reconciledReadAliasPlan), ...createSyncSchemaPlanSummaryRows(generationResult.runtimeConfig.schemaPlan)]);
    for (const warning of schemaPlan?.warnings ?? []) {
      loggerImpl.warning(warning);
    }
    for (const warning of reconciledReadAliasPlan?.warnings ?? []) {
      loggerImpl.warning(warning);
    }
    if (reconciledReadAliasPlan?.entries?.length > 0) {
      loggerImpl.summary('BigQuery read aliases', createSyncReadAliasEntryRows(reconciledReadAliasPlan));
    }
    return {
      ...generationResult,
      cacheArtifact: writtenCacheArtifact,
      readAliasPlan: reconciledReadAliasPlan,
      status: 'promoted'
    };
  } catch (error) {
    spinner?.fail('Failed to promote Atlas sync release.');
    loggerImpl.error(error.message, false);
    return {
      status: 'failed'
    };
  }
};