@puls-atlas/cli
Version:
The Puls Atlas CLI tool for managing Atlas projects
550 lines • 22.9 kB
JavaScript
import path from 'path';
import { isString } from 'es-toolkit/compat';
import { cloneDeep as clone, merge, omit } from 'es-toolkit/object';
import { isEqual, isPlainObject } from 'es-toolkit/predicate';
import { resolveAtlasWorkloadFile } from '../../../utils/atlas.js';
import { readJsonFile, writeJsonFile } from '../../../utils/file.js';
import { normalizeOptionalString } from '../../../utils/value.js';
import { SYNC_DESTINATION_BINDING_TYPES, SYNC_SOURCE_BINDING_TYPES, validateResolvedSyncConfig, validateSyncRootSection, validateSyncWorkloadConfig } from './syncValidation.js';
export const DEFAULT_SYNC_CLOUD_RUN_ARTIFACT_REGISTRY_LOCATION = 'europe-west1';
export const DEFAULT_SYNC_CLOUD_RUN_ARTIFACT_REGISTRY_PROJECT = 'puls-atlas-core';
export const DEFAULT_SYNC_CLOUD_RUN_REGION = 'europe-west1';
export const DEFAULT_SYNC_CLOUD_RUN_REPOSITORY = 'atlas-runtime-containers';
export const DEFAULT_SYNC_SERVICE_IMAGE = 'atlas-data-sync';
export const DEFAULT_SYNC_BACKFILL_JOB_IMAGE = 'atlas-data-backfill';
export const DEFAULT_SYNC_STATE_COLLECTION_PATH = 'sync/data';
export const DEFAULT_SYNC_TASK_QUEUE_LOCATION = 'europe-west1';
export const DEFAULT_SYNC_TASK_QUEUE_NAME = 'atlas-sync';
export const DEFAULT_SYNC_TERRAFORM_ROOT_DIR = 'services/sync/terraform';
export const DEFAULT_SYNC_TERRAFORM_MODULE_RELEASE_TAG = 'v0.1.0';
export const DEFAULT_SYNC_TERRAFORM_MODULE_SOURCE = `git::https://github.com/limebooth/atlas-terraform-modules.git//sync?ref=${DEFAULT_SYNC_TERRAFORM_MODULE_RELEASE_TAG}`;
const SYNC_WORKLOAD_NAME = 'sync';
const SYNC_CONFIG_FILE_NAME = 'config.json';
const SYNC_WORKLOAD_DIRECTORY_PREFIX = 'services/sync/';
const DEFAULT_SYNC_WORKLOAD_PATH = 'workloads/users.json';
const DEFAULT_SYNC_STATE_RECENT_LIMIT = 10;
const DEFAULT_SYNC_TRIGGER_REGION = 'europe-west1';
const DEFAULT_SYNC_BIGQUERY_DATASET = 'atlas_sync';
const DEFAULT_SYNC_BIGQUERY_TABLE = 'users_events';
const DEFAULT_SYNC_USERS_BIGQUERY_SCHEMA_FIELDS = [{
mode: 'REQUIRED',
name: '_sync_key',
type: 'STRING'
}, {
mode: 'NULLABLE',
name: 'displayName',
type: 'STRING'
}, {
mode: 'NULLABLE',
name: 'email',
type: 'STRING'
}, {
mode: 'NULLABLE',
name: 'fname',
type: 'STRING'
}, {
mode: 'NULLABLE',
name: 'lname',
type: 'STRING'
}, {
mode: 'NULLABLE',
name: 'phoneNumber',
type: 'STRING'
}, {
mode: 'NULLABLE',
name: 'photoURL',
type: 'STRING'
}, {
mode: 'REPEATED',
name: 'role',
type: 'STRING'
}, {
mode: 'NULLABLE',
name: 'status',
type: 'STRING'
}, {
mode: 'NULLABLE',
name: 'uid',
type: 'STRING'
}];
const DEFAULT_ATLAS_SYNC_WORKLOAD_CONTENT = {
workload: {
deletePolicy: 'emit-delete-event',
failureMode: 'independent',
name: 'users_to_bigquery_events'
},
source: {
description: 'User documents from Firestore.',
firestore: {
documentPathPattern: 'users/{documentId}'
},
name: 'users_firestore',
syncClass: 'delta-merge',
type: 'firestore'
},
mapper: {
export: 'mapUserToWarehouseRow',
name: 'userWarehouseRow',
source: 'mappers/users.ts'
},
destination: {
bigquery: {
primaryKey: ['sourceDocumentId', 'syncVersion'],
writeApi: 'storage-write',
schema: {
fields: DEFAULT_SYNC_USERS_BIGQUERY_SCHEMA_FIELDS,
mode: 'explicit'
}
},
deliveryMode: 'append',
description: 'Append-only BigQuery changelog for user events.',
name: 'users_bigquery_events',
schemaMode: 'managed',
type: 'bigquery'
}
};
const normalizePathSeparators = filePath => filePath.replaceAll(path.sep, '/');
export const resolveSyncConfigLocation = (cwd = process.cwd(), dependencies = {}) => {
const resolution = resolveAtlasWorkloadFile(SYNC_WORKLOAD_NAME, SYNC_CONFIG_FILE_NAME, cwd, dependencies);
return {
...resolution,
configDirectory: path.dirname(resolution.activePath),
configPath: resolution.activePath
};
};
const resolveSyncScopedPath = (targetPath, cwd, baseDirectory) => {
const normalizedTargetPath = normalizePathSeparators(targetPath);
if (normalizedTargetPath.startsWith(SYNC_WORKLOAD_DIRECTORY_PREFIX)) {
return path.resolve(cwd, normalizedTargetPath);
}
return path.resolve(baseDirectory, normalizedTargetPath);
};
const getSyncProjectEntries = syncConfig => isPlainObject(syncConfig?.projects) ? Object.entries(syncConfig.projects) : [];
const hasSyncProjectConfigs = syncConfig => getSyncProjectEntries(syncConfig).length > 0;
const createDefaultSyncCloudRunServiceAccountEmail = projectId => {
const normalizedProjectId = normalizeOptionalString(projectId);
return normalizedProjectId ? `${normalizedProjectId}@appspot.gserviceaccount.com` : null;
};
const createDefaultSyncDeployConfig = projectId => {
const serviceAccountEmail = createDefaultSyncCloudRunServiceAccountEmail(projectId);
return {
cloudRun: {
artifactRegistryLocation: DEFAULT_SYNC_CLOUD_RUN_ARTIFACT_REGISTRY_LOCATION,
artifactRegistryProject: DEFAULT_SYNC_CLOUD_RUN_ARTIFACT_REGISTRY_PROJECT,
backfillJobImage: DEFAULT_SYNC_BACKFILL_JOB_IMAGE,
mapperManifestUri: null,
region: DEFAULT_SYNC_CLOUD_RUN_REGION,
repository: DEFAULT_SYNC_CLOUD_RUN_REPOSITORY,
runtimeConfigUri: null,
...(serviceAccountEmail ? {
serviceAccountEmail
} : {}),
serviceImage: DEFAULT_SYNC_SERVICE_IMAGE
},
eventarc: {
triggerRegion: DEFAULT_SYNC_TRIGGER_REGION
},
syncState: {
collectionPath: DEFAULT_SYNC_STATE_COLLECTION_PATH,
recentLimit: DEFAULT_SYNC_STATE_RECENT_LIMIT
},
taskQueue: {
location: DEFAULT_SYNC_TASK_QUEUE_LOCATION,
queueName: DEFAULT_SYNC_TASK_QUEUE_NAME
},
terraform: {
moduleSource: DEFAULT_SYNC_TERRAFORM_MODULE_SOURCE,
rootDir: DEFAULT_SYNC_TERRAFORM_ROOT_DIR
}
};
};
export const createDefaultSyncProjectConfig = (options = {}) => {
const projectConfig = {
deploy: createDefaultSyncDeployConfig(options.projectId)
};
return projectConfig;
};
export const createDefaultSyncProjectBindings = () => ({
destinationBindings: {
users_bigquery_events: {
bigquery: {
dataset: DEFAULT_SYNC_BIGQUERY_DATASET,
table: DEFAULT_SYNC_BIGQUERY_TABLE
}
}
},
workloadBindings: {
users_to_bigquery_events: {
batchSize: 100,
enabled: false
}
}
});
export const createDefaultSyncSection = () => ({
version: 1,
workloads: [DEFAULT_SYNC_WORKLOAD_PATH]
});
const createDefaultProjectScopedSyncRootConfig = (options = {}) => ({
...createDefaultSyncSection(),
projects: {
[options.projectId]: {
...createDefaultSyncProjectConfig(options),
...createDefaultSyncProjectBindings()
}
}
});
const createCompleteSyncProjectConfig = (projectConfig, options = {}) => {
const defaultProjectConfig = createDefaultSyncProjectConfig(options);
const completeProjectConfig = {};
const normalizedEnvironment = normalizeOptionalString(projectConfig?.environment);
const selectedEnvironment = normalizeOptionalString(options.environment);
if (normalizedEnvironment !== null && selectedEnvironment !== null && normalizedEnvironment !== selectedEnvironment) {
throw new Error(`Atlas sync config project "${options.projectId ?? '<projectId>'}" declares environment "${normalizedEnvironment}", but the selected project resolves to "${selectedEnvironment}" via .firebaserc.`);
}
if (normalizedEnvironment) {
completeProjectConfig.environment = normalizedEnvironment;
}
completeProjectConfig.deploy = merge(clone(defaultProjectConfig.deploy), clone(projectConfig?.deploy ?? {}));
if (projectConfig?.sourceBindings !== undefined) {
completeProjectConfig.sourceBindings = clone(projectConfig.sourceBindings ?? {});
}
if (projectConfig?.destinationBindings !== undefined) {
completeProjectConfig.destinationBindings = clone(projectConfig.destinationBindings ?? {});
}
if (projectConfig?.workloadBindings !== undefined) {
completeProjectConfig.workloadBindings = clone(projectConfig.workloadBindings ?? {});
}
return completeProjectConfig;
};
const createCompleteSyncRootConfig = (syncConfig, options = {}) => {
const mergedConfig = {
...createDefaultSyncSection(),
...clone(omit(syncConfig ?? {}, ['projects']))
};
if (hasSyncProjectConfigs(syncConfig)) {
mergedConfig.projects = {};
for (const [projectId, projectConfig] of getSyncProjectEntries(syncConfig)) {
mergedConfig.projects[projectId] = createCompleteSyncProjectConfig(projectConfig, {
...options,
projectId
});
}
}
return validateSyncRootSection(mergedConfig);
};
const resolveSyncProjectConfig = (syncConfig, projectId, options = {}) => {
const validatedRootConfig = createCompleteSyncRootConfig(syncConfig, options);
const projectEntries = getSyncProjectEntries(validatedRootConfig);
const normalizedProjectId = normalizeOptionalString(projectId);
if (projectEntries.length === 0) {
return {
projectConfig: null,
projectId: normalizedProjectId,
rootConfig: validatedRootConfig
};
}
if (!normalizedProjectId) {
throw new Error('Atlas sync config defines project-specific settings under "projects". Re-run with --project to select a project.');
}
const resolvedProjectConfig = validatedRootConfig.projects?.[normalizedProjectId] ?? null;
if (!resolvedProjectConfig) {
throw new Error(`Atlas sync config does not define a "projects.${normalizedProjectId}" entry.`);
}
return {
projectConfig: resolvedProjectConfig,
projectId: normalizedProjectId,
rootConfig: validatedRootConfig
};
};
const normalizeSyncWorkloadConfig = (config, cwd, sourceBaseDirectory) => {
const normalizedConfig = clone(config);
if (isString(normalizedConfig.mapper?.source)) {
const resolvedSourcePath = resolveSyncScopedPath(normalizedConfig.mapper.source, cwd, sourceBaseDirectory);
normalizedConfig.mapper.source = normalizePathSeparators(path.relative(cwd, resolvedSourcePath));
}
return normalizedConfig;
};
const mergeRegistryEntry = (targetRegistry, entryName, entryConfig, registryName, sourceLabel) => {
const existingEntry = targetRegistry[entryName];
if (existingEntry === undefined) {
targetRegistry[entryName] = clone(entryConfig);
return;
}
if (!isEqual(existingEntry, entryConfig)) {
throw new Error(`Duplicate Atlas sync ${registryName} "${entryName}" found while loading ${sourceLabel}.`);
}
};
const mergeUniqueRegistryEntry = (targetRegistry, entryName, entryConfig, registryName, sourceLabel) => {
if (Object.hasOwn(targetRegistry, entryName)) {
throw new Error(`Duplicate Atlas sync ${registryName} "${entryName}" found while loading ${sourceLabel}.`);
}
targetRegistry[entryName] = clone(entryConfig);
};
const createResolvedSourceConfig = (sourceConfig, sourceBinding = null) => {
const resolvedSourceConfig = clone(omit(sourceConfig, ['name']));
const sourceType = sourceConfig.type.trim().toLowerCase();
if (sourceType === 'firestore') {
resolvedSourceConfig.firestore = merge(clone(sourceConfig.firestore ?? {}), sourceBinding?.firestore ?? {});
return resolvedSourceConfig;
}
if (sourceType === 'http') {
resolvedSourceConfig.http = merge(clone(sourceConfig.http ?? {}), sourceBinding?.http ?? {});
if (normalizeOptionalString(sourceBinding?.schedule)) {
resolvedSourceConfig.schedule = sourceBinding.schedule;
}
return resolvedSourceConfig;
}
resolvedSourceConfig.sql = merge(clone(sourceConfig.sql ?? {}), sourceBinding?.sql ?? {});
if (normalizeOptionalString(sourceBinding?.schedule)) {
resolvedSourceConfig.schedule = sourceBinding.schedule;
}
return resolvedSourceConfig;
};
const createResolvedDestinationConfig = (destinationConfig, destinationBinding = null) => {
const resolvedDestinationConfig = clone(omit(destinationConfig, ['name']));
const destinationType = destinationConfig.type.trim().toLowerCase();
if (destinationType === 'bigquery') {
resolvedDestinationConfig.bigquery = merge(clone(destinationConfig.bigquery ?? {}), destinationBinding?.bigquery ?? {});
return resolvedDestinationConfig;
}
if (destinationType !== 'postgres') {
throw new Error(`Atlas sync destination type "${destinationType}" is not supported while resolving project bindings.`);
}
resolvedDestinationConfig.postgres = merge(clone(destinationConfig.postgres ?? {}), destinationBinding?.postgres ?? {});
return resolvedDestinationConfig;
};
const resolveWorkloadPolicy = (workloadConfig, workloadBinding = null) => ({
batchSize: workloadBinding?.batchSize,
deletePolicy: workloadConfig.workload.deletePolicy,
enabled: workloadBinding?.enabled === true,
failureMode: workloadConfig.workload.failureMode
});
const assertMatchingSourceBinding = (sourceConfig, sourceBinding, projectId, isEnabled) => {
const sourceName = sourceConfig.name;
const sourceType = sourceConfig.type.trim().toLowerCase();
if (sourceBinding === null) {
if (isEnabled && sourceType !== 'firestore') {
throw new Error(`Atlas sync project "${projectId}" enables workload source "${sourceName}" but no matching source binding exists.`);
}
return;
}
const configuredBindingTypes = SYNC_SOURCE_BINDING_TYPES.filter(bindingType => sourceBinding?.[bindingType] !== undefined);
if (configuredBindingTypes[0] !== sourceType) {
throw new Error(`Atlas sync source binding "${sourceName}" must use the "${sourceType}" binding shape.`);
}
if (sourceType === 'firestore' && sourceBinding.schedule !== undefined) {
throw new Error(`Atlas sync source binding "${sourceName}" cannot define a schedule for firestore sources.`);
}
if ((sourceType === 'http' || sourceType === 'sql') && !normalizeOptionalString(sourceBinding.schedule)) {
throw new Error(`Atlas sync source binding "${sourceName}" must define a schedule for ${sourceType} sources.`);
}
};
const assertMatchingDestinationBinding = (destinationConfig, destinationBinding, projectId, isEnabled) => {
const destinationName = destinationConfig.name;
const destinationType = destinationConfig.type.trim().toLowerCase();
if (destinationBinding === null) {
if (isEnabled) {
throw new Error(`Atlas sync project "${projectId}" enables workload destination "${destinationName}" but no matching destination binding exists.`);
}
return;
}
const configuredBindingTypes = SYNC_DESTINATION_BINDING_TYPES.filter(bindingType => destinationBinding?.[bindingType] !== undefined);
if (configuredBindingTypes[0] !== destinationType) {
throw new Error(`Atlas sync destination binding "${destinationName}" must use the "${destinationType}" binding shape.`);
}
};
const assertNoDanglingBindings = (registry, bindings, bindingName, projectId) => {
for (const entryName of Object.keys(bindings ?? {})) {
if (!Object.hasOwn(registry, entryName)) {
throw new Error(`Atlas sync project "${projectId}" defines ${bindingName} for "${entryName}" but no matching workload-defined entry exists.`);
}
}
};
export const ensureSyncConfigSection = (cwd = process.cwd(), options = {}) => {
const createdFiles = [];
const updatedFiles = [];
const configLocation = resolveSyncConfigLocation(cwd);
const {
configPath
} = configLocation;
const existingSyncConfig = readJsonFile(configPath, {
allowMissing: true
});
const didSyncConfigExist = Boolean(existingSyncConfig);
const environment = normalizeOptionalString(options.environment);
const projectId = normalizeOptionalString(options.projectId);
let nextSyncConfigInput = existingSyncConfig;
if (!nextSyncConfigInput) {
nextSyncConfigInput = projectId ? createDefaultProjectScopedSyncRootConfig({
environment,
projectId
}) : createDefaultSyncSection();
} else if (!hasSyncProjectConfigs(nextSyncConfigInput) && projectId) {
nextSyncConfigInput = {
...clone(nextSyncConfigInput),
projects: {
[projectId]: createDefaultSyncProjectBindings()
}
};
} else if (hasSyncProjectConfigs(nextSyncConfigInput) && projectId) {
nextSyncConfigInput = {
...clone(nextSyncConfigInput),
projects: {
...(nextSyncConfigInput.projects ?? {}),
[projectId]: {
...(nextSyncConfigInput.projects?.[projectId] ?? {})
}
}
};
} else if (hasSyncProjectConfigs(nextSyncConfigInput) && !projectId) {
throw new Error('Atlas sync config defines project-specific settings under "projects". Re-run with --project to select a project.');
}
const nextSyncConfig = createCompleteSyncRootConfig(nextSyncConfigInput);
writeJsonFile(configPath, nextSyncConfig);
if (!didSyncConfigExist) {
createdFiles.push(configPath);
} else if (!isEqual(existingSyncConfig, nextSyncConfig)) {
updatedFiles.push(configPath);
}
const defaultWorkloadPath = path.join(configLocation.configDirectory, DEFAULT_SYNC_WORKLOAD_PATH);
if (!readJsonFile(defaultWorkloadPath, {
allowMissing: true
})) {
writeJsonFile(defaultWorkloadPath, DEFAULT_ATLAS_SYNC_WORKLOAD_CONTENT);
createdFiles.push(defaultWorkloadPath);
}
const {
projectConfig,
projectId: resolvedProjectId,
rootConfig
} = resolveSyncProjectConfig(nextSyncConfig, projectId);
return {
config: rootConfig,
configPath,
createdFiles,
projectConfig,
projectId: resolvedProjectId,
rootConfig,
updatedFiles
};
};
export const loadSyncConfig = (cwd = process.cwd(), options = {}) => {
const configLocation = resolveSyncConfigLocation(cwd);
const rootConfig = readJsonFile(configLocation.configPath);
const {
projectConfig,
projectId,
rootConfig: validatedRootConfig
} = resolveSyncProjectConfig(rootConfig, options.projectId, {
environment: options.environment ?? null
});
const resolvedConfig = {
deploy: clone(projectConfig?.deploy ?? {}),
destinations: {},
mappers: {},
pipelines: {},
projectId,
sources: {},
version: validatedRootConfig.version,
workloads: {}
};
const workloadPaths = [];
const sourceRegistry = {};
const destinationRegistry = {};
const workloadRegistry = {};
const sourceBindings = clone(projectConfig?.sourceBindings ?? {});
const destinationBindings = clone(projectConfig?.destinationBindings ?? {});
const workloadBindings = clone(projectConfig?.workloadBindings ?? {});
const configBaseDirectory = configLocation.configDirectory;
const sourceBaseDirectory = configLocation.configDirectory;
for (const workloadRelativePath of validatedRootConfig.workloads ?? []) {
const workloadPath = resolveSyncScopedPath(workloadRelativePath, cwd, configBaseDirectory);
const workloadConfig = normalizeSyncWorkloadConfig(validateSyncWorkloadConfig(readJsonFile(workloadPath), workloadPath), cwd, sourceBaseDirectory);
const workloadName = workloadConfig.workload.name;
const sourceName = workloadConfig.source.name;
const mapperName = workloadConfig.mapper.name;
const destinationName = workloadConfig.destination.name;
const workloadBinding = workloadBindings[workloadName] ?? null;
const sourceBinding = sourceBindings[sourceName] ?? null;
const destinationBinding = destinationBindings[destinationName] ?? null;
const workloadPolicy = resolveWorkloadPolicy(workloadConfig, workloadBinding);
assertMatchingSourceBinding(workloadConfig.source, sourceBinding, projectId ?? '<unknown>', workloadPolicy.enabled);
assertMatchingDestinationBinding(workloadConfig.destination, destinationBinding, projectId ?? '<unknown>', workloadPolicy.enabled);
mergeRegistryEntry(sourceRegistry, sourceName, createResolvedSourceConfig(workloadConfig.source, sourceBinding), 'source', workloadPath);
mergeRegistryEntry(resolvedConfig.mappers, mapperName, omit(workloadConfig.mapper, ['name']), 'mapper', workloadPath);
mergeRegistryEntry(destinationRegistry, destinationName, createResolvedDestinationConfig(workloadConfig.destination, destinationBinding), 'destination', workloadPath);
mergeUniqueRegistryEntry(workloadRegistry, workloadName, {
batchSize: workloadPolicy.batchSize,
deletePolicy: workloadPolicy.deletePolicy,
destination: destinationName,
enabled: workloadPolicy.enabled,
failureMode: workloadPolicy.failureMode,
mapper: mapperName,
source: sourceName
}, 'workload', workloadPath);
mergeUniqueRegistryEntry(resolvedConfig.pipelines, workloadName, {
destination: destinationName,
mapper: mapperName,
source: sourceName
}, 'pipeline', workloadPath);
workloadPaths.push(workloadPath);
}
assertNoDanglingBindings(sourceRegistry, sourceBindings, 'sourceBindings', projectId ?? '<unknown>');
assertNoDanglingBindings(destinationRegistry, destinationBindings, 'destinationBindings', projectId ?? '<unknown>');
assertNoDanglingBindings(workloadRegistry, workloadBindings, 'workloadBindings', projectId ?? '<unknown>');
resolvedConfig.sources = sourceRegistry;
resolvedConfig.destinations = destinationRegistry;
resolvedConfig.workloads = workloadRegistry;
return {
config: validateResolvedSyncConfig(resolvedConfig),
configPath: configLocation.configPath,
configPaths: workloadPaths,
projectConfig,
rootConfig: validatedRootConfig,
warnings: []
};
};
export const compileSyncPlan = resolvedConfig => {
const validatedConfig = validateResolvedSyncConfig(resolvedConfig);
if (!normalizeOptionalString(validatedConfig.projectId)) {
throw new Error('Atlas sync plan compilation requires a selected project.');
}
return {
version: validatedConfig.version,
projectId: validatedConfig.projectId,
pipelines: Object.entries(validatedConfig.workloads).filter(([, workloadConfig]) => workloadConfig.enabled === true).map(([workloadKey, workloadConfig]) => ({
destination: {
name: workloadConfig.destination,
...clone(validatedConfig.destinations[workloadConfig.destination])
},
mapper: {
name: workloadConfig.mapper,
...clone(validatedConfig.mappers[workloadConfig.mapper])
},
policy: {
batchSize: workloadConfig.batchSize,
deletePolicy: workloadConfig.deletePolicy,
enabled: workloadConfig.enabled,
failureMode: workloadConfig.failureMode
},
source: {
name: workloadConfig.source,
...clone(validatedConfig.sources[workloadConfig.source])
},
state: {
namespace: validatedConfig.deploy?.syncState?.collectionPath ?? null,
recentLimit: validatedConfig.deploy?.syncState?.recentLimit ?? null
},
workloadKey
}))
};
};
export default {
compileSyncPlan,
createDefaultSyncSection,
createDefaultSyncProjectBindings,
createDefaultSyncProjectConfig,
ensureSyncConfigSection,
loadSyncConfig,
resolveSyncConfigLocation
};