@puls-atlas/cli
Version:
The Puls Atlas CLI tool for managing Atlas projects
589 lines • 38.8 kB
JavaScript
import { isBoolean, isString } from 'es-toolkit/compat';
import { isPlainObject } from 'es-toolkit/predicate';
import { isStringArray, normalizeOptionalString } from '../../../utils/value.js';
export const SYNC_SOURCE_TYPES = ['firestore', 'http', 'sql'];
export const SYNC_DESTINATION_TYPES = ['bigquery', 'postgres'];
export const SYNC_SOURCE_BINDING_TYPES = SYNC_SOURCE_TYPES;
export const SYNC_DESTINATION_BINDING_TYPES = SYNC_DESTINATION_TYPES;
const SUPPORTED_SYNC_SOURCE_SYNC_CLASSES = ['append-only', 'delta-merge', 'snapshot-replace'];
const SUPPORTED_SYNC_WORKLOAD_FAILURE_MODES = ['independent', 'stop-on-first-failure'];
const SUPPORTED_SYNC_DELIVERY_MODES = ['append', 'mirror'];
const SUPPORTED_SYNC_SCHEMA_MODES = ['external', 'managed'];
const SUPPORTED_SYNC_BIGQUERY_SCHEMA_CONFIG_MODES = ['explicit', 'inferred'];
const SUPPORTED_SYNC_BIGQUERY_SCHEMA_FIELD_MODES = ['nullable', 'repeated', 'required'];
const SUPPORTED_SYNC_DELETE_POLICIES = ['emit-delete-event', 'hard-delete', 'ignore', 'soft-delete'];
const SUPPORTED_SYNC_BIGQUERY_SCHEMA_FIELD_TYPES = ['bignumeric', 'bool', 'boolean', 'bytes', 'date', 'datetime', 'float', 'float64', 'geography', 'int64', 'integer', 'json', 'numeric', 'string', 'time', 'timestamp'];
const SUPPORTED_HTTP_METHODS = ['get', 'post'];
const SUPPORTED_BIGQUERY_WRITE_APIS = ['storage-write'];
const SUPPORTED_SQL_INCREMENTAL_DELETE_STRATEGIES = ['none', 'soft-delete'];
const SUPPORTED_CLOUD_RUN_VPC_EGRESS_SETTINGS = ['all-traffic', 'private-ranges-only'];
const BASE_SYNC_ROOT_SECTION_KEYS = ['projects', 'version', 'workloads'];
const SYNC_PROJECT_SECTION_KEYS = ['deploy', 'destinationBindings', 'environment', 'sourceBindings', 'workloadBindings'];
const SYNC_WORKLOAD_FILE_KEYS = ['destination', 'mapper', 'source', 'workload'];
const SYNC_WORKLOAD_KEYS = ['deletePolicy', 'failureMode', 'name'];
const SYNC_WORKLOAD_MAPPER_KEYS = ['export', 'name', 'source'];
const SYNC_WORKLOAD_SOURCE_KEYS = ['description', 'firestore', 'http', 'name', 'sql', 'syncClass', 'type'];
const SYNC_FIRESTORE_SOURCE_KEYS = ['documentPathPattern'];
const SYNC_HTTP_SOURCE_KEYS = ['body', 'headers', 'incremental', 'method'];
const SYNC_HTTP_INCREMENTAL_KEYS = ['cursor', 'mapping', 'response', 'strategy'];
const SYNC_HTTP_CURSOR_KEYS = ['request', 'response'];
const SYNC_HTTP_CURSOR_REQUEST_KEYS = ['bodyPath', 'headerName', 'queryParam'];
const SYNC_HTTP_CURSOR_RESPONSE_KEYS = ['hasMorePath', 'nextCursorPath'];
const SYNC_HTTP_RESPONSE_KEYS = ['itemsPath'];
const SYNC_HTTP_MAPPING_KEYS = ['afterPath', 'beforePath', 'deletedPath', 'idPath', 'versionPath'];
const SYNC_SQL_SNAPSHOT_KEYS = ['query'];
const SYNC_SQL_CURSOR_KEYS = ['idColumn', 'versionColumn'];
const SYNC_SQL_SOURCE_KEYS = ['driver', 'incremental', 'snapshot'];
const SYNC_SQL_INCREMENTAL_KEYS = ['batchSize', 'cursor', 'deleteStrategy', 'deletedColumn', 'query', 'safetyLag', 'strategy'];
const SYNC_WORKLOAD_DESTINATION_KEYS = ['bigquery', 'deliveryMode', 'description', 'name', 'postgres', 'schemaMode', 'type'];
const SYNC_BIGQUERY_DESTINATION_KEYS = ['primaryKey', 'schema', 'writeApi'];
const SYNC_BIGQUERY_SCHEMA_KEYS = ['fields', 'mode', 'source'];
const SYNC_BIGQUERY_SCHEMA_FIELD_KEYS = ['description', 'mode', 'name', 'type'];
const SYNC_POSTGRES_DESTINATION_KEYS = ['deleteMode', 'primaryKey'];
const SYNC_SOURCE_BINDING_KEYS = ['firestore', 'http', 'schedule', 'sql'];
const SYNC_SOURCE_BINDING_FIRESTORE_KEYS = ['database', 'triggerRegion'];
const SYNC_SOURCE_BINDING_HTTP_KEYS = ['connectionSecret', 'url'];
const SYNC_SOURCE_BINDING_SQL_KEYS = ['connectionSecret'];
const SYNC_DESTINATION_BINDING_KEYS = ['bigquery', 'postgres'];
const SYNC_DESTINATION_BINDING_BIGQUERY_KEYS = ['dataset', 'table'];
const SYNC_DESTINATION_BINDING_POSTGRES_KEYS = ['connectionSecret', 'schema', 'table'];
const SYNC_WORKLOAD_BINDING_KEYS = ['batchSize', 'enabled'];
const SYNC_DEPLOY_KEYS = ['cloudRun', 'eventarc', 'syncState', 'taskQueue', 'terraform'];
const SYNC_DEPLOY_CLOUD_RUN_KEYS = ['artifactRegistryLocation', 'artifactRegistryProject', 'backfillJobImage', 'mapperManifestUri', 'region', 'repository', 'runtimeConfigUri', 'serviceAccountEmail', 'serviceImage', 'tag', 'vpcAccess'];
const SYNC_DEPLOY_CLOUD_RUN_VPC_ACCESS_KEYS = ['egress', 'network', 'subnetwork'];
const SYNC_DEPLOY_TASK_QUEUE_KEYS = ['location', 'queueName'];
const SYNC_DEPLOY_EVENTARC_KEYS = ['database', 'triggerRegion'];
const SYNC_DEPLOY_STATE_KEYS = ['collectionPath', 'recentLimit'];
const SYNC_DEPLOY_TERRAFORM_KEYS = ['moduleSource', 'rootDir'];
const RESOLVED_SYNC_PIPELINE_KEYS = ['destination', 'mapper', 'source'];
const RESOLVED_SYNC_CONFIG_KEYS = ['deploy', 'destinations', 'mappers', 'pipelines', 'projectId', 'sources', 'version', 'workloads'];
const throwSyncConfigError = message => {
throw new Error(`Invalid Atlas sync config. ${message}`);
};
const assertAllowedKeys = (value, propertyPath, allowedKeys) => {
if (!isPlainObject(value)) {
throwSyncConfigError(`The "${propertyPath}" property must be an object.`);
}
const invalidKeys = Object.keys(value).filter(key => !allowedKeys.includes(key));
if (invalidKeys.length > 0) {
throwSyncConfigError(`Unsupported properties found on "${propertyPath}": ${invalidKeys.join(', ')}.`);
}
};
const assertNonEmptyString = (value, propertyPath) => {
if (!isString(value) || value.trim().length === 0) {
throwSyncConfigError(`The "${propertyPath}" property must be a non-empty string.`);
}
};
const assertObjectWhenProvided = (value, propertyPath) => {
if (value !== undefined && !isPlainObject(value)) {
throwSyncConfigError(`The "${propertyPath}" property must be an object when provided.`);
}
};
const assertPositiveIntegerWhenProvided = (value, propertyPath) => {
if (value !== undefined && (!Number.isInteger(value) || value <= 0)) {
throwSyncConfigError(`The "${propertyPath}" property must be a positive integer when provided.`);
}
};
const assertStringArrayWhenProvided = (value, propertyPath) => {
if (value !== undefined && !isStringArray(value)) {
throwSyncConfigError(`The "${propertyPath}" property must be an array of strings when provided.`);
}
};
const assertStringRecordWhenProvided = (value, propertyPath) => {
if (value === undefined) {
return;
}
if (!isPlainObject(value)) {
throwSyncConfigError(`The "${propertyPath}" property must be an object when provided.`);
}
for (const [entryKey, entryValue] of Object.entries(value)) {
if (!isString(entryValue)) {
throwSyncConfigError(`The "${propertyPath}.${entryKey}" property must be a string when provided.`);
}
}
};
const assertStringWhenProvided = (value, propertyPath) => {
if (value !== undefined && !isString(value)) {
throwSyncConfigError(`The "${propertyPath}" property must be a string when provided.`);
}
};
const validateSyncBigQuerySchemaField = (fieldConfig, propertyPath) => {
assertAllowedKeys(fieldConfig, propertyPath, SYNC_BIGQUERY_SCHEMA_FIELD_KEYS);
assertNonEmptyString(fieldConfig.name, `${propertyPath}.name`);
assertNonEmptyString(fieldConfig.type, `${propertyPath}.type`);
assertAllowedStringValue(fieldConfig.type, `${propertyPath}.type`, SUPPORTED_SYNC_BIGQUERY_SCHEMA_FIELD_TYPES);
if (fieldConfig.mode !== undefined) {
assertAllowedStringValue(fieldConfig.mode, `${propertyPath}.mode`, SUPPORTED_SYNC_BIGQUERY_SCHEMA_FIELD_MODES);
}
assertStringWhenProvided(fieldConfig.description, `${propertyPath}.description`);
};
const validateSyncBigQuerySchemaConfig = schemaConfig => {
assertAllowedKeys(schemaConfig, 'destination.bigquery.schema', SYNC_BIGQUERY_SCHEMA_KEYS);
assertNonEmptyString(schemaConfig.mode, 'destination.bigquery.schema.mode');
assertAllowedStringValue(schemaConfig.mode, 'destination.bigquery.schema.mode', SUPPORTED_SYNC_BIGQUERY_SCHEMA_CONFIG_MODES);
const schemaMode = schemaConfig.mode.trim().toLowerCase();
if (schemaMode === 'explicit') {
if (!Array.isArray(schemaConfig.fields) || schemaConfig.fields.length === 0) {
throwSyncConfigError('The "destination.bigquery.schema.fields" property must be a non-empty array when "destination.bigquery.schema.mode" is "explicit".');
}
schemaConfig.fields.forEach((fieldConfig, index) => validateSyncBigQuerySchemaField(fieldConfig, `destination.bigquery.schema.fields[${index}]`));
if (schemaConfig.source !== undefined) {
throwSyncConfigError('The "destination.bigquery.schema.source" property is only supported when "destination.bigquery.schema.mode" is "inferred".');
}
return;
}
if (!isString(schemaConfig.source) || schemaConfig.source.trim().length === 0) {
throwSyncConfigError('The "destination.bigquery.schema.source" property must be a non-empty string when "destination.bigquery.schema.mode" is "inferred".');
}
if (schemaConfig.fields !== undefined) {
throwSyncConfigError('The "destination.bigquery.schema.fields" property is only supported when "destination.bigquery.schema.mode" is "explicit".');
}
};
const assertStringOrNullWhenProvided = (value, propertyPath) => {
if (value !== undefined && value !== null && !isString(value)) {
throwSyncConfigError(`The "${propertyPath}" property must be a string or null when provided.`);
}
};
const assertAllowedStringValue = (value, propertyPath, allowedValues) => {
if (value === undefined) {
return;
}
if (!isString(value) || !allowedValues.includes(value.trim().toLowerCase())) {
throwSyncConfigError(`The "${propertyPath}" property must be one of: ${allowedValues.join(', ')}.`);
}
};
const validateSyncDeployConfig = deployConfig => {
assertAllowedKeys(deployConfig, 'deploy', SYNC_DEPLOY_KEYS);
if (deployConfig.cloudRun !== undefined) {
assertAllowedKeys(deployConfig.cloudRun, 'deploy.cloudRun', SYNC_DEPLOY_CLOUD_RUN_KEYS);
assertStringWhenProvided(deployConfig.cloudRun.artifactRegistryLocation, 'deploy.cloudRun.artifactRegistryLocation');
assertStringWhenProvided(deployConfig.cloudRun.artifactRegistryProject, 'deploy.cloudRun.artifactRegistryProject');
assertStringWhenProvided(deployConfig.cloudRun.region, 'deploy.cloudRun.region');
assertStringWhenProvided(deployConfig.cloudRun.repository, 'deploy.cloudRun.repository');
assertStringWhenProvided(deployConfig.cloudRun.serviceAccountEmail, 'deploy.cloudRun.serviceAccountEmail');
assertStringWhenProvided(deployConfig.cloudRun.backfillJobImage, 'deploy.cloudRun.backfillJobImage');
assertStringOrNullWhenProvided(deployConfig.cloudRun.mapperManifestUri, 'deploy.cloudRun.mapperManifestUri');
assertStringOrNullWhenProvided(deployConfig.cloudRun.runtimeConfigUri, 'deploy.cloudRun.runtimeConfigUri');
assertStringWhenProvided(deployConfig.cloudRun.serviceImage, 'deploy.cloudRun.serviceImage');
assertStringWhenProvided(deployConfig.cloudRun.tag, 'deploy.cloudRun.tag');
if (deployConfig.cloudRun.vpcAccess !== undefined) {
assertAllowedKeys(deployConfig.cloudRun.vpcAccess, 'deploy.cloudRun.vpcAccess', SYNC_DEPLOY_CLOUD_RUN_VPC_ACCESS_KEYS);
assertStringWhenProvided(deployConfig.cloudRun.vpcAccess.network, 'deploy.cloudRun.vpcAccess.network');
assertStringOrNullWhenProvided(deployConfig.cloudRun.vpcAccess.subnetwork, 'deploy.cloudRun.vpcAccess.subnetwork');
if (deployConfig.cloudRun.vpcAccess.egress !== undefined && !SUPPORTED_CLOUD_RUN_VPC_EGRESS_SETTINGS.includes(deployConfig.cloudRun.vpcAccess.egress)) {
throwSyncConfigError('The "deploy.cloudRun.vpcAccess.egress" property must be "all-traffic" or "private-ranges-only" when provided.');
}
}
}
if (deployConfig.taskQueue !== undefined) {
assertAllowedKeys(deployConfig.taskQueue, 'deploy.taskQueue', SYNC_DEPLOY_TASK_QUEUE_KEYS);
assertStringWhenProvided(deployConfig.taskQueue.location, 'deploy.taskQueue.location');
assertStringWhenProvided(deployConfig.taskQueue.queueName, 'deploy.taskQueue.queueName');
}
if (deployConfig.eventarc !== undefined) {
assertAllowedKeys(deployConfig.eventarc, 'deploy.eventarc', SYNC_DEPLOY_EVENTARC_KEYS);
assertStringOrNullWhenProvided(deployConfig.eventarc.database, 'deploy.eventarc.database');
assertStringWhenProvided(deployConfig.eventarc.triggerRegion, 'deploy.eventarc.triggerRegion');
}
if (deployConfig.syncState !== undefined) {
assertAllowedKeys(deployConfig.syncState, 'deploy.syncState', SYNC_DEPLOY_STATE_KEYS);
assertStringOrNullWhenProvided(deployConfig.syncState.collectionPath, 'deploy.syncState.collectionPath');
assertPositiveIntegerWhenProvided(deployConfig.syncState.recentLimit, 'deploy.syncState.recentLimit');
}
if (deployConfig.terraform !== undefined) {
assertAllowedKeys(deployConfig.terraform, 'deploy.terraform', SYNC_DEPLOY_TERRAFORM_KEYS);
assertStringWhenProvided(deployConfig.terraform.moduleSource, 'deploy.terraform.moduleSource');
assertStringWhenProvided(deployConfig.terraform.rootDir, 'deploy.terraform.rootDir');
}
};
const validateSyncWorkloadBlock = workloadConfig => {
assertAllowedKeys(workloadConfig, 'workload', SYNC_WORKLOAD_KEYS);
assertNonEmptyString(workloadConfig.name, 'workload.name');
assertAllowedStringValue(workloadConfig.failureMode, 'workload.failureMode', SUPPORTED_SYNC_WORKLOAD_FAILURE_MODES);
assertAllowedStringValue(workloadConfig.deletePolicy, 'workload.deletePolicy', SUPPORTED_SYNC_DELETE_POLICIES);
if (workloadConfig.failureMode === undefined) {
throwSyncConfigError('The "workload.failureMode" property is required.');
}
if (workloadConfig.deletePolicy === undefined) {
throwSyncConfigError('The "workload.deletePolicy" property is required.');
}
};
const validateSyncMapperConfig = mapperConfig => {
assertAllowedKeys(mapperConfig, 'mapper', SYNC_WORKLOAD_MAPPER_KEYS);
assertNonEmptyString(mapperConfig.export, 'mapper.export');
assertNonEmptyString(mapperConfig.name, 'mapper.name');
assertNonEmptyString(mapperConfig.source, 'mapper.source');
};
const validateFirestoreSyncSourceConfig = sourceConfig => {
assertAllowedKeys(sourceConfig, 'source', ['description', 'firestore', 'name', 'syncClass', 'type']);
assertAllowedStringValue(sourceConfig.syncClass, 'source.syncClass', SUPPORTED_SYNC_SOURCE_SYNC_CLASSES);
if (sourceConfig.syncClass?.trim().toLowerCase() !== 'delta-merge') {
throwSyncConfigError('The "source.syncClass" property must be "delta-merge" when source.type is "firestore".');
}
if (!isPlainObject(sourceConfig.firestore)) {
throwSyncConfigError('The "source.firestore" property is required for firestore sources.');
}
assertAllowedKeys(sourceConfig.firestore, 'source.firestore', SYNC_FIRESTORE_SOURCE_KEYS);
assertNonEmptyString(sourceConfig.firestore.documentPathPattern, 'source.firestore.documentPathPattern');
};
const validateHttpSyncSourceConfig = sourceConfig => {
assertAllowedKeys(sourceConfig, 'source', ['description', 'http', 'name', 'syncClass', 'type']);
assertAllowedStringValue(sourceConfig.syncClass, 'source.syncClass', SUPPORTED_SYNC_SOURCE_SYNC_CLASSES);
const normalizedSyncClass = sourceConfig.syncClass.trim().toLowerCase();
if (normalizedSyncClass !== 'append-only' && normalizedSyncClass !== 'delta-merge') {
throwSyncConfigError('The "source.syncClass" property must be "append-only" or "delta-merge" when source.type is "http".');
}
if (!isPlainObject(sourceConfig.http)) {
throwSyncConfigError('The "source.http" property is required for http sources.');
}
assertAllowedKeys(sourceConfig.http, 'source.http', SYNC_HTTP_SOURCE_KEYS);
assertAllowedStringValue(sourceConfig.http.method, 'source.http.method', SUPPORTED_HTTP_METHODS);
assertStringRecordWhenProvided(sourceConfig.http.headers, 'source.http.headers');
if (sourceConfig.http.body !== undefined && !isPlainObject(sourceConfig.http.body)) {
throwSyncConfigError('The "source.http.body" property must be an object when provided.');
}
if (!isPlainObject(sourceConfig.http.incremental)) {
throwSyncConfigError('The "source.http.incremental" property is required for http sources.');
}
assertAllowedKeys(sourceConfig.http.incremental, 'source.http.incremental', SYNC_HTTP_INCREMENTAL_KEYS);
if (sourceConfig.http.incremental.strategy?.trim().toLowerCase() !== 'cursor') {
throwSyncConfigError('The "source.http.incremental.strategy" property must be "cursor" for http sources.');
}
if (!isPlainObject(sourceConfig.http.incremental.cursor)) {
throwSyncConfigError('The "source.http.incremental.cursor" property is required for http sources.');
}
assertAllowedKeys(sourceConfig.http.incremental.cursor, 'source.http.incremental.cursor', SYNC_HTTP_CURSOR_KEYS);
if (!isPlainObject(sourceConfig.http.incremental.cursor.request)) {
throwSyncConfigError('The "source.http.incremental.cursor.request" property is required for http sources.');
}
assertAllowedKeys(sourceConfig.http.incremental.cursor.request, 'source.http.incremental.cursor.request', SYNC_HTTP_CURSOR_REQUEST_KEYS);
const requestTransportCount = [sourceConfig.http.incremental.cursor.request.queryParam, sourceConfig.http.incremental.cursor.request.headerName, sourceConfig.http.incremental.cursor.request.bodyPath].filter(Boolean).length;
if (requestTransportCount !== 1) {
throwSyncConfigError('The "source.http.incremental.cursor.request" block must define exactly one of "queryParam", "headerName", or "bodyPath".');
}
assertStringWhenProvided(sourceConfig.http.incremental.cursor.request.queryParam, 'source.http.incremental.cursor.request.queryParam');
assertStringWhenProvided(sourceConfig.http.incremental.cursor.request.headerName, 'source.http.incremental.cursor.request.headerName');
assertStringWhenProvided(sourceConfig.http.incremental.cursor.request.bodyPath, 'source.http.incremental.cursor.request.bodyPath');
if (!isPlainObject(sourceConfig.http.incremental.cursor.response)) {
throwSyncConfigError('The "source.http.incremental.cursor.response" property is required for http sources.');
}
assertAllowedKeys(sourceConfig.http.incremental.cursor.response, 'source.http.incremental.cursor.response', SYNC_HTTP_CURSOR_RESPONSE_KEYS);
assertNonEmptyString(sourceConfig.http.incremental.cursor.response.hasMorePath, 'source.http.incremental.cursor.response.hasMorePath');
assertNonEmptyString(sourceConfig.http.incremental.cursor.response.nextCursorPath, 'source.http.incremental.cursor.response.nextCursorPath');
if (!isPlainObject(sourceConfig.http.incremental.response)) {
throwSyncConfigError('The "source.http.incremental.response" property is required for http sources.');
}
assertAllowedKeys(sourceConfig.http.incremental.response, 'source.http.incremental.response', SYNC_HTTP_RESPONSE_KEYS);
assertNonEmptyString(sourceConfig.http.incremental.response.itemsPath, 'source.http.incremental.response.itemsPath');
if (!isPlainObject(sourceConfig.http.incremental.mapping)) {
throwSyncConfigError('The "source.http.incremental.mapping" property is required for http sources.');
}
assertAllowedKeys(sourceConfig.http.incremental.mapping, 'source.http.incremental.mapping', SYNC_HTTP_MAPPING_KEYS);
assertNonEmptyString(sourceConfig.http.incremental.mapping.idPath, 'source.http.incremental.mapping.idPath');
assertNonEmptyString(sourceConfig.http.incremental.mapping.versionPath, 'source.http.incremental.mapping.versionPath');
assertNonEmptyString(sourceConfig.http.incremental.mapping.afterPath, 'source.http.incremental.mapping.afterPath');
assertStringWhenProvided(sourceConfig.http.incremental.mapping.beforePath, 'source.http.incremental.mapping.beforePath');
assertStringWhenProvided(sourceConfig.http.incremental.mapping.deletedPath, 'source.http.incremental.mapping.deletedPath');
if (normalizedSyncClass === 'append-only') {
if (sourceConfig.http.incremental.mapping.beforePath !== undefined) {
throwSyncConfigError('The "source.http.incremental.mapping.beforePath" property is not supported when source.syncClass is "append-only".');
}
if (sourceConfig.http.incremental.mapping.deletedPath !== undefined) {
throwSyncConfigError('The "source.http.incremental.mapping.deletedPath" property is not supported when source.syncClass is "append-only".');
}
}
};
const validateSqlSyncSourceConfig = sourceConfig => {
assertAllowedKeys(sourceConfig, 'source', ['description', 'name', 'sql', 'syncClass', 'type']);
assertAllowedStringValue(sourceConfig.syncClass, 'source.syncClass', SUPPORTED_SYNC_SOURCE_SYNC_CLASSES);
const normalizedSyncClass = sourceConfig.syncClass.trim().toLowerCase();
if (!isPlainObject(sourceConfig.sql)) {
throwSyncConfigError('The "source.sql" property is required for sql sources.');
}
assertAllowedKeys(sourceConfig.sql, 'source.sql', SYNC_SQL_SOURCE_KEYS);
assertNonEmptyString(sourceConfig.sql.driver, 'source.sql.driver');
if (normalizedSyncClass === 'snapshot-replace') {
if (sourceConfig.sql.incremental !== undefined) {
throwSyncConfigError('The "source.sql.incremental" property is not supported when source.syncClass is "snapshot-replace".');
}
if (!isPlainObject(sourceConfig.sql.snapshot)) {
throwSyncConfigError('The "source.sql.snapshot" property is required when source.syncClass is "snapshot-replace".');
}
assertAllowedKeys(sourceConfig.sql.snapshot, 'source.sql.snapshot', SYNC_SQL_SNAPSHOT_KEYS);
assertNonEmptyString(sourceConfig.sql.snapshot.query, 'source.sql.snapshot.query');
return;
}
if (!isPlainObject(sourceConfig.sql.incremental)) {
throwSyncConfigError('The "source.sql.incremental" property is required unless source.syncClass is "snapshot-replace".');
}
if (sourceConfig.sql.snapshot !== undefined) {
throwSyncConfigError('The "source.sql.snapshot" property is only supported when source.syncClass is "snapshot-replace".');
}
assertAllowedKeys(sourceConfig.sql.incremental, 'source.sql.incremental', SYNC_SQL_INCREMENTAL_KEYS);
if (sourceConfig.sql.incremental.strategy?.trim().toLowerCase() !== 'watermark') {
throwSyncConfigError('The "source.sql.incremental.strategy" property must be "watermark" for sql sources.');
}
assertPositiveIntegerWhenProvided(sourceConfig.sql.incremental.batchSize, 'source.sql.incremental.batchSize');
assertStringWhenProvided(sourceConfig.sql.incremental.deletedColumn, 'source.sql.incremental.deletedColumn');
assertStringWhenProvided(sourceConfig.sql.incremental.safetyLag, 'source.sql.incremental.safetyLag');
assertNonEmptyString(sourceConfig.sql.incremental.query, 'source.sql.incremental.query');
if (!isPlainObject(sourceConfig.sql.incremental.cursor)) {
throwSyncConfigError('The "source.sql.incremental.cursor" property is required for incremental sql sources.');
}
assertAllowedKeys(sourceConfig.sql.incremental.cursor, 'source.sql.incremental.cursor', SYNC_SQL_CURSOR_KEYS);
assertNonEmptyString(sourceConfig.sql.incremental.cursor.idColumn, 'source.sql.incremental.cursor.idColumn');
assertNonEmptyString(sourceConfig.sql.incremental.cursor.versionColumn, 'source.sql.incremental.cursor.versionColumn');
assertAllowedStringValue(sourceConfig.sql.incremental.deleteStrategy, 'source.sql.incremental.deleteStrategy', SUPPORTED_SQL_INCREMENTAL_DELETE_STRATEGIES);
if (normalizedSyncClass === 'append-only') {
if (sourceConfig.sql.incremental.deleteStrategy !== undefined) {
throwSyncConfigError('The "source.sql.incremental.deleteStrategy" property is not supported when source.syncClass is "append-only".');
}
if (sourceConfig.sql.incremental.deletedColumn !== undefined) {
throwSyncConfigError('The "source.sql.incremental.deletedColumn" property is not supported when source.syncClass is "append-only".');
}
return;
}
if (sourceConfig.sql.incremental.deleteStrategy?.trim().toLowerCase() === 'soft-delete' && !isString(sourceConfig.sql.incremental.deletedColumn)) {
throwSyncConfigError('The "source.sql.incremental.deletedColumn" property is required when deleteStrategy is "soft-delete".');
}
};
const validateSyncSourceConfig = sourceConfig => {
assertAllowedKeys(sourceConfig, 'source', SYNC_WORKLOAD_SOURCE_KEYS);
assertNonEmptyString(sourceConfig.name, 'source.name');
assertNonEmptyString(sourceConfig.type, 'source.type');
assertAllowedStringValue(sourceConfig.type, 'source.type', SYNC_SOURCE_TYPES);
assertStringWhenProvided(sourceConfig.description, 'source.description');
const sourceType = sourceConfig.type.trim().toLowerCase();
if (sourceType === 'firestore') {
validateFirestoreSyncSourceConfig(sourceConfig);
return;
}
if (sourceType === 'http') {
validateHttpSyncSourceConfig(sourceConfig);
return;
}
validateSqlSyncSourceConfig(sourceConfig);
};
const validateBigQueryDestinationConfig = destinationConfig => {
assertAllowedKeys(destinationConfig, 'destination', ['bigquery', 'deliveryMode', 'description', 'name', 'schemaMode', 'type']);
assertAllowedStringValue(destinationConfig.deliveryMode, 'destination.deliveryMode', SUPPORTED_SYNC_DELIVERY_MODES);
assertAllowedStringValue(destinationConfig.schemaMode, 'destination.schemaMode', SUPPORTED_SYNC_SCHEMA_MODES);
if (!isPlainObject(destinationConfig.bigquery)) {
throwSyncConfigError('The "destination.bigquery" property is required for bigquery destinations.');
}
assertAllowedKeys(destinationConfig.bigquery, 'destination.bigquery', SYNC_BIGQUERY_DESTINATION_KEYS);
assertAllowedStringValue(destinationConfig.bigquery.writeApi, 'destination.bigquery.writeApi', SUPPORTED_BIGQUERY_WRITE_APIS);
assertStringArrayWhenProvided(destinationConfig.bigquery.primaryKey, 'destination.bigquery.primaryKey');
if (destinationConfig.bigquery.schema !== undefined) {
validateSyncBigQuerySchemaConfig(destinationConfig.bigquery.schema);
}
if (destinationConfig.schemaMode?.trim().toLowerCase() === 'managed' && normalizeOptionalString(destinationConfig.bigquery.schema?.mode)?.toLowerCase() !== 'explicit') {
throwSyncConfigError('Managed BigQuery destinations must define an explicit schema via "destination.bigquery.schema.mode = \"explicit\"".');
}
};
const validatePostgresDestinationConfig = destinationConfig => {
assertAllowedKeys(destinationConfig, 'destination', ['deliveryMode', 'description', 'name', 'postgres', 'schemaMode', 'type']);
assertAllowedStringValue(destinationConfig.deliveryMode, 'destination.deliveryMode', SUPPORTED_SYNC_DELIVERY_MODES);
assertAllowedStringValue(destinationConfig.schemaMode, 'destination.schemaMode', SUPPORTED_SYNC_SCHEMA_MODES);
if (destinationConfig.deliveryMode?.trim().toLowerCase() !== 'mirror') {
throwSyncConfigError('The "destination.deliveryMode" property must be "mirror" for postgres destinations.');
}
if (!isPlainObject(destinationConfig.postgres)) {
throwSyncConfigError('The "destination.postgres" property is required for postgres destinations.');
}
assertAllowedKeys(destinationConfig.postgres, 'destination.postgres', SYNC_POSTGRES_DESTINATION_KEYS);
assertStringArrayWhenProvided(destinationConfig.postgres.primaryKey, 'destination.postgres.primaryKey');
assertAllowedStringValue(destinationConfig.postgres.deleteMode, 'destination.postgres.deleteMode', ['hard-delete', 'soft-delete']);
if (!isStringArray(destinationConfig.postgres.primaryKey) || destinationConfig.postgres.primaryKey.length === 0) {
throwSyncConfigError('The "destination.postgres.primaryKey" property must be a non-empty array of strings.');
}
};
const validateSyncDestinationConfig = destinationConfig => {
assertAllowedKeys(destinationConfig, 'destination', SYNC_WORKLOAD_DESTINATION_KEYS);
assertNonEmptyString(destinationConfig.name, 'destination.name');
assertNonEmptyString(destinationConfig.type, 'destination.type');
assertAllowedStringValue(destinationConfig.type, 'destination.type', SYNC_DESTINATION_TYPES);
assertNonEmptyString(destinationConfig.deliveryMode, 'destination.deliveryMode');
assertNonEmptyString(destinationConfig.schemaMode, 'destination.schemaMode');
assertStringWhenProvided(destinationConfig.description, 'destination.description');
const destinationType = destinationConfig.type.trim().toLowerCase();
if (destinationType === 'bigquery') {
validateBigQueryDestinationConfig(destinationConfig);
return;
}
validatePostgresDestinationConfig(destinationConfig);
};
export const validateSyncWorkloadConfig = (config, sourceLabel = 'Atlas sync workload') => {
if (!isPlainObject(config)) {
throwSyncConfigError(`The ${sourceLabel} must be an object.`);
}
assertAllowedKeys(config, sourceLabel, SYNC_WORKLOAD_FILE_KEYS);
for (const requiredKey of SYNC_WORKLOAD_FILE_KEYS) {
if (!Object.hasOwn(config, requiredKey)) {
throwSyncConfigError(`The "${sourceLabel}.${requiredKey}" property is required.`);
}
}
validateSyncWorkloadBlock(config.workload);
validateSyncSourceConfig(config.source);
validateSyncMapperConfig(config.mapper);
validateSyncDestinationConfig(config.destination);
return config;
};
export const validateSyncSourceBindingConfig = (sourceName, sourceBinding) => {
if (!isPlainObject(sourceBinding)) {
throwSyncConfigError(`The "projects.<projectId>.sourceBindings.${sourceName}" entry must be an object.`);
}
assertAllowedKeys(sourceBinding, `projects.<projectId>.sourceBindings.${sourceName}`, SYNC_SOURCE_BINDING_KEYS);
const configuredBindingTypes = SYNC_SOURCE_BINDING_TYPES.filter(bindingType => sourceBinding[bindingType] !== undefined);
if (configuredBindingTypes.length !== 1) {
throwSyncConfigError(`The "projects.<projectId>.sourceBindings.${sourceName}" entry must define exactly one type-specific binding via "firestore", "http", or "sql".`);
}
assertStringWhenProvided(sourceBinding.schedule, `projects.<projectId>.sourceBindings.${sourceName}.schedule`);
if (sourceBinding.firestore !== undefined) {
assertAllowedKeys(sourceBinding.firestore, `projects.<projectId>.sourceBindings.${sourceName}.firestore`, SYNC_SOURCE_BINDING_FIRESTORE_KEYS);
assertStringWhenProvided(sourceBinding.firestore.database, `projects.<projectId>.sourceBindings.${sourceName}.firestore.database`);
assertStringWhenProvided(sourceBinding.firestore.triggerRegion, `projects.<projectId>.sourceBindings.${sourceName}.firestore.triggerRegion`);
}
if (sourceBinding.http !== undefined) {
assertAllowedKeys(sourceBinding.http, `projects.<projectId>.sourceBindings.${sourceName}.http`, SYNC_SOURCE_BINDING_HTTP_KEYS);
assertNonEmptyString(sourceBinding.http.url, `projects.<projectId>.sourceBindings.${sourceName}.http.url`);
assertStringWhenProvided(sourceBinding.http.connectionSecret, `projects.<projectId>.sourceBindings.${sourceName}.http.connectionSecret`);
}
if (sourceBinding.sql !== undefined) {
assertAllowedKeys(sourceBinding.sql, `projects.<projectId>.sourceBindings.${sourceName}.sql`, SYNC_SOURCE_BINDING_SQL_KEYS);
assertNonEmptyString(sourceBinding.sql.connectionSecret, `projects.<projectId>.sourceBindings.${sourceName}.sql.connectionSecret`);
}
return sourceBinding;
};
export const validateSyncDestinationBindingConfig = (destinationName, destinationBinding) => {
if (!isPlainObject(destinationBinding)) {
throwSyncConfigError(`The "projects.<projectId>.destinationBindings.${destinationName}" entry must be an object.`);
}
assertAllowedKeys(destinationBinding, `projects.<projectId>.destinationBindings.${destinationName}`, SYNC_DESTINATION_BINDING_KEYS);
const configuredBindingTypes = SYNC_DESTINATION_BINDING_TYPES.filter(bindingType => destinationBinding[bindingType] !== undefined);
if (configuredBindingTypes.length !== 1) {
throwSyncConfigError(`The "projects.<projectId>.destinationBindings.${destinationName}" entry must define exactly one type-specific binding via "bigquery" or "postgres".`);
}
if (destinationBinding.bigquery !== undefined) {
assertAllowedKeys(destinationBinding.bigquery, `projects.<projectId>.destinationBindings.${destinationName}.bigquery`, SYNC_DESTINATION_BINDING_BIGQUERY_KEYS);
assertNonEmptyString(destinationBinding.bigquery.dataset, `projects.<projectId>.destinationBindings.${destinationName}.bigquery.dataset`);
assertNonEmptyString(destinationBinding.bigquery.table, `projects.<projectId>.destinationBindings.${destinationName}.bigquery.table`);
}
if (destinationBinding.postgres !== undefined) {
assertAllowedKeys(destinationBinding.postgres, `projects.<projectId>.destinationBindings.${destinationName}.postgres`, SYNC_DESTINATION_BINDING_POSTGRES_KEYS);
assertNonEmptyString(destinationBinding.postgres.connectionSecret, `projects.<projectId>.destinationBindings.${destinationName}.postgres.connectionSecret`);
assertNonEmptyString(destinationBinding.postgres.schema, `projects.<projectId>.destinationBindings.${destinationName}.postgres.schema`);
assertNonEmptyString(destinationBinding.postgres.table, `projects.<projectId>.destinationBindings.${destinationName}.postgres.table`);
}
return destinationBinding;
};
export const validateSyncWorkloadBindingConfig = (workloadName, workloadBinding) => {
if (!isPlainObject(workloadBinding)) {
throwSyncConfigError(`The "projects.<projectId>.workloadBindings.${workloadName}" entry must be an object.`);
}
assertAllowedKeys(workloadBinding, `projects.<projectId>.workloadBindings.${workloadName}`, SYNC_WORKLOAD_BINDING_KEYS);
if (!isBoolean(workloadBinding.enabled)) {
throwSyncConfigError(`The "projects.<projectId>.workloadBindings.${workloadName}.enabled" property must be a boolean.`);
}
assertPositiveIntegerWhenProvided(workloadBinding.batchSize, `projects.<projectId>.workloadBindings.${workloadName}.batchSize`);
return workloadBinding;
};
export const validateSyncProjectSection = (projectConfig, projectId = '<projectId>') => {
if (!isPlainObject(projectConfig)) {
throwSyncConfigError(`The "projects.${projectId}" entry must be an object.`);
}
assertAllowedKeys(projectConfig, `projects.${projectId}`, SYNC_PROJECT_SECTION_KEYS);
assertStringWhenProvided(projectConfig.environment, `projects.${projectId}.environment`);
if (projectConfig.deploy !== undefined) {
validateSyncDeployConfig(projectConfig.deploy);
}
if (projectConfig.sourceBindings !== undefined) {
if (!isPlainObject(projectConfig.sourceBindings)) {
throwSyncConfigError(`The "projects.${projectId}.sourceBindings" property must be an object.`);
}
for (const [sourceName, sourceBinding] of Object.entries(projectConfig.sourceBindings)) {
validateSyncSourceBindingConfig(sourceName, sourceBinding);
}
}
if (projectConfig.destinationBindings !== undefined) {
if (!isPlainObject(projectConfig.destinationBindings)) {
throwSyncConfigError(`The "projects.${projectId}.destinationBindings" property must be an object.`);
}
for (const [destinationName, destinationBinding] of Object.entries(projectConfig.destinationBindings)) {
validateSyncDestinationBindingConfig(destinationName, destinationBinding);
}
}
if (projectConfig.workloadBindings !== undefined) {
if (!isPlainObject(projectConfig.workloadBindings)) {
throwSyncConfigError(`The "projects.${projectId}.workloadBindings" property must be an object.`);
}
for (const [workloadName, workloadBinding] of Object.entries(projectConfig.workloadBindings)) {
validateSyncWorkloadBindingConfig(workloadName, workloadBinding);
}
}
return projectConfig;
};
export const validateSyncRootSection = config => {
if (!isPlainObject(config)) {
throwSyncConfigError('The root sync config must be an object.');
}
assertAllowedKeys(config, 'sync', BASE_SYNC_ROOT_SECTION_KEYS);
if (!Number.isInteger(config.version) || config.version <= 0) {
throwSyncConfigError('The "version" property must be a positive integer.');
}
if (!isStringArray(config.workloads) || config.workloads.length === 0) {
throwSyncConfigError('The "workloads" property must be a non-empty array of strings.');
}
if (config.projects !== undefined) {
if (!isPlainObject(config.projects)) {
throwSyncConfigError('The "projects" property must be an object when provided.');
}
for (const [projectId, projectConfig] of Object.entries(config.projects)) {
validateSyncProjectSection(projectConfig, projectId);
}
}
return config;
};
export const validateResolvedSyncConfig = config => {
if (!isPlainObject(config)) {
throwSyncConfigError('The resolved sync config must be an object.');
}
assertAllowedKeys(config, 'resolvedSyncConfig', RESOLVED_SYNC_CONFIG_KEYS);
if (!Number.isInteger(config.version) || config.version <= 0) {
throwSyncConfigError('The "resolvedSyncConfig.version" property must be a positive integer.');
}
if (config.projectId !== null && config.projectId !== undefined) {
assertNonEmptyString(config.projectId, 'resolvedSyncConfig.projectId');
}
assertObjectWhenProvided(config.deploy, 'resolvedSyncConfig.deploy');
assertObjectWhenProvided(config.workloads, 'resolvedSyncConfig.workloads');
assertObjectWhenProvided(config.sources, 'resolvedSyncConfig.sources');
assertObjectWhenProvided(config.mappers, 'resolvedSyncConfig.mappers');
assertObjectWhenProvided(config.destinations, 'resolvedSyncConfig.destinations');
assertObjectWhenProvided(config.pipelines, 'resolvedSyncConfig.pipelines');
for (const [pipelineName, pipelineConfig] of Object.entries(config.pipelines ?? {})) {
assertAllowedKeys(pipelineConfig, `resolvedSyncConfig.pipelines.${pipelineName}`, RESOLVED_SYNC_PIPELINE_KEYS);
assertNonEmptyString(pipelineConfig.source, `resolvedSyncConfig.pipelines.${pipelineName}.source`);
assertNonEmptyString(pipelineConfig.mapper, `resolvedSyncConfig.pipelines.${pipelineName}.mapper`);
assertNonEmptyString(pipelineConfig.destination, `resolvedSyncConfig.pipelines.${pipelineName}.destination`);
if (!Object.hasOwn(config.sources ?? {}, pipelineConfig.source)) {
throwSyncConfigError(`The "resolvedSyncConfig.pipelines.${pipelineName}.source" property references an unknown source.`);
}
if (!Object.hasOwn(config.mappers ?? {}, pipelineConfig.mapper)) {
throwSyncConfigError(`The "resolvedSyncConfig.pipelines.${pipelineName}.mapper" property references an unknown mapper.`);
}
if (!Object.hasOwn(config.destinations ?? {}, pipelineConfig.destination)) {
throwSyncConfigError(`The "resolvedSyncConfig.pipelines.${pipelineName}.destination" property references an unknown destination.`);
}
}
return config;
};
export default {
validateResolvedSyncConfig,
validateSyncDestinationBindingConfig,
validateSyncProjectSection,
validateSyncRootSection,
validateSyncSourceBindingConfig,
validateSyncWorkloadBindingConfig,
validateSyncWorkloadConfig
};