@puls-atlas/cli
Version:
The Puls Atlas CLI tool for managing Atlas projects
246 lines • 9.43 kB
JavaScript
import path from 'path';
import { getAtlasGeneratedTerraformDir } from '../../utils/atlas.js';
import { writeJsonFile } from '../../utils/file.js';
import { normalizeOptionalString } from '../../utils/value.js';
import { resolveSyncArtifactBucketName, resolveSyncDlqBucketName } from './artifactBucket.js';
import { DEFAULT_SYNC_TASK_QUEUE_NAME } from './config/syncConfig.js';
import { resolveSyncCloudRunDeployConfig } from './deploymentConfig.js';
import { SYNC_BACKFILL_JOB_NAME, SYNC_EVENTARC_SERVICE_ACCOUNT_ID, SYNC_SERVICE_NAME } from './resourceNames.js';
export const SYNC_TERRAFORM_CONTRACT_VERSION = 2;
export const SYNC_TERRAFORM_INPUT_VARIABLE = 'atlas_sync_tfvars_path';
const SYNC_CLOUD_RUN_SUBPAYLOAD_CONTRACT_VERSION = 1;
const SYNC_EVENTARC_DATABASE_DEFAULT = '(default)';
const SYNC_EVENTARC_EVENT_TYPE = 'google.cloud.firestore.document.v1.written';
const SYNC_EVENTARC_EVENT_DATA_CONTENT_TYPE = 'application/protobuf';
const SYNC_EVENTARC_SERVICE_PATH_PREFIX = '/sync';
const SYNC_DLQ_BUCKET_RUNTIME_ROLE = 'roles/storage.objectAdmin';
const SYNC_ARTIFACT_BUCKET_RUNTIME_ROLE = 'roles/storage.objectViewer';
const SYNC_RUNTIME_SECRET_ROLE = 'roles/secretmanager.secretAccessor';
const SYNC_DEFAULT_BACKFILL_EXECUTION = {
maxRetries: 0,
parallelism: 1,
taskCount: 1,
timeout: '3600s'
};
const normalizeLabelValue = value => {
const normalized = normalizeOptionalString(value);
if (!normalized) {
return null;
}
const normalizedValue = normalized.toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/-+/g, '-').replace(/^-+|-+$/g, '').slice(0, 63).replace(/-+$/g, '');
return normalizedValue.length > 0 ? normalizedValue : null;
};
const sortObjectKeys = objectValue => Object.fromEntries(Object.entries(objectValue).sort(([leftKey], [rightKey]) => leftKey.localeCompare(rightKey)));
const createSyncTerraformLabels = (projectId, environment, runtime) => sortObjectKeys(Object.fromEntries(Object.entries({
'atlas-component': 'sync',
'atlas-environment': normalizeLabelValue(environment),
'atlas-managed-by': 'terraform',
'atlas-project': normalizeLabelValue(projectId),
'atlas-runtime': runtime
}).filter(([, value]) => value !== null)));
const resolveConnectionSecretNames = config => {
const secretNames = new Set();
for (const source of Object.values(config?.sources ?? {})) {
const secretName = normalizeOptionalString(source?.sql?.connectionSecret ?? source?.http?.connectionSecret);
if (secretName) {
secretNames.add(secretName);
}
}
for (const destination of Object.values(config?.destinations ?? {})) {
const secretName = normalizeOptionalString(destination?.postgres?.connectionSecret);
if (secretName) {
secretNames.add(secretName);
}
}
return [...secretNames].sort();
};
const resolveFirestoreEventarcWorkloads = (config, options = {}) => {
const workloads = [];
const environment = normalizeOptionalString(options.environment);
const triggerRegion = normalizeOptionalString(config?.deploy?.eventarc?.triggerRegion);
for (const [workloadKey, workload] of Object.entries(config?.workloads ?? {})) {
if (!workload.enabled) {
continue;
}
const sourceName = workload.source;
const source = config?.sources?.[sourceName];
if (!source || source.type?.trim().toLowerCase() !== 'firestore') {
continue;
}
const labelSuffix = workloadKey.replace(/_/g, '-').toLowerCase().slice(0, 40);
const triggerName = `atlas-sync-${labelSuffix}`;
const documentPathPattern = normalizeOptionalString(source.firestore?.documentPathPattern);
if (!documentPathPattern) {
continue;
}
workloads.push({
document_path_pattern: documentPathPattern,
...(environment ? {
resource_labels: createSyncTerraformLabels(null, environment, 'eventarc')
} : {}),
trigger_name: triggerName,
workload_key: workloadKey,
...(triggerRegion ? {
trigger_region: triggerRegion
} : {})
});
}
return workloads;
};
const createSyncTerraformFirestoreEventarcPayload = (context, deployConfig) => {
const {
region
} = deployConfig;
const triggerRegion = normalizeOptionalString(context.config?.deploy?.eventarc?.triggerRegion) ?? region;
const database = normalizeOptionalString(context.config?.deploy?.eventarc?.database) ?? SYNC_EVENTARC_DATABASE_DEFAULT;
const workloads = resolveFirestoreEventarcWorkloads(context.config, {
environment: context.environment
});
if (workloads.length === 0) {
return null;
}
return {
database,
event_data_content_type: SYNC_EVENTARC_EVENT_DATA_CONTENT_TYPE,
event_type: SYNC_EVENTARC_EVENT_TYPE,
region,
resource_labels: createSyncTerraformLabels(context.projectId, context.environment, 'eventarc'),
service_name: SYNC_SERVICE_NAME,
service_path_prefix: SYNC_EVENTARC_SERVICE_PATH_PREFIX,
trigger_region: triggerRegion,
trigger_service_account: {
account_id: SYNC_EVENTARC_SERVICE_ACCOUNT_ID,
display_name: 'Atlas Sync Eventarc Trigger Service Account'
},
workloads
};
};
export const writeSyncTerraformArtifact = (context, runtimeConfigUri, mapperManifestUri, options = {}, cwd = process.cwd()) => {
const deployConfig = resolveSyncCloudRunDeployConfig(context);
const {
projectId
} = context;
const {
region,
serviceAccountEmail
} = deployConfig;
const environment = normalizeOptionalString(context.environment);
const artifactBucketName = resolveSyncArtifactBucketName(projectId);
const dlqBucketName = resolveSyncDlqBucketName(projectId);
const taskQueueName = normalizeOptionalString(context.config?.deploy?.taskQueue?.queueName) ?? DEFAULT_SYNC_TASK_QUEUE_NAME;
const taskQueueLocation = normalizeOptionalString(context.config?.deploy?.taskQueue?.location) ?? region;
const secretNames = resolveConnectionSecretNames(context.config);
const firestoreEventarc = createSyncTerraformFirestoreEventarcPayload(context, deployConfig);
const runtimeEnvVars = sortObjectKeys({
PULS_ATLAS_DLQ_BUCKET: dlqBucketName,
PULS_ATLAS_PROJECT_ID: projectId,
PULS_ATLAS_SYNC_CONFIG_URI: runtimeConfigUri,
PULS_ATLAS_SYNC_MAPPER_MANIFEST_URI: mapperManifestUri
});
const servicePayload = {
contract_version: SYNC_CLOUD_RUN_SUBPAYLOAD_CONTRACT_VERSION,
...(environment ? {
environment
} : {}),
image: deployConfig.serviceImage,
max_instances: 100,
min_instances: 0,
project_id: projectId,
region,
resource_labels: createSyncTerraformLabels(projectId, environment, 'service'),
runtime_env_vars: runtimeEnvVars,
service_account_email: serviceAccountEmail,
service_name: SYNC_SERVICE_NAME,
...(deployConfig.vpcAccess ? {
vpc_access: deployConfig.vpcAccess
} : {})
};
const backfillJobPayload = {
contract_version: SYNC_CLOUD_RUN_SUBPAYLOAD_CONTRACT_VERSION,
...(environment ? {
environment
} : {}),
image: deployConfig.backfillJobImage,
job_name: SYNC_BACKFILL_JOB_NAME,
max_retries: SYNC_DEFAULT_BACKFILL_EXECUTION.maxRetries,
parallelism: SYNC_DEFAULT_BACKFILL_EXECUTION.parallelism,
project_id: projectId,
region,
resource_labels: createSyncTerraformLabels(projectId, environment, 'backfill-job'),
runtime_env_vars: runtimeEnvVars,
service_account_email: serviceAccountEmail,
task_count: SYNC_DEFAULT_BACKFILL_EXECUTION.taskCount,
timeout: SYNC_DEFAULT_BACKFILL_EXECUTION.timeout,
...(deployConfig.vpcAccess ? {
vpc_access: deployConfig.vpcAccess
} : {})
};
const payload = {
contract_version: SYNC_TERRAFORM_CONTRACT_VERSION,
project_id: projectId,
service: servicePayload,
backfill_job: backfillJobPayload,
artifact_bucket: {
bucket_name: artifactBucketName,
location: region,
uniform_bucket_level_access: true
},
artifact_bucket_access: {
bucket_name: artifactBucketName,
role: SYNC_ARTIFACT_BUCKET_RUNTIME_ROLE,
service_account_email: serviceAccountEmail
},
dlq_bucket: {
bucket_name: dlqBucketName,
location: region,
retention_days: 365,
uniform_bucket_level_access: true
},
dlq_bucket_access: {
bucket_name: dlqBucketName,
role: SYNC_DLQ_BUCKET_RUNTIME_ROLE,
service_account_email: serviceAccountEmail
},
task_queue: {
location: taskQueueLocation,
name: taskQueueName,
rate_limits: {
max_concurrent_dispatches: 1000,
max_dispatches_per_second: 500
},
retry_config: {
max_attempts: 100,
max_backoff: '3600s',
max_doublings: 16,
max_retry_duration: '0s',
min_backoff: '0.100s'
},
stackdriver_logging_config: {
sampling_ratio: 1.0
}
},
...(secretNames.length > 0 ? {
runtime_secret_access: {
role: SYNC_RUNTIME_SECRET_ROLE,
secret_names: secretNames,
service_account_email: serviceAccountEmail
}
} : {}),
...(firestoreEventarc ? {
firestore_eventarc: firestoreEventarc
} : {})
};
const filePath = path.join(getAtlasGeneratedTerraformDir(cwd), 'sync', `${projectId}.tfvars.json`);
if (!options.dryRun) {
writeJsonFile(filePath, payload);
}
return {
filePath,
payload
};
};
export default {
SYNC_TERRAFORM_CONTRACT_VERSION,
SYNC_TERRAFORM_INPUT_VARIABLE,
writeSyncTerraformArtifact
};