@puls-atlas/cli
Version:
The Puls Atlas CLI tool for managing Atlas projects
288 lines • 14.2 kB
JavaScript
import fs from 'fs';
import path from 'path';
import { resolveRootPath } from '../../utils/atlas.js';
import { isGcloudResourceNotFoundError, runGcloudFileCommand } from '../../utils/gcloud.js';
import { createTerraformStringVariableArgument, runTerraformCommand } from '../../utils/terraform.js';
import { normalizeOptionalString } from '../../utils/value.js';
import { DEFAULT_SYNC_TERRAFORM_MODULE_SOURCE, DEFAULT_SYNC_TERRAFORM_ROOT_DIR } from './config/syncConfig.js';
import { SYNC_TERRAFORM_CONTRACT_VERSION, SYNC_TERRAFORM_INPUT_VARIABLE } from './terraformAdapter.js';
import { loadTaggedSyncTerraformRootFiles } from './terraformRootTemplates.js';
import { SYNC_BACKFILL_JOB_NAME, SYNC_EVENTARC_SERVICE_ACCOUNT_ID, SYNC_SERVICE_NAME } from './resourceNames.js';
const SYNC_TERRAFORM_STATE_NOT_FOUND_PATTERN = /(no instance found for the given address|resource address .* does not exist in the state|no state file was found)/iu;
const SYNC_TERRAFORM_IMPORT_ALREADY_MANAGED_PATTERN = /already managed by Terraform/iu;
const GCLOUD_PERMISSION_DENIED_ERROR_PATTERN = /(permission[ -]?denied|PERMISSION_DENIED|does not have permission|insufficient permission|not authorized|forbidden|iam\.serviceAccounts\.get)/iu;
const SYNC_TERRAFORM_PREREQUISITE_TARGET_KINDS = new Set(['secret-access']);
const escapeTerraformString = value => String(value).replaceAll('\\', '\\\\').replaceAll('"', '\\"');
const normalizeCommandErrorMessage = error => {
const stderr = error?.stderr && Buffer.isBuffer(error.stderr) ? error.stderr.toString('utf8') : typeof error?.stderr === 'string' ? error.stderr : '';
return [error?.message, stderr].filter(Boolean).join('\n');
};
const isGcloudPermissionDeniedError = error => GCLOUD_PERMISSION_DENIED_ERROR_PATTERN.test(normalizeCommandErrorMessage(error));
export const resolveSyncTerraformConfig = syncConfig => ({
moduleSource: normalizeOptionalString(syncConfig?.deploy?.terraform?.moduleSource) ?? DEFAULT_SYNC_TERRAFORM_MODULE_SOURCE,
rootDir: normalizeOptionalString(syncConfig?.deploy?.terraform?.rootDir) ?? DEFAULT_SYNC_TERRAFORM_ROOT_DIR
});
export const resolveSyncTerraformRootPath = (syncConfig, cwd = process.cwd()) => resolveRootPath(resolveSyncTerraformConfig(syncConfig).rootDir, cwd);
export const createSyncTerraformWorkflowSummary = (syncConfig, cwd = process.cwd()) => {
const terraformConfig = resolveSyncTerraformConfig(syncConfig);
return {
moduleSource: terraformConfig.moduleSource,
rootDir: terraformConfig.rootDir,
rootPath: resolveSyncTerraformRootPath(syncConfig, cwd),
managedResources: [{
description: 'Atlas sync data service running on Cloud Run.',
kind: 'service',
name: SYNC_SERVICE_NAME
}, {
description: 'Atlas sync backfill Cloud Run job.',
kind: 'job',
name: SYNC_BACKFILL_JOB_NAME
}, {
description: 'Cloud Tasks queue that durably buffers Atlas sync work.',
kind: 'task-queue',
name: 'atlas-sync'
}, {
description: 'Eventarc trigger service account for Firestore change delivery.',
kind: 'service-account',
name: SYNC_EVENTARC_SERVICE_ACCOUNT_ID
}]
};
};
const renderSyncTerraformRootFiles = async (moduleSource, rootPath, options = {}) => {
const templateFiles = await loadTaggedSyncTerraformRootFiles(moduleSource, rootPath, options);
return Object.fromEntries(Object.entries(templateFiles).map(([fileName, content]) => [fileName, content.replaceAll('__ATLAS_SYNC_TERRAFORM_CONTRACT_VERSION__', String(SYNC_TERRAFORM_CONTRACT_VERSION)).replaceAll('__ATLAS_SYNC_TERRAFORM_INPUT_VARIABLE__', SYNC_TERRAFORM_INPUT_VARIABLE).replaceAll('__ATLAS_SYNC_TERRAFORM_MODULE_SOURCE__', escapeTerraformString(moduleSource))]));
};
export const ensureSyncTerraformRoot = async (syncConfig, cwd = process.cwd(), options = {}) => {
const {
existsSyncImpl = fs.existsSync,
mkdirSyncImpl = fs.mkdirSync,
readFileSyncImpl = fs.readFileSync,
writeFileSyncImpl = fs.writeFileSync
} = options;
const workflowSummary = createSyncTerraformWorkflowSummary(syncConfig, cwd);
const rootFiles = await renderSyncTerraformRootFiles(workflowSummary.moduleSource, workflowSummary.rootPath, options);
const createdFiles = [];
const updatedFiles = [];
if (!existsSyncImpl(workflowSummary.rootPath)) {
mkdirSyncImpl(workflowSummary.rootPath, {
recursive: true
});
}
for (const [fileName, content] of Object.entries(rootFiles)) {
const filePath = path.join(workflowSummary.rootPath, fileName);
if (!existsSyncImpl(filePath)) {
writeFileSyncImpl(filePath, content);
createdFiles.push(filePath);
continue;
}
if (readFileSyncImpl(filePath, 'utf-8') === content) {
continue;
}
writeFileSyncImpl(filePath, content);
updatedFiles.push(filePath);
}
return {
...workflowSummary,
createdFiles,
updatedFiles
};
};
const createSyncTerraformWorkflowCommand = (mode, terraformArtifact, options = {}) => {
const targetArgs = (options.targets ?? []).map(target => `-target=${target}`);
const baseArgs = [createTerraformStringVariableArgument(SYNC_TERRAFORM_INPUT_VARIABLE, terraformArtifact.filePath), ...targetArgs, '-input=false'];
if (mode === 'plan') {
return ['plan', ...baseArgs];
}
return [mode, '-auto-approve', ...baseArgs];
};
const createSyncTerraformStateShowCommand = address => ['state', 'show', address];
const createSyncTerraformImportCommand = (terraformArtifact, target) => ['import', createTerraformStringVariableArgument(SYNC_TERRAFORM_INPUT_VARIABLE, terraformArtifact.filePath), '-input=false', target.address, target.id];
const getSyncTerraformImportTargets = terraformArtifact => {
const payload = terraformArtifact?.payload;
if (!payload || typeof payload !== 'object') {
return [];
}
const targets = [];
const projectId = payload.project_id;
const {
service,
backfill_job: backfillJob,
task_queue: taskQueue,
firestore_eventarc: firestoreEventarc,
runtime_secret_access: runtimeSecretAccess
} = payload;
if (typeof projectId === 'string' && typeof runtimeSecretAccess?.service_account_email === 'string' && typeof runtimeSecretAccess?.role === 'string' && Array.isArray(runtimeSecretAccess?.secret_names)) {
for (const secretName of runtimeSecretAccess.secret_names) {
if (typeof secretName !== 'string') {
continue;
}
const member = `serviceAccount:${runtimeSecretAccess.service_account_email}`;
targets.push({
address: `google_secret_manager_secret_iam_member.atlas_sync_runtime["${secretName}"]`,
describeArgs: ['secrets', 'get-iam-policy', secretName, `--project=${projectId}`, '--format=json'],
id: `projects/${projectId}/secrets/${secretName} ${runtimeSecretAccess.role} ${member}`,
kind: 'secret-access',
name: secretName
});
}
}
if (typeof projectId === 'string' && typeof service?.service_name === 'string' && typeof service?.region === 'string') {
targets.push({
address: 'module.atlas_sync_service[0].google_cloud_run_v2_service.this',
describeArgs: ['run', 'services', 'describe', service.service_name, `--project=${projectId}`, `--region=${service.region}`, '--format=value(name)'],
id: `projects/${projectId}/locations/${service.region}/services/${service.service_name}`,
kind: 'service',
name: service.service_name
});
}
if (typeof projectId === 'string' && typeof backfillJob?.job_name === 'string' && typeof backfillJob?.region === 'string') {
targets.push({
address: 'module.atlas_sync_backfill_job[0].google_cloud_run_v2_job.this',
describeArgs: ['run', 'jobs', 'describe', backfillJob.job_name, `--project=${projectId}`, `--region=${backfillJob.region}`, '--format=value(name)'],
id: `projects/${projectId}/locations/${backfillJob.region}/jobs/${backfillJob.job_name}`,
kind: 'job',
name: backfillJob.job_name
});
}
if (typeof projectId === 'string' && typeof taskQueue?.name === 'string' && typeof taskQueue?.location === 'string') {
targets.push({
address: 'google_cloud_tasks_queue.atlas_sync[0]',
describeArgs: ['tasks', 'queues', 'describe', taskQueue.name, `--location=${taskQueue.location}`, `--project=${projectId}`, '--format=value(name)'],
id: `projects/${projectId}/locations/${taskQueue.location}/queues/${taskQueue.name}`,
kind: 'task-queue',
name: taskQueue.name
});
}
if (typeof projectId === 'string' && typeof firestoreEventarc?.trigger_service_account?.account_id === 'string') {
const serviceAccountEmail = `${firestoreEventarc.trigger_service_account.account_id}@${projectId}.iam.gserviceaccount.com`;
targets.push({
address: 'google_service_account.atlas_sync_eventarc[0]',
describeArgs: ['iam', 'service-accounts', 'describe', serviceAccountEmail, `--project=${projectId}`, '--format=value(email)'],
id: `projects/${projectId}/serviceAccounts/${serviceAccountEmail}`,
kind: 'service-account',
name: serviceAccountEmail
});
}
if (typeof projectId === 'string' && typeof firestoreEventarc?.trigger_region === 'string' && Array.isArray(firestoreEventarc?.workloads)) {
for (const workload of firestoreEventarc.workloads) {
if (typeof workload?.workload_key !== 'string' || typeof workload?.trigger_name !== 'string') {
continue;
}
const triggerRegion = firestoreEventarc.trigger_region;
targets.push({
address: `google_eventarc_trigger.atlas_sync_firestore["${workload.workload_key}"]`,
describeArgs: ['eventarc', 'triggers', 'describe', workload.trigger_name, `--project=${projectId}`, `--location=${triggerRegion}`, '--format=value(name)'],
id: `projects/${projectId}/locations/${triggerRegion}/triggers/${workload.trigger_name}`,
kind: 'eventarc-trigger',
name: workload.trigger_name
});
}
}
return targets;
};
const createSyncTerraformImportResult = (target, status) => ({
address: target.address,
kind: target.kind,
name: target.name,
status
});
const doesSyncTerraformImportTargetExist = (target, runGcloudFileCommandImpl = runGcloudFileCommand) => {
try {
runGcloudFileCommandImpl(target.describeArgs, {
encoding: 'utf8',
stdio: ['ignore', 'pipe', 'pipe']
});
return 'exists';
} catch (error) {
if (isGcloudResourceNotFoundError(error)) {
return 'missing';
}
if (isGcloudPermissionDeniedError(error)) {
return 'unknown';
}
throw new Error(`Could not inspect existing Atlas sync ${target.kind} ${target.name}. ${normalizeCommandErrorMessage(error)}`);
}
};
export const importExistingSyncTerraformResources = async (terraformArtifact, workflowSummary, dependencies = {}) => {
const importResults = [];
const runTerraformCommandImpl = dependencies.runTerraformCommand ?? runTerraformCommand;
const runGcloudFileCommandImpl = dependencies.runGcloudFileCommand ?? runGcloudFileCommand;
for (const target of getSyncTerraformImportTargets(terraformArtifact, workflowSummary)) {
const existenceStatus = doesSyncTerraformImportTargetExist(target, runGcloudFileCommandImpl);
if (existenceStatus === 'missing') {
importResults.push(createSyncTerraformImportResult(target, 'missing'));
continue;
}
if (existenceStatus === 'unknown') {
importResults.push(createSyncTerraformImportResult(target, 'unknown'));
continue;
}
try {
await runTerraformCommandImpl(createSyncTerraformStateShowCommand(target.address), {
captureOutput: true,
cwd: workflowSummary.rootPath
});
importResults.push(createSyncTerraformImportResult(target, 'already-managed'));
continue;
} catch (error) {
if (!SYNC_TERRAFORM_STATE_NOT_FOUND_PATTERN.test(error.message)) {
throw error;
}
}
try {
await runTerraformCommandImpl(createSyncTerraformImportCommand(terraformArtifact, target), {
captureOutput: true,
cwd: workflowSummary.rootPath
});
importResults.push(createSyncTerraformImportResult(target, 'imported'));
} catch (error) {
if (SYNC_TERRAFORM_IMPORT_ALREADY_MANAGED_PATTERN.test(error.message)) {
importResults.push(createSyncTerraformImportResult(target, 'already-managed'));
continue;
}
throw error;
}
}
return importResults;
};
export const getSyncTerraformPrerequisiteTargets = (terraformArtifact, workflowSummary) => getSyncTerraformImportTargets(terraformArtifact, workflowSummary).filter(target => SYNC_TERRAFORM_PREREQUISITE_TARGET_KINDS.has(target.kind)).map(target => target.address);
export const runSyncTerraformWorkflow = async (syncConfig, terraformArtifact, options = {}, dependencies = {}, cwd = process.cwd()) => {
const ensureTerraformRoot = dependencies.ensureSyncTerraformRoot ?? ensureSyncTerraformRoot;
const runTerraformCommandImpl = dependencies.runTerraformCommand ?? runTerraformCommand;
const workflowSummary = await ensureTerraformRoot(syncConfig, cwd, dependencies);
const mode = options.mode ?? (options.dryRun ? 'plan' : 'apply');
const initArgs = ['init', '-input=false'];
const importResults = [];
await runTerraformCommandImpl(initArgs, {
cwd: workflowSummary.rootPath,
stdio: 'inherit'
});
if (mode !== 'plan') {
importResults.push(...(await importExistingSyncTerraformResources(terraformArtifact, workflowSummary, {
runGcloudFileCommand: dependencies.runGcloudFileCommand,
runTerraformCommand: runTerraformCommandImpl
})));
}
const workflowArgs = createSyncTerraformWorkflowCommand(mode, terraformArtifact, {
targets: options.targets
});
await runTerraformCommandImpl(workflowArgs, {
cwd: workflowSummary.rootPath,
stdio: 'inherit'
});
return {
...workflowSummary,
commands: [initArgs, workflowArgs],
importResults,
mode
};
};
export default {
createSyncTerraformWorkflowSummary,
ensureSyncTerraformRoot,
getSyncTerraformPrerequisiteTargets,
importExistingSyncTerraformResources,
resolveSyncTerraformConfig,
resolveSyncTerraformRootPath,
runSyncTerraformWorkflow
};