@puls-atlas/cli/dist/cmd/search/terraformAdapter.js

The Puls Atlas CLI tool for managing Atlas projects

import path from 'path';
import { resolveSearchBackfillJobDeployment } from './backfillJob.js';
import { resolveSearchManagedArtifactBuckets } from './artifactBucket.js';
import { getEnabledSearchEventarcCollectionPlans, SEARCH_EVENTARC_DEFAULTS } from './eventarc.js';
import { resolveSearchCloudRunServiceAccountEmail } from './config/searchConfig.js';
import { resolveSearchDlqBucketConfig, resolveSearchTaskQueueConfig } from './planning.js';
import { resolveSearchRuntimeSecretAccessContract } from './runtimeSecrets.js';
import { getAtlasGeneratedTerraformDir } from '../../utils/atlas.js';
import { writeJsonFile } from '../../utils/file.js';
import { normalizeOptionalString } from '../../utils/value.js';
import { ATLAS_SEARCH_TASK_QUEUE_DEFAULTS } from '../../utils/taskQueue.js';
import { SEARCH_SOURCE_RUNNER_REQUESTED_SOURCE_ENV, getEnabledScheduledSearchSourcePlans } from './sources/scheduled.js';
import { SEARCH_SOURCE_RUNNER_CONTAINER_NAME, resolveSearchSourceRunnerJobDeployment } from './sources/runnerJob.js';
import { getConfiguredSearchFirestoreEventarcDatabase, getConfiguredSearchFirestoreEventarcTriggerRegion } from './firestoreEventarc.js';
export const SEARCH_TERRAFORM_CONTRACT_VERSION = 6;
export const SEARCH_TERRAFORM_INPUT_VARIABLE = 'atlas_search_tfvars_path';
const ATLAS_SEARCH_DLQ_BUCKET_RUNTIME_ROLE = 'roles/storage.objectAdmin';
const SEARCH_SOURCE_SCHEDULER_OAUTH_SCOPE = 'https://www.googleapis.com/auth/cloud-platform';
const DEFAULT_BACKFILL_JOB_EXECUTION = {
  maxRetries: 0,
  parallelism: 1,
  taskCount: 1,
  timeout: '3600s'
};
const normalizeLabelValue = value => {
  const normalized = normalizeOptionalString(value);
  if (!normalized) {
    return null;
  }
  const normalizedValue = normalized.toLowerCase().replace(/[^a-z0-9-]/g, '-').replace(/-+/g, '-').replace(/^-+|-+$/g, '').slice(0, 63).replace(/-+$/g, '');
  return normalizedValue.length > 0 ? normalizedValue : null;
};
const sortObjectKeys = objectValue => Object.fromEntries(Object.entries(objectValue).sort(([leftKey], [rightKey]) => leftKey.localeCompare(rightKey)));
const createSearchTerraformLabels = (context, runtime) => sortObjectKeys(Object.fromEntries(Object.entries({
  'atlas-component': 'search',
  'atlas-environment': normalizeLabelValue(context.environment),
  'atlas-managed-by': 'terraform',
  'atlas-project': normalizeLabelValue(context.projectId),
  'atlas-provider': normalizeLabelValue(context.config.provider),
  'atlas-runtime': runtime
}).filter(([, value]) => value !== null)));
const createSearchTerraformBucketPayload = ({
  bucketName,
  location,
  retentionDays = null
}) => ({
  bucket_name: bucketName,
  location,
  ...(retentionDays !== null ? {
    retention_days: retentionDays
  } : {}),
  uniform_bucket_level_access: true
});
const getConfiguredServiceAccountEmail = context => resolveSearchCloudRunServiceAccountEmail(context.config, context.projectId);
const getConfiguredVpcAccess = context => {
  const vpcAccess = context.config?.deploy?.cloudRun?.vpcAccess;
  if (!vpcAccess || typeof vpcAccess !== 'object') {
    return null;
  }
  const network = normalizeOptionalString(vpcAccess.network);
  if (!network) {
    return null;
  }
  return {
    egress: normalizeOptionalString(vpcAccess.egress) ?? 'private-ranges-only',
    network,
    subnetwork: normalizeOptionalString(vpcAccess.subnetwork)
  };
};
const createSearchTerraformBackfillJobPayload = context => {
  const deployment = resolveSearchBackfillJobDeployment(context);
  const runtimeEnvVars = sortObjectKeys(deployment.runtimeEnvironmentVariables);
  const searchConfigPayloadSize = 0;
  const warnings = [];
  const serviceAccountEmail = getConfiguredServiceAccountEmail(context);
  const vpcAccess = getConfiguredVpcAccess(context);
  return {
    payload: {
      contract_version: 1,
      environment: context.environment ?? null,
      image: deployment.image,
      job_name: deployment.jobName,
      max_retries: DEFAULT_BACKFILL_JOB_EXECUTION.maxRetries,
      parallelism: DEFAULT_BACKFILL_JOB_EXECUTION.parallelism,
      project_id: deployment.projectId,
      region: deployment.region,
      resource_labels: createSearchTerraformLabels(context, 'backfill-job'),
      runtime_env_vars: runtimeEnvVars,
      service_account_email: serviceAccountEmail,
      task_count: DEFAULT_BACKFILL_JOB_EXECUTION.taskCount,
      timeout: DEFAULT_BACKFILL_JOB_EXECUTION.timeout,
      ...(vpcAccess ? {
        vpc_access: vpcAccess
      } : {})
    },
    searchConfigPayloadSize,
    warnings
  };
};
const createSearchTerraformSourceRunnerJobPayload = context => {
  const deployment = resolveSearchSourceRunnerJobDeployment(context);
  if (!deployment) {
    return null;
  }
  const runtimeEnvVars = sortObjectKeys(deployment.runtimeEnvironmentVariables);
  const serviceAccountEmail = getConfiguredServiceAccountEmail(context);
  const vpcAccess = getConfiguredVpcAccess(context);
  return {
    contract_version: 1,
    environment: context.environment ?? null,
    image: deployment.image,
    job_name: deployment.jobName,
    max_retries: DEFAULT_BACKFILL_JOB_EXECUTION.maxRetries,
    parallelism: DEFAULT_BACKFILL_JOB_EXECUTION.parallelism,
    project_id: deployment.projectId,
    region: deployment.region,
    resource_labels: createSearchTerraformLabels(context, 'source-runner-job'),
    runtime_env_vars: runtimeEnvVars,
    ...(Object.keys(deployment.runtimeSecretEnvironmentVariables ?? {}).length > 0 ? {
      secret_env_vars: sortObjectKeys(deployment.runtimeSecretEnvironmentVariables)
    } : {}),
    service_account_email: serviceAccountEmail,
    task_count: DEFAULT_BACKFILL_JOB_EXECUTION.taskCount,
    timeout: DEFAULT_BACKFILL_JOB_EXECUTION.timeout,
    ...(vpcAccess ? {
      vpc_access: vpcAccess
    } : {})
  };
};
const createSearchTerraformSourceSchedulerPayload = context => {
  const deployment = resolveSearchSourceRunnerJobDeployment(context);
  if (!deployment) {
    return null;
  }
  const scheduledSourcePlans = getEnabledScheduledSearchSourcePlans(context.config);
  if (scheduledSourcePlans.length === 0) {
    return null;
  }
  return {
    container_name: SEARCH_SOURCE_RUNNER_CONTAINER_NAME,
    contract_version: 1,
    jobs: scheduledSourcePlans.map(sourcePlan => ({
      name: sourcePlan.schedulerJobName,
      schedule: sourcePlan.schedule,
      source_name: sourcePlan.sourceName,
      time_zone: sourcePlan.timeZone
    })),
    oauth_scope: SEARCH_SOURCE_SCHEDULER_OAUTH_SCOPE,
    region: deployment.region,
    service_account_email: deployment.serviceAccountEmail,
    source_name_env_var: SEARCH_SOURCE_RUNNER_REQUESTED_SOURCE_ENV,
    source_runner_job_name: deployment.jobName,
    source_runner_region: deployment.region
  };
};
const createSearchTerraformFirestoreEventarcPayload = (context, options = {}) => {
  const deploymentRegion = resolveSearchBackfillJobDeployment(context).region;
  const configuredTriggerRegion = getConfiguredSearchFirestoreEventarcTriggerRegion(context, options);
  return {
    collections: getEnabledSearchEventarcCollectionPlans(context.config).map(collectionPlan => ({
      collection_name: collectionPlan.collectionName,
      document_path_pattern: collectionPlan.documentPathPattern,
      trigger_name: collectionPlan.triggerName
    })),
    database: getConfiguredSearchFirestoreEventarcDatabase(context, options),
    event_data_content_type: 'application/protobuf',
    event_type: SEARCH_EVENTARC_DEFAULTS.eventType,
    region: deploymentRegion,
    resource_labels: createSearchTerraformLabels(context, 'firestore-sync'),
    service_name: 'atlas-search-sync',
    service_path_prefix: SEARCH_EVENTARC_DEFAULTS.pathPrefix,
    trigger_service_account: {
      account_id: SEARCH_EVENTARC_DEFAULTS.triggerServiceAccountId,
      display_name: SEARCH_EVENTARC_DEFAULTS.triggerServiceAccountDisplayName
    },
    ...(configuredTriggerRegion && configuredTriggerRegion !== deploymentRegion ? {
      trigger_region: configuredTriggerRegion
    } : {})
  };
};
const createSearchTerraformSyncTaskQueuePayload = context => {
  const taskQueueConfig = resolveSearchTaskQueueConfig(context);
  return {
    contract_version: 1,
    location: taskQueueConfig.location,
    name: taskQueueConfig.name,
    rate_limits: {
      max_concurrent_dispatches: ATLAS_SEARCH_TASK_QUEUE_DEFAULTS.maxConcurrentDispatches,
      max_dispatches_per_second: ATLAS_SEARCH_TASK_QUEUE_DEFAULTS.maxDispatchesPerSecond
    },
    retry_config: {
      max_attempts: ATLAS_SEARCH_TASK_QUEUE_DEFAULTS.maxAttempts,
      max_backoff: ATLAS_SEARCH_TASK_QUEUE_DEFAULTS.maxBackoff,
      max_doublings: ATLAS_SEARCH_TASK_QUEUE_DEFAULTS.maxDoublings,
      max_retry_duration: ATLAS_SEARCH_TASK_QUEUE_DEFAULTS.maxRetryDuration,
      min_backoff: ATLAS_SEARCH_TASK_QUEUE_DEFAULTS.minBackoff
    },
    stackdriver_logging_config: {
      sampling_ratio: ATLAS_SEARCH_TASK_QUEUE_DEFAULTS.logSamplingRatio
    }
  };
};
const createSearchTerraformArtifactBucketsPayload = context => resolveSearchManagedArtifactBuckets(context).map(bucketState => createSearchTerraformBucketPayload({
  bucketName: bucketState.bucketName,
  location: bucketState.location
}));
const createSearchTerraformDlqBucketPayload = context => {
  const dlqBucketConfig = resolveSearchDlqBucketConfig(context);
  return createSearchTerraformBucketPayload({
    bucketName: dlqBucketConfig.name,
    location: dlqBucketConfig.location,
    retentionDays: dlqBucketConfig.retentionDays
  });
};
const createSearchTerraformDlqBucketAccessPayload = context => {
  const serviceAccountEmail = getConfiguredServiceAccountEmail(context);
  if (!serviceAccountEmail) {
    return null;
  }
  return {
    bucket_name: resolveSearchDlqBucketConfig(context).name,
    role: ATLAS_SEARCH_DLQ_BUCKET_RUNTIME_ROLE,
    service_account_email: serviceAccountEmail
  };
};
const createSearchTerraformRuntimeSecretAccessPayload = context => {
  const runtimeSecretAccess = resolveSearchRuntimeSecretAccessContract(context);
  if (!runtimeSecretAccess.serviceAccountEmail || runtimeSecretAccess.secretAccess.length === 0) {
    return null;
  }
  return {
    contract_version: 1,
    role: runtimeSecretAccess.secretAccess[0].role,
    secret_names: runtimeSecretAccess.secretAccess.map(secretAccess => secretAccess.secretName),
    service_account_email: runtimeSecretAccess.serviceAccountEmail
  };
};
export const createSearchTerraformAdapter = (context, options = {}) => {
  const backfillJobAdapter = createSearchTerraformBackfillJobPayload(context);
  const sourceRunnerJobPayload = createSearchTerraformSourceRunnerJobPayload(context);
  const sourceSchedulerPayload = createSearchTerraformSourceSchedulerPayload(context);
  return {
    payload: {
      artifact_buckets: createSearchTerraformArtifactBucketsPayload(context),
      backfill_job: backfillJobAdapter.payload,
      contract_version: SEARCH_TERRAFORM_CONTRACT_VERSION,
      dlq_bucket: createSearchTerraformDlqBucketPayload(context),
      dlq_bucket_access: createSearchTerraformDlqBucketAccessPayload(context),
      environment: context.environment ?? null,
      firestore_eventarc: createSearchTerraformFirestoreEventarcPayload(context, options),
      project_id: context.projectId,
      region: backfillJobAdapter.payload.region,
      runtime_secret_access: createSearchTerraformRuntimeSecretAccessPayload(context),
      ...(sourceRunnerJobPayload ? {
        source_runner_job: sourceRunnerJobPayload
      } : {}),
      ...(sourceSchedulerPayload ? {
        source_scheduler: sourceSchedulerPayload
      } : {}),
      sync_task_queue: createSearchTerraformSyncTaskQueuePayload(context)
    },
    searchConfigPayloadSize: backfillJobAdapter.searchConfigPayloadSize,
    warnings: backfillJobAdapter.warnings
  };
};
export const getSearchTerraformArtifactPath = (projectId, cwd = process.cwd()) => path.join(getAtlasGeneratedTerraformDir(cwd), 'search', `search.${projectId}.tfvars.json`);
export const writeSearchTerraformArtifact = (context, options = {}, cwd = process.cwd()) => {
  const {
    writeJsonFileImpl = writeJsonFile
  } = options;
  const adapter = createSearchTerraformAdapter(context, options);
  const filePath = getSearchTerraformArtifactPath(context.projectId, cwd);
  writeJsonFileImpl(filePath, adapter.payload);
  return {
    contractVersion: adapter.payload.contract_version,
    filePath,
    payload: adapter.payload,
    searchConfigPayloadSize: adapter.searchConfigPayloadSize,
    warnings: adapter.warnings
  };
};
export const SEARCH_TERRAFORM_BACKFILL_JOB_CONTRACT_VERSION = SEARCH_TERRAFORM_CONTRACT_VERSION;
export const SEARCH_TERRAFORM_BACKFILL_JOB_INPUT_VARIABLE = SEARCH_TERRAFORM_INPUT_VARIABLE;
export const createSearchTerraformBackfillJobAdapter = createSearchTerraformAdapter;
export const getSearchTerraformBackfillJobArtifactPath = getSearchTerraformArtifactPath;
export const writeSearchTerraformBackfillJobArtifact = writeSearchTerraformArtifact;
export default {
  SEARCH_TERRAFORM_BACKFILL_JOB_CONTRACT_VERSION,
  SEARCH_TERRAFORM_BACKFILL_JOB_INPUT_VARIABLE,
  SEARCH_TERRAFORM_CONTRACT_VERSION,
  SEARCH_TERRAFORM_INPUT_VARIABLE,
  createSearchTerraformAdapter,
  createSearchTerraformBackfillJobAdapter,
  getSearchTerraformArtifactPath,
  getSearchTerraformBackfillJobArtifactPath,
  writeSearchTerraformArtifact,
  writeSearchTerraformBackfillJobArtifact
};