@puls-atlas/cli/dist/cmd/search/discovery.js

The Puls Atlas CLI tool for managing Atlas projects

import { execFileSync as nodeExecFileSync } from 'child_process';
import { pick } from 'es-toolkit/object';
import { runGcloudFileCommand } from '../../utils/gcloud.js';
import { dedupeMessages, normalizeOptionalString, spreadIf } from '../../utils/value.js';
import { getEnabledSearchEventarcCollectionPlans } from './eventarc.js';
import { resolveSearchFirestoreEventarcTriggerRegion } from './firestoreEventarc.js';
import { getSearchProvider, getSearchProviderDisplayName } from './providers/index.js';
import { discoverSearchProviderTargetsViaSearchApi } from './searchApiAdmin.js';
import { getEnabledScheduledSearchSourcePlans } from './sources/scheduled.js';
import { SEARCH_RESOURCE_NAMES } from './resourceNames.js';
import { resolveSearchCloudRunDeployConfig, resolveSearchTaskQueueConfig } from './deploymentConfig.js';
const DEFAULT_CLOUD_RUN_REGIONS = ['europe-west1', 'europe-west4', 'europe-west3', 'us-central1'];
const GCLOUD_LIST_FORMAT_ARGS = ['--quiet', '--format=json'];
const runJsonCommand = (args, execute = nodeExecFileSync) => {
  try {
    const output = runGcloudFileCommand(args, {
      encoding: 'utf8',
      stdio: ['ignore', 'pipe', 'pipe']
    }, execute);
    return {
      data: JSON.parse(output),
      error: null
    };
  } catch (error) {
    return {
      data: null,
      error: error.stderr?.toString().trim() || error.message
    };
  }
};
const mergeDiscoveredResources = resources => Array.from(new Map(resources.filter(resource => resource.name).map(resource => [resource.name, resource])).values());
const normalizeCloudRunResource = (resource, fallbackLocation = null) => ({
  name: resource?.metadata?.name ?? resource?.name ?? null,
  location: resource?.metadata?.labels?.['cloud.googleapis.com/location'] ?? resource?.metadata?.labels?.location ?? resource?.location ?? fallbackLocation
});
const normalizeLocatedResource = (resource, resourceGroup) => {
  const fullName = normalizeOptionalString(resource?.name);
  const resourceName = fullName ? fullName.split('/').pop() : null;
  const locationMatch = fullName?.match(new RegExp(`/locations/([^/]+)/${resourceGroup}/`, 'i'));
  return {
    location: locationMatch?.[1] ?? resource?.location ?? null,
    name: resourceName
  };
};
const discoverCloudRunResourceGroup = (resourceType, projectId, execute = nodeExecFileSync) => {
  const baseArgs = ['run', resourceType, 'list', `--project=${projectId}`];
  const platformArgs = resourceType === 'services' ? ['--platform=managed'] : [];
  const globalResult = runJsonCommand([...baseArgs, ...platformArgs, ...GCLOUD_LIST_FORMAT_ARGS], execute);
  if (!globalResult.error) {
    return {
      resources: mergeDiscoveredResources(globalResult.data.map(resource => normalizeCloudRunResource(resource))),
      warnings: []
    };
  }
  const fallbackResources = [];
  const successfulRegions = [];
  for (const region of DEFAULT_CLOUD_RUN_REGIONS) {
    const regionalResult = runJsonCommand([...baseArgs, ...platformArgs, `--region=${region}`, ...GCLOUD_LIST_FORMAT_ARGS], execute);
    if (regionalResult.error) {
      continue;
    }
    successfulRegions.push(region);
    fallbackResources.push(...regionalResult.data.map(resource => normalizeCloudRunResource(resource, region)));
  }
  if (successfulRegions.length > 0) {
    return {
      resources: mergeDiscoveredResources(fallbackResources),
      warnings: [`Global Cloud Run ${resourceType} discovery failed. Regional fallback succeeded for: ${successfulRegions.join(', ')}.`]
    };
  }
  return {
    resources: null,
    warnings: [`Could not inspect Cloud Run ${resourceType}. Global lookup failed and no regional fallback succeeded in: ${DEFAULT_CLOUD_RUN_REGIONS.join(', ')}.`]
  };
};
const discoverResourceGroup = (args, normalizeResource, warningMessage, execute = nodeExecFileSync) => {
  const result = runJsonCommand([...args, ...GCLOUD_LIST_FORMAT_ARGS], execute);
  if (!result.error) {
    return {
      warnings: [],
      resources: mergeDiscoveredResources(result.data.map(resource => normalizeResource(resource)))
    };
  }
  return {
    resources: null,
    warnings: [warningMessage(result.error)]
  };
};
const discoverEventarcTriggerGroup = (projectId, region, execute = nodeExecFileSync) => discoverResourceGroup(['eventarc', 'triggers', 'list', `--project=${projectId}`, `--location=${region}`], resource => normalizeLocatedResource(resource, 'triggers'), error => `Could not inspect Eventarc triggers in ${region}: ${error}`, execute);
const discoverTaskQueueGroup = (projectId, location, execute = nodeExecFileSync) => discoverResourceGroup(['tasks', 'queues', 'list', `--project=${projectId}`, `--location=${location}`], resource => normalizeLocatedResource(resource, 'queues'), error => `Could not inspect Cloud Tasks queues in ${location}: ${error}`, execute);
const discoverSchedulerJobGroup = (projectId, region, execute = nodeExecFileSync) => discoverResourceGroup(['scheduler', 'jobs', 'list', `--project=${projectId}`, `--location=${region}`], resource => normalizeLocatedResource(resource, 'jobs'), error => `Could not inspect Cloud Scheduler jobs in ${region}: ${error}`, execute);
const resolveSearchEventarcDiscoveryRegion = (context, options = {}) => {
  const fallbackRegion = context.config?.deploy?.cloudRun?.region ?? 'europe-west1';
  try {
    const firestoreEventarc = resolveSearchFirestoreEventarcTriggerRegion(context, options);
    return {
      region: firestoreEventarc.triggerRegion,
      warnings: []
    };
  } catch (error) {
    return {
      region: fallbackRegion,
      warnings: ['Could not resolve the Firestore Eventarc trigger region for remote discovery. ' + `Falling back to the Cloud Run region ${fallbackRegion}. ${error.message}`]
    };
  }
};
const getExpectedSearchEventarcTriggerResources = context => getEnabledSearchEventarcCollectionPlans(context.config).map(collectionPlan => ({
  name: collectionPlan.triggerName,
  projectId: context.projectId,
  purpose: 'Eventarc Standard trigger that routes Firestore direct events into atlas-search-sync.'
}));
const getExpectedSearchTaskQueueResources = context => {
  const taskQueue = resolveSearchTaskQueueConfig(context);
  return [{
    location: taskQueue.location,
    name: taskQueue.name,
    projectId: context.projectId,
    purpose: 'Managed Cloud Tasks queue that durably buffers Atlas search incremental sync work before atlas-search-sync processes it.'
  }];
};
const getExpectedSearchSchedulerJobResources = context => {
  const deploymentRegion = resolveSearchCloudRunDeployConfig(context).region;
  return getEnabledScheduledSearchSourcePlans(context.config).map(sourcePlan => ({
    location: deploymentRegion,
    name: sourcePlan.schedulerJobName,
    projectId: context.projectId,
    purpose: 'Cloud Scheduler job that executes atlas-search-source-runner for a single scheduled Atlas search source.'
  }));
};
const toRemoteStatus = (resource, discoveredResources) => {
  if (!discoveredResources) {
    return {
      ...resource,
      remoteStatus: 'unknown'
    };
  }
  const match = discoveredResources.find(entry => entry.name === resource.name);
  if (!match) {
    return {
      ...resource,
      remoteStatus: 'not-found'
    };
  }
  return {
    ...resource,
    remoteLocation: match.location ?? null,
    remoteStatus: 'found'
  };
};
const createProviderDiscoveryFallbackWarning = (provider, error) => `Could not inspect ${provider.descriptor.labels.targetGroup.toLowerCase()} through atlas-search-api: ${error.message}`;
const createProviderDiscoveryUnavailableWarning = provider => `atlas-search-api did not return ${provider.descriptor.labels.targetGroup.toLowerCase()}, so Atlas fell back to direct provider inspection from this machine.`;
const collectDiscoveryWarnings = (...warningGroups) => dedupeMessages(warningGroups.flatMap(warningGroup => warningGroup ?? []));
const createProviderDiscoveryResult = (targets, ...warningGroups) => ({
  targets,
  warnings: collectDiscoveryWarnings(...warningGroups)
});
const discoverProviderTargetsWithFallback = async (context, provider, options = {}) => {
  const discoverProviderTargetsViaSearchApiImpl = options.discoverProviderTargetsViaSearchApiImpl ?? discoverSearchProviderTargetsViaSearchApi;
  const discoverProviderTargetsDirectImpl = options.discoverProviderTargetsDirectImpl ?? (async (providerContext, directOptions = {}) => provider.discoverRemoteTargets(providerContext, directOptions));
  const apiRequestOptions = pick(options, ['fetchImpl', 'getSecretImpl', 'runCommand']);
  try {
    const result = await discoverProviderTargetsViaSearchApiImpl(context, apiRequestOptions);
    if (Array.isArray(result?.targets)) {
      return result;
    }
    const directResult = await discoverProviderTargetsDirectImpl(context, apiRequestOptions);
    return createProviderDiscoveryResult(directResult.targets, result?.warnings ?? [], createProviderDiscoveryUnavailableWarning(provider), directResult.warnings ?? []);
  } catch (error) {
    const directResult = await discoverProviderTargetsDirectImpl(context, apiRequestOptions);
    return createProviderDiscoveryResult(directResult.targets, createProviderDiscoveryFallbackWarning(provider, error), directResult.warnings ?? []);
  }
};
export const getExpectedSearchResources = context => {
  const provider = getSearchProvider(context.config.provider);
  const providerDisplayName = getSearchProviderDisplayName(provider, context.config.provider);
  const scheduledSourcePlans = getEnabledScheduledSearchSourcePlans(context.config);
  const providerTargets = Object.keys(context.config.indexes).map(indexName => ({
    indexName,
    name: indexName
  }));
  return {
    cloudRunJobs: [{
      name: SEARCH_RESOURCE_NAMES.backfillJob,
      projectId: context.projectId,
      purpose: `Batch job that backfills and reindexes Firestore data into the separate ${providerDisplayName} engine.`
    }, ...spreadIf(scheduledSourcePlans.length > 0, [{
      name: SEARCH_RESOURCE_NAMES.sourceRunnerJob,
      projectId: context.projectId,
      purpose: 'Cloud Run job that executes scheduled Atlas search source adapters and advances source cursors.'
    }])],
    cloudRunServices: [{
      name: SEARCH_RESOURCE_NAMES.searchSyncService,
      projectId: context.projectId,
      purpose: 'Write-side Atlas search sync service that resolves mappers and applies provider updates centrally.'
    }, {
      name: SEARCH_RESOURCE_NAMES.searchApiService,
      projectId: context.projectId,
      purpose: `Optional stateless API layer that queries the separate ${providerDisplayName} engine for search requests.`
    }],
    computeInstances: [],
    eventarcTriggers: getExpectedSearchEventarcTriggerResources(context),
    schedulerJobs: getExpectedSearchSchedulerJobResources(context),
    taskQueues: getExpectedSearchTaskQueueResources(context),
    providerTargets: providerTargets.map(target => ({
      indexName: target.indexName,
      name: target.name,
      projectId: context.projectId,
      purpose: `Expected ${provider.descriptor.labels.targetGroup.toLowerCase()} entry derived from Atlas search index config.`
    }))
  };
};
export const discoverSearchRemoteResources = async (context, options = {}) => {
  const expectedResources = getExpectedSearchResources(context);
  const warnings = [];
  const provider = options.provider ?? getSearchProvider(context.config.provider);
  const runCommand = options.runCommand ?? nodeExecFileSync;
  const servicesResult = discoverCloudRunResourceGroup('services', context.projectId, runCommand);
  const jobsResult = discoverCloudRunResourceGroup('jobs', context.projectId, runCommand);
  const taskQueueConfig = resolveSearchTaskQueueConfig(context);
  const taskQueuesResult = discoverTaskQueueGroup(context.projectId, taskQueueConfig.location, runCommand);
  const schedulerJobsResult = expectedResources.schedulerJobs.length > 0 ? discoverSchedulerJobGroup(context.projectId, resolveSearchCloudRunDeployConfig(context).region, runCommand) : {
    resources: [],
    warnings: []
  };
  const eventarcTriggerRegionResult = resolveSearchEventarcDiscoveryRegion(context, {
    ...options,
    runCommand
  });
  const eventarcTriggersResult = discoverEventarcTriggerGroup(context.projectId, eventarcTriggerRegionResult.region, runCommand);
  const providerTargetsResult = await discoverProviderTargetsWithFallback(context, provider, {
    discoverProviderTargetsDirectImpl: options.discoverProviderTargetsDirectImpl,
    discoverProviderTargetsViaSearchApiImpl: options.discoverProviderTargetsImpl,
    fetchImpl: options.fetchImpl,
    getSecretImpl: options.getSecretImpl,
    runCommand
  });
  warnings.push(...collectDiscoveryWarnings(servicesResult.warnings, jobsResult.warnings, taskQueuesResult.warnings, schedulerJobsResult.warnings, eventarcTriggerRegionResult.warnings, eventarcTriggersResult.warnings, providerTargetsResult.warnings));
  return {
    warnings,
    cloudRunJobs: expectedResources.cloudRunJobs.map(resource => toRemoteStatus(resource, jobsResult.resources)),
    cloudRunServices: expectedResources.cloudRunServices.map(resource => toRemoteStatus(resource, servicesResult.resources)),
    eventarcTriggers: expectedResources.eventarcTriggers.map(resource => toRemoteStatus(resource, eventarcTriggersResult.resources)),
    schedulerJobs: expectedResources.schedulerJobs.map(resource => toRemoteStatus(resource, schedulerJobsResult.resources)),
    taskQueues: expectedResources.taskQueues.map(resource => toRemoteStatus(resource, taskQueuesResult.resources)),
    providerTargets: expectedResources.providerTargets.map(resource => toRemoteStatus(resource, providerTargetsResult.targets))
  };
};
export default {
  discoverSearchRemoteResources,
  getExpectedSearchResources
};