@puls-atlas/cli
Version:
The Puls Atlas CLI tool for managing Atlas projects
405 lines • 22.1 kB
JavaScript
import { isPlainObject } from 'es-toolkit/compat';
import { discoverSearchRemoteResources, getExpectedSearchResources } from './discovery.js';
import { resolveSearchDlqBucketProvisioning } from './artifactBucket.js';
import { inspectSearchMapperManifest } from './manifestValidation.js';
import { getSearchProvider } from './providers/index.js';
import { hasScheduledSearchSources } from './sources/scheduled.js';
import { normalizeSearchReleaseConfig, resolveSearchReleaseTarget } from './release.js';
import { resolveSearchRuntimeSources } from './sources/runtimeConfig.js';
import { ATLAS_SEARCH_TASK_QUEUE_DEFAULTS } from '../../utils/taskQueue.js';
import { normalizeOptionalString } from '../../utils/value.js';
import { SEARCH_API_SERVICE_NAME, SEARCH_SOURCE_RUNNER_JOB_NAME, SEARCH_SYNC_SERVICE_NAME } from './resourceNames.js';
import { resolveSearchCloudRunDeployConfig, resolveSearchTaskQueueConfig, SHARED_REQUEST_AUTH_SECRET } from './deploymentConfig.js';
import { DEFAULT_SEARCH_SYNC_FAILURE_COLLECTION_PATH, DEFAULT_SEARCH_SYNC_FAILURE_RECENT_LIMIT } from './config/searchConfig.js';
import { PULS_ATLAS_DEFAULT_REGION_ENV, PULS_ATLAS_DLQ_BUCKET_ENV, PULS_ATLAS_DLQ_RETENTION_DAYS_ENV, PULS_ATLAS_PROJECT_ID_ENV, PULS_ATLAS_SEARCH_CONFIG_URI_ENV, PULS_ATLAS_SEARCH_INDEXES_ENV, PULS_ATLAS_SEARCH_MAPPER_MANIFEST_URI_ENV, PULS_ATLAS_SEARCH_PROVIDER_ENV, SEARCH_SYNC_TASK_LOCATION_ENV, SEARCH_SYNC_TASK_MAX_ATTEMPTS_ENV, SEARCH_SYNC_TASK_QUEUE_ENV } from './runtimeEnv.js';
const SEARCH_RUNTIME_CONFIG_VERSION = 2;
const SEARCH_SYNC_DLQ_REPLAY_PREFIX = 'search/sync/replay/<collection>/<documentId>/<syncVersion>/';
const PRIVATE_SEARCH_PROVIDER_HOST_SUFFIXES = ['.internal', '.svc.cluster.local'];
export { resolveSearchCloudRunDeployConfig, resolveSearchTaskQueueConfig, SHARED_REQUEST_AUTH_SECRET } from './deploymentConfig.js';
export const getSearchResourceGroups = context => {
const provider = getSearchProvider(context.config.provider);
const scheduledSourcesEnabled = hasScheduledSearchSources(context.config);
return [{
key: 'cloudRunServices',
label: 'Cloud Run services'
}, {
key: 'cloudRunJobs',
label: 'Cloud Run jobs'
}, {
key: 'taskQueues',
label: 'Cloud Tasks queues'
}, ...(scheduledSourcesEnabled ? [{
key: 'schedulerJobs',
label: 'Cloud Scheduler jobs'
}] : []), {
key: 'eventarcTriggers',
label: 'Eventarc triggers'
}, {
key: 'providerTargets',
label: provider.descriptor.labels.targetGroup
}];
};
export const createSearchRemoteSummary = (context, remoteResources) => getSearchResourceGroups(context).map(group => {
const resources = remoteResources[group.key] ?? [];
return {
found: getFoundResourceCount(resources),
key: group.key,
label: group.label,
total: resources.length,
unknown: getUnknownResourceCount(resources)
};
});
export const hasLimitedSearchRemoteDiscovery = remoteSummary => remoteSummary.some(resourceGroup => (resourceGroup.unknown ?? 0) > 0);
const isPrivateIpv4Address = hostName => {
const octets = hostName.split('.').map(Number);
if (octets.length !== 4 || octets.some(octet => Number.isNaN(octet))) {
return false;
}
if (octets[0] === 10) {
return true;
}
if (octets[0] === 172 && octets[1] >= 16 && octets[1] <= 31) {
return true;
}
return octets[0] === 192 && octets[1] === 168;
};
const isPrivateSearchProviderHost = host => {
if (typeof host !== 'string') {
return false;
}
let hostname;
try {
hostname = new URL(host).hostname.toLowerCase();
} catch {
return false;
}
return hostname === 'localhost' || PRIVATE_SEARCH_PROVIDER_HOST_SUFFIXES.some(suffix => hostname.endsWith(suffix)) || isPrivateIpv4Address(hostname);
};
export const getSearchPrivateProviderConnectivityIssues = (context, providerAccess) => {
if (normalizeOptionalString(context?.config?.provider) !== 'typesense') {
return [];
}
const providerHost = typeof providerAccess?.host === 'string' ? normalizeOptionalString(providerAccess.host) : null;
if (!providerHost || !isPrivateSearchProviderHost(providerHost)) {
return [];
}
const cloudRunVpcAccess = resolveSearchCloudRunDeployConfig(context).vpcAccess;
if (cloudRunVpcAccess?.network) {
return [];
}
const providerRuntimePlatform = normalizeOptionalString(context?.config?.deploy?.providerRuntime?.platform);
return [`Configured Typesense host ${providerHost} is private, but deploy.cloudRun.vpcAccess is missing for project ${context?.projectId ?? 'unknown'}. ` + `Atlas search Cloud Run services and jobs cannot resolve private Typesense endpoints without VPC access. ${providerRuntimePlatform ? 'Rerun "atlas search provider deploy" to repersist the managed private runtime networking, or restore deploy.cloudRun.vpcAccess manually before rerunning "atlas search apply".' : 'Add deploy.providerRuntime and deploy.cloudRun.vpcAccess, or run "atlas search provider deploy --platform compute --zone <zone>" to persist the managed private runtime networking before rerunning "atlas search apply".'}`];
};
const normalizeSearchNameList = names => [...new Set((Array.isArray(names) ? names : []).map(name => normalizeOptionalString(name)).filter(Boolean))].sort((left, right) => left.localeCompare(right));
const resolveIndexNamesForRuntimeEntry = runtimeEntry => Array.isArray(runtimeEntry?.indexes) ? runtimeEntry.indexes.map(indexName => normalizeOptionalString(indexName)).filter(Boolean) : [];
const createSourceReplayMessage = (indexName, sourceNames) => {
const normalizedSourceNames = normalizeSearchNameList(sourceNames);
if (normalizedSourceNames.length === 0) {
return `Inspect the enabled Atlas search sources for ${indexName}.`;
}
if (normalizedSourceNames.length === 1) {
return `Run "atlas search run source --source ${normalizedSourceNames[0]}" to enqueue documents for ${indexName}, ` + 'or wait for its schedule.';
}
return `Run "atlas search run source --source <name>" for each of ${normalizedSourceNames.join(', ')} to enqueue documents for ${indexName}, ` + 'or wait for their schedules.';
};
const createSourceReindexMessage = (indexName, sourceNames) => {
const normalizedSourceNames = normalizeSearchNameList(sourceNames);
if (normalizedSourceNames.length === 1) {
return `Run "atlas search run reindex --indexes ${indexName}" to build the updated candidate target from source ${normalizedSourceNames[0]}.`;
}
return createSourceReplayMessage(indexName, normalizedSourceNames);
};
const supportsSourceBackedReindex = sourceConfig => sourceConfig?.capabilities?.supportsBackfill !== true && sourceConfig?.capabilities?.supportsIncremental === true && normalizeOptionalString(sourceConfig?.schedule) !== null;
export const resolveSearchBackfillSupportEntries = (context, indexNames = Object.keys(context?.config?.indexes ?? {})) => {
const runtimeConfig = resolveSearchRuntimeConfig(context);
const enabledCollectionEntries = Object.entries(runtimeConfig.collections ?? {}).filter(([, collectionConfig]) => collectionConfig?.enabled !== false);
const enabledSourceEntries = Object.entries(runtimeConfig.sources ?? {}).filter(([, sourceConfig]) => sourceConfig?.enabled !== false);
return normalizeSearchNameList(indexNames).map(indexName => {
const backfillableCollectionNames = enabledCollectionEntries.filter(([, collectionConfig]) => resolveIndexNamesForRuntimeEntry(collectionConfig).includes(indexName)).map(([collectionName]) => collectionName);
const backfillableSourceNames = enabledSourceEntries.filter(([, sourceConfig]) => sourceConfig?.capabilities?.supportsBackfill === true && resolveIndexNamesForRuntimeEntry(sourceConfig).includes(indexName)).map(([sourceName]) => sourceName);
const enabledNonBackfillableSourceNames = enabledSourceEntries.filter(([, sourceConfig]) => sourceConfig?.capabilities?.supportsBackfill !== true && resolveIndexNamesForRuntimeEntry(sourceConfig).includes(indexName)).map(([sourceName]) => sourceName);
const sourceReindexSourceNames = enabledSourceEntries.filter(([, sourceConfig]) => supportsSourceBackedReindex(sourceConfig) && resolveIndexNamesForRuntimeEntry(sourceConfig).includes(indexName)).map(([sourceName]) => sourceName);
const backfillableSourceScope = normalizeSearchNameList([...backfillableCollectionNames, ...backfillableSourceNames]);
return {
indexName,
supportsBackfill: backfillableSourceScope.length > 0,
supportsSourceReindex: normalizeSearchNameList(sourceReindexSourceNames).length === 1,
backfillableSourceNames: backfillableSourceScope,
enabledSourceNames: normalizeSearchNameList([...backfillableSourceScope, ...enabledNonBackfillableSourceNames]),
sourceReindexSourceNames: normalizeSearchNameList(sourceReindexSourceNames)
};
});
};
export const createSearchNonBackfillableIndexGuidanceEntry = indexEntry => indexEntry?.supportsSourceReindex ? createSourceReindexMessage(normalizeOptionalString(indexEntry?.indexName) ?? 'unknown', indexEntry?.sourceReindexSourceNames ?? []) : createSourceReplayMessage(normalizeOptionalString(indexEntry?.indexName) ?? 'unknown', indexEntry?.enabledSourceNames ?? []);
export const createUnsupportedSearchBackfillIndexMessage = (indexEntry, action = 'backfill') => {
const indexName = normalizeOptionalString(indexEntry?.indexName) ?? 'unknown';
const sourceNames = normalizeSearchNameList(indexEntry?.enabledSourceNames ?? []);
const remediationMessage = indexEntry?.supportsSourceReindex ? createSourceReindexMessage(indexName, indexEntry?.sourceReindexSourceNames ?? sourceNames) : createSourceReplayMessage(indexName, sourceNames);
if (action === 'reindex') {
return `Atlas search reindex cannot run for ${indexName} through the shared backfill job` + `${sourceNames.length > 0 ? ` because it is backed by source-managed indexes: ${sourceNames.join(', ')}.` : '.'} ${remediationMessage}`;
}
return `Atlas search ${action} cannot run for ${indexName} because its enabled sources do not support backfill` + `${sourceNames.length > 0 ? `: ${sourceNames.join(', ')}.` : '.'} ${remediationMessage}`;
};
export const createUnsupportedSearchReindexIndexMessage = indexEntry => {
const indexName = normalizeOptionalString(indexEntry?.indexName) ?? 'unknown';
const sourceNames = normalizeSearchNameList(indexEntry?.enabledSourceNames ?? []);
const reindexSourceNames = normalizeSearchNameList(indexEntry?.sourceReindexSourceNames ?? []);
if (reindexSourceNames.length > 1) {
return `Atlas search reindex cannot run for ${indexName} because it is backed by multiple scheduled incremental sources: ${reindexSourceNames.join(', ')}. ` + 'Run "atlas search run source --source <name>" per source, or narrow the requested index scope.';
}
return `Atlas search reindex cannot run for ${indexName}` + `${sourceNames.length > 0 ? ` because its enabled sources are not eligible for candidate reindex orchestration: ${sourceNames.join(', ')}.` : '.'} ` + `${createSourceReplayMessage(indexName, sourceNames)}`;
};
export const logLimitedSearchRemoteDiscoveryNotice = (remoteSummary, loggerImpl, message = 'Remote discovery is currently limited from this machine.') => {
if (!hasLimitedSearchRemoteDiscovery(remoteSummary)) {
return false;
}
loggerImpl.info(message);
return true;
};
export const createSearchRemoteSummaryEntry = (resourceGroup, {
successVerb = 'already present'
} = {}) => {
if ((resourceGroup.unknown ?? 0) > 0) {
return {
label: resourceGroup.label,
tone: 'warning',
value: 'could not be verified remotely from this machine.'
};
}
return {
label: resourceGroup.label,
tone: resourceGroup.found === resourceGroup.total ? 'success' : 'warning',
value: `${resourceGroup.found}/${resourceGroup.total} ${successVerb}.`
};
};
const getFoundResourceCount = resources => resources.filter(resource => resource.remoteStatus === 'found').length;
const getUnknownResourceCount = resources => resources.filter(resource => resource.remoteStatus === 'unknown').length;
export const getSearchProvisionBlockingIssues = (context, providerAccess, manifestInspection) => {
const issues = [];
const indexNames = Object.keys(context.config.indexes);
const provider = getSearchProvider(context.config.provider);
if (indexNames.length === 0) {
issues.push('Atlas search provisioning requires at least one configured index in the resolved search ' + 'config and any referenced search workloads.');
}
issues.push(...provider.getProvisionAccessIssues(providerAccess));
issues.push(...getSearchPrivateProviderConnectivityIssues(context, providerAccess));
if (manifestInspection?.status !== 'missing') {
issues.push(...(manifestInspection?.issues ?? []));
}
return issues;
};
export const createSearchProvisionPlan = (context, expectedResources, remoteResources, providerAccess, manifestInspection) => ({
version: 1,
createdAt: new Date().toISOString(),
environment: context.environment,
expectedResources,
feature: context.featureName,
projectId: context.projectId,
release: normalizeSearchReleaseConfig(context.config.release),
releaseTarget: resolveSearchReleaseTarget(context.config.release),
readiness: {
provider: context.config.provider,
remoteSummary: createSearchRemoteSummary(context, remoteResources),
mapperManifest: {
status: manifestInspection.status,
uri: manifestInspection.manifestUri
},
providerAccess: getSearchProvider(context.config.provider).getAccessReadiness(providerAccess)
}
});
export const loadSearchProvisionState = async context => {
const expectedResources = getExpectedSearchResources(context);
const provider = getSearchProvider(context.config.provider);
const [remoteResources, providerAccess, manifestInspection] = await Promise.all([discoverSearchRemoteResources(context), provider.resolveAccess(context), inspectSearchMapperManifest(context)]);
const blockingIssues = getSearchProvisionBlockingIssues(context, providerAccess, manifestInspection);
const plan = createSearchProvisionPlan(context, expectedResources, remoteResources, providerAccess, manifestInspection);
return {
blockingIssues,
expectedResources,
manifestInspection,
plan,
providerAccess,
remoteResources
};
};
const resolveSharedRequestAuthSecretReference = () => ({
env: 'REQUEST_AUTH_TOKEN',
secretName: SHARED_REQUEST_AUTH_SECRET,
services: [SEARCH_API_SERVICE_NAME, SEARCH_SYNC_SERVICE_NAME, SEARCH_SOURCE_RUNNER_JOB_NAME],
source: 'atlas search apply',
version: 'latest'
});
const normalizeStringArray = value => {
if (!Array.isArray(value)) {
return [];
}
return [...new Set(value.map(entry => normalizeOptionalString(entry)).filter(Boolean))];
};
const normalizeMapperConfig = mapperConfig => ({
export: normalizeOptionalString(mapperConfig?.export),
source: normalizeOptionalString(mapperConfig?.source)
});
const normalizeSearchIndexFields = fields => {
if (!Array.isArray(fields)) {
return [];
}
return fields.filter(fieldConfig => fieldConfig && isPlainObject(fieldConfig)).map(fieldConfig => ({
facet: fieldConfig.facet === true,
filter: fieldConfig.filter === true,
group: fieldConfig.group === true,
name: fieldConfig.name,
optional: fieldConfig.optional === true,
sort: fieldConfig.sort === true,
type: fieldConfig.type
}));
};
const normalizeDefaultSort = value => {
if (!Array.isArray(value)) {
return [];
}
return value.filter(entry => entry && isPlainObject(entry)).map(entry => ({
direction: normalizeOptionalString(entry.direction)?.toLowerCase() ?? 'asc',
field: entry.field
})).filter(entry => normalizeOptionalString(entry.field) !== null);
};
export const resolveSearchRuntimeConfig = context => ({
collections: Object.fromEntries(Object.entries(context.config.collections ?? {}).map(([collectionName, collectionConfig]) => [collectionName, {
enabled: collectionConfig?.enabled !== false,
indexes: normalizeStringArray(collectionConfig?.indexes),
mapper: normalizeOptionalString(collectionConfig?.mapper),
routeTemplate: normalizeOptionalString(collectionConfig?.routeTemplate)
}])),
indexes: Object.fromEntries(Object.entries(context.config.indexes).map(([indexName, indexConfig]) => [indexName, {
defaultSort: normalizeDefaultSort(indexConfig?.defaultSort),
fields: normalizeSearchIndexFields(indexConfig?.fields),
facets: normalizeStringArray(indexConfig?.facets),
filterBy: normalizeStringArray(indexConfig?.filterBy),
groupBy: normalizeStringArray(indexConfig?.groupBy),
queryBy: normalizeStringArray(indexConfig?.queryBy)
}])),
mappers: Object.fromEntries(Object.entries(context.config.mappers ?? {}).map(([mapperName, mapperConfig]) => [mapperName, normalizeMapperConfig(mapperConfig)])),
provider: context.config.provider,
release: normalizeSearchReleaseConfig(context.config.release),
sources: resolveSearchRuntimeSources(context),
version: Math.max(context.config.version ?? 1, SEARCH_RUNTIME_CONFIG_VERSION)
});
export const resolveSearchProviderSecretReference = context => getSearchProvider(context.config.provider).getRuntimeSecretReference(context);
export const resolveSearchDlqBucketConfig = context => {
const bucketState = resolveSearchDlqBucketProvisioning(context);
return {
location: bucketState.location,
name: bucketState.bucketName,
retentionDays: bucketState.retentionDays
};
};
export const resolveSearchSyncFailureInspectionConfig = context => {
const syncStateConfig = context?.config?.deploy?.syncState ?? {};
const configuredPreviewLimit = Number.parseInt(syncStateConfig.recentLimit ?? DEFAULT_SEARCH_SYNC_FAILURE_RECENT_LIMIT, 10);
const hasExplicitCollectionPath = Object.hasOwn(syncStateConfig, 'collectionPath');
return {
collectionPath: hasExplicitCollectionPath ? normalizeOptionalString(syncStateConfig.collectionPath) ?? null : DEFAULT_SEARCH_SYNC_FAILURE_COLLECTION_PATH,
recentLimit: Number.isInteger(configuredPreviewLimit) && configuredPreviewLimit > 0 ? configuredPreviewLimit : DEFAULT_SEARCH_SYNC_FAILURE_RECENT_LIMIT
};
};
export const createSearchFailureHandlingSummary = context => {
const taskQueueConfig = resolveSearchTaskQueueConfig(context);
const dlqBucketConfig = resolveSearchDlqBucketConfig(context);
const dlqLocation = dlqBucketConfig.location ?? taskQueueConfig.location;
const syncFailureInspectionConfig = resolveSearchSyncFailureInspectionConfig(context);
return [{
label: 'Sync queue',
value: `${taskQueueConfig.name} (${taskQueueConfig.location}, ` + `${ATLAS_SEARCH_TASK_QUEUE_DEFAULTS.maxAttempts} max attempts)`
}, {
label: 'DLQ bucket',
value: `${dlqBucketConfig.name} (${dlqLocation}, ` + `${dlqBucketConfig.retentionDays}-day retention)`
}, {
label: 'Dead-letter handling',
value: 'Exhausted retries persist Firestore failure state with DLQ payload pointers.'
}, {
label: 'Replay prefix',
value: SEARCH_SYNC_DLQ_REPLAY_PREFIX
}, {
label: 'Repair command',
value: 'atlas search repair sync --collection <collection> --document-id <document-id>'
}, {
label: 'Failure state collection',
value: syncFailureInspectionConfig.collectionPath ?? 'not configured for CLI inspection'
}];
};
export const resolveSearchRuntimeEnvironmentVariables = context => {
const runtimeConfig = resolveSearchRuntimeConfig(context);
const cloudRunConfig = resolveSearchCloudRunDeployConfig(context);
const dlqBucketConfig = resolveSearchDlqBucketConfig(context);
const providerSecretReference = resolveSearchProviderSecretReference(context);
const taskQueueConfig = resolveSearchTaskQueueConfig(context);
if (!providerSecretReference) {
throw new Error(`Atlas search provider "${context.config.provider}" did not return a runtime secret reference.`);
}
const runtimeEnvironmentVariables = [{
name: PULS_ATLAS_PROJECT_ID_ENV,
required: true,
source: 'atlas search apply',
value: context.projectId
}, {
name: PULS_ATLAS_DEFAULT_REGION_ENV,
required: true,
source: 'atlas search apply',
value: cloudRunConfig.region
}, {
name: PULS_ATLAS_SEARCH_PROVIDER_ENV,
required: true,
source: 'atlas search apply',
value: runtimeConfig.provider
}, {
name: PULS_ATLAS_SEARCH_INDEXES_ENV,
required: true,
source: 'atlas search apply',
value: Object.keys(runtimeConfig.indexes).join(',')
}, {
name: PULS_ATLAS_SEARCH_MAPPER_MANIFEST_URI_ENV,
required: true,
source: 'atlas search apply',
value: cloudRunConfig.mapperManifestUri ?? ''
}, {
name: SEARCH_SYNC_TASK_LOCATION_ENV,
required: true,
source: 'atlas search apply',
value: taskQueueConfig.location
}, {
name: SEARCH_SYNC_TASK_QUEUE_ENV,
required: true,
source: 'atlas search apply',
value: taskQueueConfig.name
}, {
name: SEARCH_SYNC_TASK_MAX_ATTEMPTS_ENV,
required: true,
source: 'atlas search apply',
value: String(ATLAS_SEARCH_TASK_QUEUE_DEFAULTS.maxAttempts)
}, {
name: PULS_ATLAS_DLQ_BUCKET_ENV,
required: true,
source: 'atlas search apply',
value: dlqBucketConfig.name
}, {
name: PULS_ATLAS_DLQ_RETENTION_DAYS_ENV,
required: true,
source: 'atlas search apply',
value: String(dlqBucketConfig.retentionDays)
}, {
name: PULS_ATLAS_SEARCH_CONFIG_URI_ENV,
required: true,
source: 'atlas search apply',
value: cloudRunConfig.runtimeConfigUri ?? ''
}, {
name: providerSecretReference.env,
required: true,
source: 'atlas search apply',
value: providerSecretReference.secretName
}];
return runtimeEnvironmentVariables;
};
export const resolveSearchRuntimeContract = context => ({
localDevelopmentFallbackEnvVars: getSearchProvider(context.config.provider).getLocalDevelopmentFallbackEnvVars(),
requiredEnvironmentVariables: resolveSearchRuntimeEnvironmentVariables(context),
requiredSecretEnvironmentVariables: [resolveSharedRequestAuthSecretReference()]
});