@puls-atlas/cli
Version:
The Puls Atlas CLI tool for managing Atlas projects
502 lines • 20.3 kB
JavaScript
import fs from 'fs';
import path from 'path';
import inquirer from 'inquirer';
import { isEqual } from 'es-toolkit';
import * as features from '../../../utils/feature.js';
import { readJsonFile, writeJsonFile, writeTextFile } from '../../../utils/file.js';
import { logger, normalizeOptionalString } from '../../../utils/index.js';
import { applySearchProjectConfigPatch } from '../config/searchConfig.js';
import { validateSearchRootSection, validateSearchWorkloadConfig } from '../config/searchValidation.js';
const DEFAULT_HTTP_SOURCE_METHOD = 'get';
const DEFAULT_HTTP_SOURCE_SCHEDULE = '*/5 * * * *';
const DEFAULT_HTTP_SOURCE_SYNC_CLASS = 'delta-merge';
const HTTP_SOURCE_METHOD_CHOICES = ['get', 'post'];
const HTTP_SOURCE_SYNC_CLASS_CHOICES = ['append-only', 'delta-merge'];
const MAPPER_NAME_PATTERN = /^[A-Za-z0-9_-]+$/;
const normalizePathSeparators = targetPath => targetPath.split(path.sep).join('/');
const sanitizeSourceName = value => String(value ?? '').trim().toLowerCase().replace(/[^a-z0-9_]+/g, '_').replace(/_{2,}/g, '_').replace(/^_+|_+$/g, '');
const toPascalCase = value => String(value ?? '').split(/[^A-Za-z0-9]+/).map(segment => segment.trim()).filter(Boolean).map(segment => segment[0].toUpperCase() + segment.slice(1)).join('');
const createDefaultMapperName = sourceName => {
const normalizedSourceName = sanitizeSourceName(sourceName).replace(/_(firestore|http|sql)$/u, '');
const segments = normalizedSourceName.split('_').filter(Boolean);
if (segments.length === 0) {
return 'source';
}
const lastSegment = segments.at(-1);
if (lastSegment.endsWith('s') && lastSegment.length > 1) {
segments[segments.length - 1] = lastSegment.slice(0, -1);
}
return segments.join('_');
};
const createDefaultIndexName = sourceName => {
const normalizedSourceName = sanitizeSourceName(sourceName).replace(/_(firestore|http|sql)$/u, '');
return `${normalizedSourceName || 'source'}_search`;
};
const createDefaultConnectionSecretName = sourceName => `atlas-search-${sanitizeSourceName(sourceName).replaceAll('_', '-')}`;
const createDefaultDescription = sourceName => {
const normalizedName = sanitizeSourceName(sourceName).replace(/_http$/u, '');
const readableName = normalizedName.replaceAll('_', ' ') || 'source';
return `Incremental HTTP polling for ${readableName}.`;
};
const parseCommaSeparatedValues = value => {
if (Array.isArray(value)) {
return [...new Set(value.flatMap(parseCommaSeparatedValues).filter(Boolean))];
}
const normalizedValue = normalizeOptionalString(value);
if (!normalizedValue) {
return [];
}
return [...new Set(normalizedValue.split(',').map(entry => entry.trim()).filter(Boolean))];
};
const normalizeMapperName = value => {
const mapperName = normalizeOptionalString(value);
if (!mapperName) {
return null;
}
if (!MAPPER_NAME_PATTERN.test(mapperName)) {
throw new Error('Atlas search source add requires mapper names to use only letters, numbers, hyphens, or underscores.');
}
return mapperName;
};
const normalizeHttpSourceMethod = value => {
const method = normalizeOptionalString(value)?.toLowerCase() ?? DEFAULT_HTTP_SOURCE_METHOD;
if (!HTTP_SOURCE_METHOD_CHOICES.includes(method)) {
throw new Error(`Atlas search source add supports only HTTP methods ${HTTP_SOURCE_METHOD_CHOICES.join(' or ')} right now.`);
}
return method;
};
const normalizeHttpSourceSyncClass = value => {
const syncClass = normalizeOptionalString(value)?.toLowerCase() ?? DEFAULT_HTTP_SOURCE_SYNC_CLASS;
if (!HTTP_SOURCE_SYNC_CLASS_CHOICES.includes(syncClass)) {
throw new Error(`Atlas search source add supports only HTTP sync classes ${HTTP_SOURCE_SYNC_CLASS_CHOICES.join(' or ')} right now.`);
}
return syncClass;
};
const createMapperStub = mapperExportName => `${['type SourceDocument = Record<string, unknown>;', '', '// TODO: Map the HTTP payload fields that should become part of the search document.', `export const ${mapperExportName} = (source: SourceDocument) => ({`, " id: typeof source.id === 'string' ? source.id : String(source.id ?? source.uid ?? source.documentId ?? '')", '});', '', `export default ${mapperExportName};`].join('\n')}
`;
const resolveMapperWritePlan = (mapperFilePath, mapperSource) => {
if (!fs.existsSync(mapperFilePath)) {
return {
action: 'create',
reason: null,
write: true
};
}
const existingSource = fs.readFileSync(mapperFilePath, 'utf-8');
if (existingSource === mapperSource) {
return {
action: 'unchanged',
reason: null,
write: false
};
}
return {
action: 'keep-existing',
reason: 'Mapper file already exists with custom content. Atlas kept the existing mapper unchanged.',
write: false
};
};
const createDefaultIndexConfig = indexName => ({
defaultSort: [{
direction: 'desc',
field: 'updatedAt'
}],
fields: [{
name: 'id',
type: 'string'
}, {
name: 'title',
type: 'string'
}, {
name: 'updatedAt',
optional: true,
sort: true,
type: 'int64'
}],
name: indexName,
queryBy: ['title']
});
const createHttpSourceDefinitionHttpConfig = method => ({
headers: {
Accept: 'application/json'
},
...(method === 'post' ? {
body: {}
} : {}),
method
});
const createHttpWorkload = (input, mapperRelativePath, mapperExportName) => validateSearchWorkloadConfig({
source: {
description: createDefaultDescription(input.sourceName),
http: {
...createHttpSourceDefinitionHttpConfig(input.method),
incremental: {
strategy: 'cursor',
cursor: {
request: input.method === 'post' ? {
bodyPath: 'cursor.token'
} : {
queryParam: 'cursor'
},
response: {
hasMorePath: 'meta.hasMore',
nextCursorPath: 'meta.nextCursor'
}
},
response: {
itemsPath: 'data.items'
},
mapping: {
afterPath: '$',
...(input.syncClass === 'delta-merge' ? {
beforePath: 'previous',
deletedPath: 'deleted'
} : {}),
idPath: 'id',
versionPath: 'updatedAt'
}
}
},
name: input.sourceName,
syncClass: input.syncClass,
type: 'http'
},
mapper: {
export: mapperExportName,
name: input.mapperName,
source: mapperRelativePath
},
index: createDefaultIndexConfig(input.indexName)
}, `workloads/${input.sourceName}.json`);
const createHttpSourceBinding = input => ({
enabled: true,
schedule: input.schedule,
http: {
connectionSecret: input.connectionSecret,
url: input.url
}
});
const patchSearchRootConfigWithWorkload = (rootConfig, workloadRelativePath) => {
const validatedRootConfig = validateSearchRootSection(rootConfig);
const normalizedWorkloadPath = normalizePathSeparators(workloadRelativePath);
const workloadEntries = Array.isArray(validatedRootConfig.workloads) ? validatedRootConfig.workloads : [];
if (workloadEntries.includes(normalizedWorkloadPath)) {
return {
config: rootConfig,
updated: false
};
}
return {
updated: true,
config: {
...rootConfig,
workloads: [...workloadEntries, normalizedWorkloadPath]
}
};
};
const logHttpSourceAddSummary = (loggerImpl, payload, options = {}) => {
const detailOnly = options.detailOnly ?? false;
loggerImpl.summary(options.dryRun === true ? 'HTTP source add dry-run summary' : 'HTTP source add summary', [{
label: 'Project',
value: payload.projectId
}, payload.environment ? {
label: 'Environment',
value: payload.environment
} : null, {
label: 'Config',
value: payload.configPath
}, {
label: 'Source name',
value: payload.sourceName
}, {
label: 'Index',
value: payload.indexName
}, {
label: 'Mapper name',
value: payload.mapperName
}, {
label: 'HTTP method',
value: payload.method.toUpperCase()
}, {
label: 'URL',
value: payload.url
}, {
label: 'Sync class',
value: payload.syncClass
}, {
label: 'Schedule',
value: payload.schedule
}, {
label: 'Connection secret',
value: payload.connectionSecret
}], {
detailOnly
});
loggerImpl.summary('Planned file actions', [{
label: payload.workloadPath,
value: payload.workloadAction
}, {
label: payload.configPath,
value: payload.rootConfigAction
}, {
label: payload.mapperPath,
value: payload.mapperAction
}], {
detailOnly
});
};
const assertProjectScopedSourceBindingContext = context => {
const projectId = normalizeOptionalString(context.projectId);
if (projectId) {
return projectId;
}
throw new Error('Atlas search source add --type http requires a selected project because scheduled sources are stored under projects.<projectId>.sourceBindings.');
};
const resolveRequestedIndexName = value => {
const requestedIndexes = parseCommaSeparatedValues(value);
if (requestedIndexes.length > 1) {
throw new Error('Atlas search source add currently supports exactly one logical index per workload.');
}
return requestedIndexes[0] ?? null;
};
const resolveHttpSourceInput = async (context, options = {}, dependencies = {}) => {
const prompt = dependencies.prompt ?? inquirer.prompt;
let sourceName = sanitizeSourceName(options.name ?? options.sourceName);
let indexName = resolveRequestedIndexName(options.index ?? options.indexes) ?? normalizeOptionalString(options.indexName);
let mapperName = normalizeMapperName(options.mapper);
let method = normalizeOptionalString(options.method)?.toLowerCase() ?? null;
let url = normalizeOptionalString(options.url);
let syncClass = normalizeOptionalString(options.syncClass)?.toLowerCase() ?? null;
let schedule = normalizeOptionalString(options.schedule);
let connectionSecret = normalizeOptionalString(options.connectionSecret);
if (!sourceName && options.interactive === true) {
const answers = await prompt([{
type: 'input',
name: 'sourceName',
message: 'HTTP source name',
validate: value => sanitizeSourceName(value) ? true : 'Enter a stable source name. Atlas will normalize it to snake_case.'
}]);
sourceName = sanitizeSourceName(answers.sourceName);
}
if (!sourceName) {
throw new Error('Atlas search source add requires --name <sourceName> or --interactive for HTTP onboarding.');
}
if (options.interactive === true) {
const answers = await prompt([!indexName ? {
type: 'input',
name: 'indexName',
message: 'Logical index name',
default: createDefaultIndexName(sourceName),
validate: value => normalizeOptionalString(value) ? true : 'Enter a non-empty index name.'
} : null, options.mapper === undefined ? {
type: 'input',
name: 'mapperName',
message: 'Mapper name',
default: createDefaultMapperName(sourceName),
validate: value => {
try {
return normalizeMapperName(value) ? true : 'Enter a mapper name.';
} catch (error) {
return error.message;
}
}
} : null, options.method === undefined ? {
type: 'list',
name: 'method',
message: 'HTTP request method',
choices: HTTP_SOURCE_METHOD_CHOICES.map(choice => ({
name: choice.toUpperCase(),
value: choice
})),
default: DEFAULT_HTTP_SOURCE_METHOD
} : null, !url ? {
type: 'input',
name: 'url',
message: 'HTTP endpoint URL',
validate: value => normalizeOptionalString(value) ? true : 'Enter a non-empty URL.'
} : null, options.syncClass === undefined ? {
type: 'list',
name: 'syncClass',
message: 'Sync class',
choices: HTTP_SOURCE_SYNC_CLASS_CHOICES,
default: DEFAULT_HTTP_SOURCE_SYNC_CLASS
} : null, options.schedule === undefined ? {
type: 'input',
name: 'schedule',
message: 'Cloud Scheduler cron expression',
default: DEFAULT_HTTP_SOURCE_SCHEDULE,
validate: value => normalizeOptionalString(value) ? true : 'Enter a non-empty cron expression.'
} : null, options.connectionSecret === undefined ? {
type: 'input',
name: 'connectionSecret',
message: 'Connection secret name',
default: createDefaultConnectionSecretName(sourceName),
validate: value => normalizeOptionalString(value) ? true : 'Enter a non-empty secret name.'
} : null].filter(Boolean));
indexName = indexName ?? normalizeOptionalString(answers.indexName);
mapperName = mapperName ?? normalizeMapperName(answers.mapperName);
method = method ?? answers.method ?? null;
url = url ?? normalizeOptionalString(answers.url);
syncClass = syncClass ?? answers.syncClass ?? null;
schedule = schedule ?? normalizeOptionalString(answers.schedule);
connectionSecret = connectionSecret ?? normalizeOptionalString(answers.connectionSecret);
}
const normalizedIndexName = normalizeOptionalString(indexName) ?? createDefaultIndexName(sourceName);
if (!url) {
throw new Error('Atlas search source add requires --url <url> or --interactive for HTTP onboarding.');
}
return {
connectionSecret: connectionSecret ?? createDefaultConnectionSecretName(sourceName),
indexName: normalizedIndexName,
mapperName: mapperName ?? createDefaultMapperName(sourceName),
method: normalizeHttpSourceMethod(method),
schedule: schedule ?? DEFAULT_HTTP_SOURCE_SCHEDULE,
sourceName,
syncClass: normalizeHttpSourceSyncClass(syncClass),
url
};
};
const createHttpSourceArtifacts = input => {
const mapperExportName = `map${toPascalCase(input.mapperName)}ToSearchDocument`;
const workloadRelativePath = normalizePathSeparators(path.join('workloads', `${input.sourceName}.json`));
const mapperRelativePath = normalizePathSeparators(path.join('mappers', `${input.mapperName}.ts`));
return {
mapperRelativePath,
mapperSource: createMapperStub(mapperExportName),
sourceBinding: createHttpSourceBinding(input),
workload: createHttpWorkload(input, mapperRelativePath, mapperExportName),
workloadRelativePath
};
};
const assertHttpSourceDoesNotDrift = ({
context,
currentWorkload,
rootConfig,
selectedProjectId,
input,
artifacts,
workloadPath
}) => {
const existingSource = context.config.sources?.[input.sourceName] ?? null;
const existingMapper = context.config.mappers?.[input.mapperName] ?? null;
const existingIndex = context.config.indexes?.[input.indexName] ?? null;
const currentProjectSourceBinding = rootConfig.projects?.[selectedProjectId]?.sourceBindings?.[input.sourceName] ?? null;
if (existingSource && currentWorkload === null) {
throw new Error(`Atlas search source "${input.sourceName}" already exists in another loaded workload. Update it manually instead of re-running onboarding.`);
}
if (existingMapper && currentWorkload === null) {
throw new Error(`Atlas search mapper "${input.mapperName}" already exists in another loaded workload. Choose a new mapper name for this source.`);
}
if (existingIndex && currentWorkload === null) {
throw new Error(`Atlas search index "${input.indexName}" already exists in another loaded workload. Choose a new index name for this source.`);
}
if (currentWorkload && !isEqual(currentWorkload, artifacts.workload)) {
throw new Error(`Atlas search workload "${input.sourceName}" already exists in ${workloadPath} with different content. Update it manually instead of re-running onboarding.`);
}
if (currentProjectSourceBinding && !isEqual(currentProjectSourceBinding, artifacts.sourceBinding)) {
throw new Error(`Atlas search project binding for source "${input.sourceName}" already exists in projects.${selectedProjectId}.sourceBindings with different content. Update it manually instead of re-running onboarding.`);
}
};
export const runSearchHttpSourceAdd = async (options = {}, dependencies = {}, cwd = process.cwd()) => {
const loadFeatureContextImpl = dependencies.loadFeatureContext ?? features.loadFeatureContext;
const loggerImpl = dependencies.logger ?? logger;
const readJsonFileImpl = dependencies.readJsonFile ?? readJsonFile;
const writeJsonFileImpl = dependencies.writeJsonFile ?? writeJsonFile;
const writeTextFileImpl = dependencies.writeTextFile ?? writeTextFile;
let spinner;
try {
const context = await loadFeatureContextImpl('search', options, {
cwd
});
const selectedProjectId = assertProjectScopedSourceBindingContext(context);
const rootConfig = context.rootConfig ?? readJsonFileImpl(context.configPath);
const input = await resolveHttpSourceInput(context, options, {
prompt: dependencies.prompt
});
const artifacts = createHttpSourceArtifacts(input);
const workloadPath = path.join(path.dirname(context.configPath), artifacts.workloadRelativePath);
const currentWorkload = readJsonFileImpl(workloadPath, {
allowMissing: true
});
const mapperPath = path.join(path.dirname(context.configPath), artifacts.mapperRelativePath);
const mapperWritePlan = resolveMapperWritePlan(mapperPath, artifacts.mapperSource);
assertHttpSourceDoesNotDrift({
artifacts,
context,
currentWorkload,
input,
rootConfig,
selectedProjectId,
workloadPath
});
const rootConfigWithWorkloadPatch = patchSearchRootConfigWithWorkload(rootConfig, artifacts.workloadRelativePath);
const nextRootConfig = applySearchProjectConfigPatch(rootConfigWithWorkloadPatch.config, selectedProjectId, {
...(normalizeOptionalString(context.environment) ? {
environment: context.environment
} : {}),
sourceBindings: {
[input.sourceName]: artifacts.sourceBinding
}
});
const warnings = ['Atlas generated a starter HTTP workload with default cursor, response, mapping, and index fields. Review the new workload before running apply.'];
if (mapperWritePlan.reason) {
warnings.push(mapperWritePlan.reason);
}
spinner = loggerImpl.spinner(options.dryRun === true ? 'Preparing Atlas search http workload onboarding dry run...' : 'Adding http workload to Atlas search config...');
const workloadAction = currentWorkload === null ? options.dryRun ? 'would-create' : 'created' : 'unchanged';
const mapperAction = mapperWritePlan.action === 'create' ? options.dryRun ? 'would-create' : 'created' : mapperWritePlan.action === 'keep-existing' ? 'kept-existing' : 'reused-existing';
const rootConfigAction = !isEqual(rootConfig, nextRootConfig) ? options.dryRun ? 'would-update' : 'updated' : 'unchanged';
if (options.dryRun !== true) {
if (currentWorkload === null) {
writeJsonFileImpl(workloadPath, artifacts.workload);
}
if (rootConfigAction !== 'unchanged') {
writeJsonFileImpl(context.configPath, nextRootConfig);
}
if (mapperWritePlan.write) {
writeTextFileImpl(mapperPath, artifacts.mapperSource);
}
}
spinner.succeed(options.dryRun === true ? 'Atlas search HTTP workload onboarding dry run is ready.' : 'Atlas search HTTP workload onboarding completed.');
const result = {
configPath: context.configPath,
connectionSecret: input.connectionSecret,
environment: context.environment,
indexName: input.indexName,
mapperAction,
mapperName: input.mapperName,
mapperPath,
method: input.method,
projectId: context.projectId,
rootConfigAction,
schedule: input.schedule,
sourceName: input.sourceName,
status: options.dryRun === true ? 'dry-run' : 'updated',
syncClass: input.syncClass,
type: 'http',
url: input.url,
warnings,
workloadAction,
workloadPath
};
logHttpSourceAddSummary(loggerImpl, result, {
dryRun: options.dryRun === true
});
for (const warning of warnings) {
loggerImpl.warning(warning);
}
if (options.dryRun === true) {
loggerImpl.info('Dry run requested: no files were changed.');
}
return result;
} catch (error) {
if (spinner) {
spinner.fail('Failed to onboard HTTP workload into Atlas search config.');
}
loggerImpl.error(error.message, false);
return {
status: 'failed'
};
}
};
export default async options => runSearchHttpSourceAdd(options);