@puls-atlas/cli
Version:
The Puls Atlas CLI tool for managing Atlas projects
907 lines • 34.4 kB
JavaScript
import fs from 'fs';
import path from 'path';
import { cloneDeep as clone } from 'es-toolkit/object';
import { isEqual } from 'es-toolkit/predicate';
import * as features from '../../../utils/feature.js';
import { resolveSyncInitProjectSelection } from '../init.js';
import { readJsonFile, writeJsonFile, writeTextFile } from '../../../utils/file.js';
import { validateSyncRootSection, validateSyncWorkloadConfig } from '../config/syncValidation.js';
import { logger, inferSchemaFromDocuments, mapInferredSchemaToBigQueryFields, normalizeFirestoreCollectionPath, normalizeOptionalString, sampleFirestoreDocuments } from '../../../utils/index.js';
import { createDefaultSyncProjectBindings, createDefaultSyncProjectConfig, ensureSyncConfigSection } from '../config/syncConfig.js';
const DEFAULT_SYNC_WORKLOAD_ADD_SOURCE_TYPE = 'firestore';
const DEFAULT_SYNC_WORKLOAD_ADD_DESTINATION_TYPE = 'bigquery';
const DEFAULT_SYNC_BIGQUERY_SCHEMA_SAMPLE_LIMIT = 50;
const SUPPORTED_SYNC_WORKLOAD_ADD_SOURCE_TYPES = ['firestore', 'http', 'sql'];
const SUPPORTED_SYNC_WORKLOAD_ADD_DESTINATION_TYPES = ['bigquery', 'postgres'];
const IDENTIFIER_PATTERN = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
const DEFAULT_SYNC_BATCH_SIZES = {
bigquery: 100,
postgres: 50
};
const normalizePathSeparators = targetPath => targetPath.split(path.sep).join('/');
const sanitizeNameSegment = value => value.toLowerCase().replace(/[^a-z0-9_]/g, '_').replace(/_{2,}/g, '_').replace(/^_+|_+$/g, '');
const isValidIdentifierName = value => IDENTIFIER_PATTERN.test(value);
const toPascalCase = value => value.split(/[^A-Za-z0-9]+/).map(segment => segment.trim()).filter(Boolean).map(segment => segment[0].toUpperCase() + segment.slice(1)).join('');
const toCamelCase = value => {
const pascalCaseValue = toPascalCase(value);
return pascalCaseValue ? `${pascalCaseValue[0].toLowerCase()}${pascalCaseValue.slice(1)}` : '';
};
const singularizeName = value => value.endsWith('s') && value.length > 1 ? value.slice(0, -1) : value;
const resolvePositiveIntegerOption = (value, description) => {
if (value === undefined) {
return null;
}
const parsedValue = Number.parseInt(String(value), 10);
if (!Number.isInteger(parsedValue) || parsedValue <= 0) {
throw new Error(`${description} must be a positive integer when provided.`);
}
return parsedValue;
};
const resolveSyncWorkloadAddSourceType = (options = {}) => {
const sourceType = normalizeOptionalString(options.sourceType)?.toLowerCase() ?? DEFAULT_SYNC_WORKLOAD_ADD_SOURCE_TYPE;
if (!SUPPORTED_SYNC_WORKLOAD_ADD_SOURCE_TYPES.includes(sourceType)) {
throw new Error('Atlas sync workload add supports only --source-type firestore, http, or sql.');
}
return sourceType;
};
const resolveSyncWorkloadAddDestinationType = (options = {}) => {
const destinationType = normalizeOptionalString(options.destinationType)?.toLowerCase() ?? DEFAULT_SYNC_WORKLOAD_ADD_DESTINATION_TYPE;
if (!SUPPORTED_SYNC_WORKLOAD_ADD_DESTINATION_TYPES.includes(destinationType)) {
throw new Error('Atlas sync workload add supports only --destination-type bigquery or postgres.');
}
return destinationType;
};
const toCollectionSegments = collectionPath => normalizeFirestoreCollectionPath(collectionPath).split('/').filter((segment, index) => segment.length > 0 && index % 2 === 0);
const resolveSyncWorkloadBaseName = (options, sourceType) => {
if (sourceType === 'firestore') {
const explicitCollectionPath = normalizeOptionalString(options.path);
if (explicitCollectionPath) {
const collectionSegments = toCollectionSegments(explicitCollectionPath);
const derivedName = sanitizeNameSegment(collectionSegments.join('_'));
if (derivedName) {
return derivedName;
}
}
}
const configuredName = sanitizeNameSegment(normalizeOptionalString(options.name) ?? '');
if (configuredName) {
return configuredName;
}
if (sourceType === 'firestore') {
throw new Error('Atlas sync workload add requires --path <collectionPath> or --name <name> when --source-type firestore is used.');
}
throw new Error('Atlas sync workload add requires --name <name>.');
};
const resolveWorkloadNames = ({
baseName,
destinationType,
mapperName,
sourceType
}) => {
const singularBaseName = singularizeName(baseName);
const mapperStem = destinationType === 'postgres' ? `${singularBaseName} mirror row` : `${singularBaseName} warehouse row`;
const normalizedMapperName = normalizeOptionalString(mapperName) ?? toCamelCase(mapperStem);
const mapperPascalName = toPascalCase(normalizedMapperName);
return {
destinationName: destinationType === 'postgres' ? `${baseName}_postgres_mirror` : `${baseName}_bigquery_events`,
mapperExport: `map${mapperPascalName}`,
mapperFileName: `${baseName}.ts`,
mapperName: normalizedMapperName,
sourceName: sourceType === 'firestore' ? `${baseName}_firestore` : `${baseName}_${sourceType}`,
workloadFileName: `${baseName}.json`,
workloadName: destinationType === 'postgres' ? `${baseName}_to_postgres_mirror` : `${baseName}_to_bigquery_events`
};
};
const createDestinationDescription = (baseName, destinationType) => destinationType === 'postgres' ? `Current-state PostgreSQL mirror for ${baseName.replaceAll('_', ' ')}.` : `Append-only BigQuery changelog for ${baseName.replaceAll('_', ' ')} events.`;
const createSourceDescription = (baseName, sourceType) => {
if (sourceType === 'firestore') {
return `${toPascalCase(baseName)} documents from Firestore.`;
}
if (sourceType === 'http') {
return `Scheduled HTTP sync for ${baseName.replaceAll('_', ' ')}.`;
}
return `Incremental SQL polling for ${baseName.replaceAll('_', ' ')}.`;
};
const createExplicitBigQuerySchemaFieldsFromFieldNames = fieldNames => {
const schemaFields = [{
mode: 'REQUIRED',
name: '_sync_key',
type: 'STRING'
}];
const seenFieldNames = new Set(schemaFields.map(field => field.name));
for (const fieldName of fieldNames) {
const normalizedFieldName = normalizeOptionalString(fieldName);
if (!normalizedFieldName || seenFieldNames.has(normalizedFieldName)) {
continue;
}
schemaFields.push({
mode: 'NULLABLE',
name: normalizedFieldName,
type: 'STRING'
});
seenFieldNames.add(normalizedFieldName);
}
return schemaFields;
};
const createExplicitBigQuerySchemaFromFieldNames = fieldNames => ({
fields: createExplicitBigQuerySchemaFieldsFromFieldNames(fieldNames),
mode: 'explicit'
});
const createMapperFieldNamesFromBigQuerySchemaFields = schemaFields => schemaFields.map(field => normalizeOptionalString(field?.name)).filter(fieldName => fieldName && fieldName !== '_sync_key');
const createFallbackBigQueryScaffoldDetails = () => {
const mapperFieldNames = ['id'];
return {
bigquerySchema: createExplicitBigQuerySchemaFromFieldNames(mapperFieldNames),
mapperFieldNames,
scaffoldWarnings: []
};
};
const resolveSyncBigQueryScaffoldDetails = async (options, projectSelection, dependencies = {}) => {
const sampleFirestoreDocumentsImpl = dependencies.sampleFirestoreDocuments ?? sampleFirestoreDocuments;
const inferSchemaFromDocumentsImpl = dependencies.inferSchemaFromDocuments ?? inferSchemaFromDocuments;
const mapInferredSchemaToBigQueryFieldsImpl = dependencies.mapInferredSchemaToBigQueryFields ?? mapInferredSchemaToBigQueryFields;
const collectionPath = normalizeOptionalString(options.path);
const sampleLimit = resolvePositiveIntegerOption(options.sampleLimit, 'Atlas sync workload add --sample-limit') ?? DEFAULT_SYNC_BIGQUERY_SCHEMA_SAMPLE_LIMIT;
const fallback = createFallbackBigQueryScaffoldDetails();
if (!collectionPath) {
return fallback;
}
try {
const sampledDocuments = await sampleFirestoreDocumentsImpl({
collectionPath,
limit: sampleLimit,
projectId: projectSelection.projectId
});
if (sampledDocuments.length === 0) {
return {
...fallback,
scaffoldWarnings: ['Atlas could not infer a sync BigQuery schema because the sampled Firestore collection returned no documents. Atlas scaffolded a minimal explicit schema instead.']
};
}
const inferredSchema = inferSchemaFromDocumentsImpl(sampledDocuments);
const inferredFields = mapInferredSchemaToBigQueryFieldsImpl(inferredSchema);
if (inferredFields.length === 0) {
return {
...fallback,
scaffoldWarnings: ['Atlas could not derive BigQuery fields from the sampled Firestore documents. Atlas scaffolded a minimal explicit schema instead.']
};
}
const schemaFields = createExplicitBigQuerySchemaFieldsFromFieldNames(createMapperFieldNamesFromBigQuerySchemaFields(inferredFields)).map(field => {
const inferredField = inferredFields.find(candidate => candidate.name === field.name);
return inferredField ? {
...field,
mode: inferredField.mode,
type: inferredField.type
} : field;
});
return {
bigquerySchema: {
fields: schemaFields,
mode: 'explicit'
},
mapperFieldNames: createMapperFieldNamesFromBigQuerySchemaFields(schemaFields),
scaffoldWarnings: []
};
} catch (error) {
return {
...fallback,
scaffoldWarnings: [`Atlas could not infer a sync BigQuery schema from Firestore samples. ${error.message}`]
};
}
};
const resolveSyncWorkloadScaffoldOptions = async (options, projectSelection, dependencies = {}) => {
const sourceType = resolveSyncWorkloadAddSourceType(options);
const destinationType = resolveSyncWorkloadAddDestinationType(options);
if (sourceType !== 'firestore' || destinationType !== 'bigquery') {
return options;
}
return {
...options,
...(await resolveSyncBigQueryScaffoldDetails(options, projectSelection, dependencies))
};
};
const createMapperPropertyAssignment = fieldName => {
const targetProperty = isValidIdentifierName(fieldName) ? fieldName : `[${JSON.stringify(fieldName)}]`;
const sourceAccessor = isValidIdentifierName(fieldName) ? `payload.${fieldName}` : `payload[${JSON.stringify(fieldName)}]`;
return `${targetProperty}: ${sourceAccessor} ?? null`;
};
const createSyncMapperStub = ({
destinationType,
exportName,
fieldNames
}) => {
const normalizedFieldNames = [...new Set(fieldNames.filter(Boolean))];
const assignmentLines = normalizedFieldNames.map(fieldName => ` ${createMapperPropertyAssignment(fieldName)}`);
const operationExpression = destinationType === 'postgres' ? "change.afterData ? 'upsert' : 'delete'" : "change.afterData ? 'append' : 'delete'";
return `${['type SyncChangeEnvelope = {', ' afterData?: Record<string, unknown> | null;', ' beforeData?: Record<string, unknown> | null;', ' sourceDocumentId?: string | null;', '};', '', `export const ${exportName} = (change: SyncChangeEnvelope) => {`, ' const payload = change.afterData ?? change.beforeData ?? {};', ' const recordKey =', " typeof payload.id === 'string' ? payload.id : String(payload.id ?? change.sourceDocumentId ?? '');", '', ' return {', ' records: [', ' {', ' key: recordKey,', ` operation: ${operationExpression},`, ' payload:', ' Object.keys(payload).length === 0', ' ? null', ' : {', assignmentLines.length > 0 ? assignmentLines.join(',\n') : ' id: recordKey', ' },', ' metadata: {}', ' }', ' ]', ' };', '};', '', `export default ${exportName};`].join('\n')}\n`;
};
const createFirestoreSourceConfig = ({
baseName,
collectionPath,
sourceName
}) => ({
description: createSourceDescription(baseName, 'firestore'),
firestore: {
documentPathPattern: `${collectionPath}/{documentId}`
},
name: sourceName,
syncClass: 'delta-merge',
type: 'firestore'
});
const createHttpSourceConfig = ({
baseName,
options,
sourceName
}) => {
const syncClass = normalizeOptionalString(options.syncClass)?.toLowerCase() ?? 'delta-merge';
const mapping = {
afterPath: '$',
idPath: 'id',
versionPath: 'updatedAt'
};
if (syncClass !== 'append-only') {
mapping.beforePath = 'before';
mapping.deletedPath = 'deleted';
}
return {
description: createSourceDescription(baseName, 'http'),
http: {
incremental: {
cursor: {
request: {
queryParam: 'cursor'
},
response: {
hasMorePath: 'pageInfo.hasMore',
nextCursorPath: 'pageInfo.nextCursor'
}
},
mapping,
response: {
itemsPath: 'items'
},
strategy: 'cursor'
},
method: normalizeOptionalString(options.method)?.toUpperCase() ?? 'GET'
},
name: sourceName,
syncClass,
type: 'http'
};
};
const createSqlSourceConfig = ({
baseName,
options,
sourceName
}) => {
const syncClass = normalizeOptionalString(options.syncClass)?.toLowerCase() ?? 'delta-merge';
const query = normalizeOptionalString(options.query) ?? `SELECT id, updated_at${syncClass === 'append-only' ? '' : ', deleted_at'} FROM ${baseName} WHERE (updated_at, id) > (@cursor_updated_at, @cursor_id) ORDER BY updated_at ASC, id ASC LIMIT @batch_size`;
const incremental = {
batchSize: 500,
cursor: {
idColumn: 'id',
versionColumn: 'updated_at'
},
query,
safetyLag: '30s',
strategy: 'watermark'
};
if (syncClass !== 'append-only') {
incremental.deleteStrategy = 'soft-delete';
incremental.deletedColumn = 'deleted_at';
}
return {
description: createSourceDescription(baseName, 'sql'),
name: sourceName,
sql: {
driver: normalizeOptionalString(options.driver) ?? 'cloudsql-postgres',
incremental
},
syncClass,
type: 'sql'
};
};
const createSourceConfig = ({
baseName,
collectionPath,
options,
sourceName,
sourceType
}) => {
if (sourceType === 'firestore') {
return createFirestoreSourceConfig({
baseName,
collectionPath,
sourceName
});
}
if (sourceType === 'http') {
return createHttpSourceConfig({
baseName,
options,
sourceName
});
}
return createSqlSourceConfig({
baseName,
options,
sourceName
});
};
const createBigQueryDestinationConfig = ({
baseName,
bigquerySchema,
destinationName,
sourceName
}) => ({
bigquery: {
primaryKey: ['sourceDocumentId', 'syncVersion'],
schema: bigquerySchema ?? {
mode: 'inferred',
source: sourceName
},
writeApi: 'storage-write'
},
deliveryMode: 'append',
description: createDestinationDescription(baseName, 'bigquery'),
name: destinationName,
schemaMode: 'managed',
type: 'bigquery'
});
const createPostgresDestinationConfig = ({
baseName,
destinationName
}) => ({
deliveryMode: 'mirror',
description: createDestinationDescription(baseName, 'postgres'),
name: destinationName,
postgres: {
deleteMode: 'hard-delete',
primaryKey: ['id']
},
schemaMode: 'managed',
type: 'postgres'
});
const createDestinationConfig = ({
baseName,
bigquerySchema,
destinationName,
destinationType,
sourceName
}) => destinationType === 'postgres' ? createPostgresDestinationConfig({
baseName,
destinationName
}) : createBigQueryDestinationConfig({
baseName,
bigquerySchema,
destinationName,
sourceName
});
const createSourceBinding = ({
baseName,
options,
sourceName,
sourceType
}) => {
if (sourceType === 'firestore') {
const database = normalizeOptionalString(options.firestoreDatabase);
const triggerRegion = normalizeOptionalString(options.triggerRegion);
if (!database && !triggerRegion) {
return null;
}
return {
[sourceName]: {
firestore: {
...(database ? {
database
} : {}),
...(triggerRegion ? {
triggerRegion
} : {})
}
}
};
}
if (sourceType === 'http') {
return {
[sourceName]: {
http: {
connectionSecret: normalizeOptionalString(options.sourceConnectionSecret) ?? `atlas-sync-${baseName}-http`,
url: normalizeOptionalString(options.url) ?? `https://api.example.com/${baseName.replaceAll('_', '-')}`
},
schedule: normalizeOptionalString(options.schedule) ?? '*/5 * * * *'
}
};
}
return {
[sourceName]: {
schedule: normalizeOptionalString(options.schedule) ?? '*/5 * * * *',
sql: {
connectionSecret: normalizeOptionalString(options.sourceConnectionSecret) ?? `atlas-sync-${baseName}-sql`
}
}
};
};
const createDestinationBinding = ({
baseName,
destinationName,
destinationType,
options
}) => {
if (destinationType === 'postgres') {
return {
[destinationName]: {
postgres: {
connectionSecret: normalizeOptionalString(options.destinationConnectionSecret) ?? 'atlas-sync-postgres',
schema: normalizeOptionalString(options.schema) ?? 'public',
table: normalizeOptionalString(options.table) ?? `${baseName}_mirror`
}
}
};
}
return {
[destinationName]: {
bigquery: {
dataset: normalizeOptionalString(options.dataset) ?? 'atlas_sync',
table: normalizeOptionalString(options.table) ?? `${baseName}_events`
}
}
};
};
const createWorkloadBinding = ({
destinationType,
options,
workloadName
}) => ({
[workloadName]: {
batchSize: resolvePositiveIntegerOption(options.batchSize, 'Atlas sync workload add --batch-size') ?? DEFAULT_SYNC_BATCH_SIZES[destinationType],
enabled: options.enable === true
}
});
const createScaffoldWarnings = ({
destinationType,
enabled,
sourceType
}) => {
const warnings = ['Atlas scaffolded a mapper stub. Review the mapped record payload before enabling the workload.'];
if (sourceType === 'http') {
warnings.push('Atlas scaffolded placeholder HTTP cursor and response mapping paths. Review source.http.incremental before apply.');
}
if (sourceType === 'sql') {
warnings.push('Atlas scaffolded a placeholder SQL watermark query and connection secret. Review source.sql.incremental and the project binding before apply.');
}
if (destinationType === 'postgres') {
warnings.push('Atlas scaffolded PostgreSQL primary-key and delete-mode defaults. Review destination.postgres before apply.');
}
if (!enabled) {
warnings.push('New workload bindings default to disabled. Re-run with --enable or edit workloadBindings when the mapper and bindings are ready.');
}
return warnings;
};
export const createSyncWorkloadScaffoldArtifacts = (options = {}) => {
const sourceType = resolveSyncWorkloadAddSourceType(options);
const destinationType = resolveSyncWorkloadAddDestinationType(options);
const collectionPath = sourceType === 'firestore' && normalizeOptionalString(options.path) ? normalizeFirestoreCollectionPath(options.path) : null;
const baseName = resolveSyncWorkloadBaseName(options, sourceType);
const names = resolveWorkloadNames({
baseName,
destinationType,
mapperName: options.mapper,
sourceType
});
const mapperRelativePath = normalizePathSeparators(path.join('mappers', names.mapperFileName));
const workloadRelativePath = normalizePathSeparators(path.join('workloads', names.workloadFileName));
const fieldNames = destinationType === 'postgres' ? ['id', 'status', 'updatedAt'] : options.mapperFieldNames ?? ['id'];
const bigquerySchema = destinationType === 'bigquery' ? options.bigquerySchema ?? createExplicitBigQuerySchemaFromFieldNames(fieldNames) : null;
const workload = validateSyncWorkloadConfig({
destination: createDestinationConfig({
baseName,
bigquerySchema,
destinationName: names.destinationName,
destinationType,
sourceName: names.sourceName
}),
mapper: {
export: names.mapperExport,
name: names.mapperName,
source: mapperRelativePath
},
source: createSourceConfig({
baseName,
collectionPath,
options,
sourceName: names.sourceName,
sourceType
}),
workload: {
deletePolicy: destinationType === 'postgres' ? 'hard-delete' : 'emit-delete-event',
failureMode: 'independent',
name: names.workloadName
}
}, workloadRelativePath);
return {
baseName,
collectionPath,
destinationBinding: createDestinationBinding({
baseName,
destinationName: names.destinationName,
destinationType,
options
}),
destinationName: names.destinationName,
destinationType,
mapperPath: mapperRelativePath,
mapperSource: createSyncMapperStub({
destinationType,
exportName: names.mapperExport,
fieldNames
}),
mapperName: names.mapperName,
sourceBinding: createSourceBinding({
baseName,
options,
sourceName: names.sourceName,
sourceType
}),
sourceName: names.sourceName,
sourceType,
warnings: createScaffoldWarnings({
destinationType,
enabled: options.enable === true,
sourceType
}),
workload,
workloadBinding: createWorkloadBinding({
destinationType,
options,
workloadName: names.workloadName
}),
workloadName: names.workloadName,
workloadPath: workloadRelativePath
};
};
const resolveMapperWritePlan = (mapperFilePath, mapperSource) => {
if (!fs.existsSync(mapperFilePath)) {
return {
action: 'create',
reason: null,
write: true
};
}
const existingSource = fs.readFileSync(mapperFilePath, 'utf-8');
if (existingSource === mapperSource) {
return {
action: 'unchanged',
reason: null,
write: false
};
}
return {
action: 'keep-existing',
reason: 'Mapper file already exists with custom content. Atlas kept the existing mapper unchanged.',
write: false
};
};
const mergeExistingBinding = ({
bindingLabel,
existingValue,
nextValue,
warnings
}) => {
if (!existingValue) {
return {
action: 'updated',
value: nextValue
};
}
if (isEqual(existingValue, nextValue)) {
return {
action: 'unchanged',
value: existingValue
};
}
warnings.push(`Atlas kept the existing ${bindingLabel} unchanged because it already contains custom values.`);
return {
action: 'kept-existing',
value: existingValue
};
};
const assertSyncWorkloadDoesNotDrift = ({
artifacts,
context,
currentWorkload,
workloadPath
}) => {
const existingSource = context.config.sources?.[artifacts.sourceName] ?? null;
const existingDestination = context.config.destinations?.[artifacts.destinationName] ?? null;
const existingMapper = context.config.mappers?.[artifacts.mapperName] ?? null;
const existingWorkload = context.config.workloads?.[artifacts.workloadName] ?? null;
if ((existingSource || existingDestination || existingMapper || existingWorkload) && currentWorkload === null) {
throw new Error(`Atlas sync workload references one or more existing names (${artifacts.workloadName}, ${artifacts.sourceName}, ${artifacts.destinationName}, ${artifacts.mapperName}) from another workload. Update the existing workload manually instead of re-running scaffolding.`);
}
if (currentWorkload && !isEqual(currentWorkload, artifacts.workload)) {
throw new Error(`Atlas sync workload "${artifacts.workloadName}" already exists in ${workloadPath} with different content. Update it manually instead of re-running scaffolding.`);
}
};
const ensureProjectEntry = (rootConfig, {
environment,
projectId
}) => {
const defaultProjectConfig = {
...createDefaultSyncProjectConfig({
environment,
projectId
}),
...createDefaultSyncProjectBindings()
};
const existingProjectConfig = rootConfig.projects?.[projectId] ?? null;
if (!existingProjectConfig) {
return {
created: true,
projectConfig: clone(defaultProjectConfig),
rootConfig: {
...clone(rootConfig),
projects: {
...clone(rootConfig.projects ?? {}),
[projectId]: clone(defaultProjectConfig)
}
}
};
}
return {
created: false,
projectConfig: {
...clone(defaultProjectConfig),
...clone(existingProjectConfig),
deploy: {
...clone(defaultProjectConfig.deploy ?? {}),
...clone(existingProjectConfig.deploy ?? {})
},
destinationBindings: {
...clone(defaultProjectConfig.destinationBindings ?? {}),
...clone(existingProjectConfig.destinationBindings ?? {})
},
sourceBindings: {
...clone(defaultProjectConfig.sourceBindings ?? {}),
...clone(existingProjectConfig.sourceBindings ?? {})
},
workloadBindings: {
...clone(defaultProjectConfig.workloadBindings ?? {}),
...clone(existingProjectConfig.workloadBindings ?? {})
}
},
rootConfig: clone(rootConfig)
};
};
const patchSyncRootConfigWithWorkload = (rootConfig, workloadRelativePath) => {
const normalizedWorkloadPath = normalizePathSeparators(workloadRelativePath);
const workloadEntries = Array.isArray(rootConfig.workloads) ? [...rootConfig.workloads] : [];
if (workloadEntries.includes(normalizedWorkloadPath)) {
return {
config: clone(rootConfig),
updated: false
};
}
return {
config: validateSyncRootSection({
...clone(rootConfig),
workloads: [...workloadEntries, normalizedWorkloadPath]
}),
updated: true
};
};
const logSyncWorkloadAddSummary = (loggerImpl, payload, options = {}) => {
loggerImpl.summary(options.dryRun === true ? 'Sync workload add dry-run summary' : 'Sync workload add summary', [{
label: 'Project',
value: payload.projectId
}, payload.environment ? {
label: 'Environment',
value: payload.environment
} : null, {
label: 'Config',
value: payload.configPath
}, {
label: 'Workload',
value: payload.workloadName
}, {
label: 'Source',
value: `${payload.sourceName} [${payload.sourceType}]`
}, {
label: 'Destination',
value: `${payload.destinationName} [${payload.destinationType}]`
}, {
label: 'Mapper',
value: payload.mapperName
}]);
loggerImpl.summary('Planned file actions', [{
label: payload.workloadFilePath,
value: payload.workloadAction
}, {
label: payload.mapperFilePath,
value: payload.mapperAction
}, {
label: payload.configPath,
value: payload.rootConfigAction
}]);
};
const createSyncWorkloadAddResult = ({
artifacts,
configPath,
destinationBindingAction,
environment,
mapperAction,
mapperFilePath,
projectId,
rootConfigAction,
sourceBindingAction,
status,
warnings,
workloadAction,
workloadBindingAction,
workloadFilePath
}) => ({
configPath,
destinationBindingAction,
destinationName: artifacts.destinationName,
destinationType: artifacts.destinationType,
environment,
mapperAction,
mapperFilePath,
mapperName: artifacts.mapperName,
projectId,
rootConfigAction,
sourceBindingAction,
sourceName: artifacts.sourceName,
sourceType: artifacts.sourceType,
status,
warnings,
workloadAction,
workloadBindingAction,
workloadFilePath,
workloadName: artifacts.workloadName
});
const resolveSyncWorkloadAddProjectSelection = async (options, dependencies = {}, cwd = process.cwd()) => {
const projectSelection = await resolveSyncInitProjectSelection(options, dependencies, cwd);
if (!projectSelection) {
throw new Error('Atlas sync workload add requires a resolved project. Re-run with --project or add .firebaserc project aliases first.');
}
return projectSelection;
};
export const runSyncWorkloadAdd = async (options = {}, dependencies = {}, cwd = process.cwd()) => {
const ensureSyncConfigSectionImpl = dependencies.ensureSyncConfigSection ?? ensureSyncConfigSection;
const loadFeatureContextImpl = dependencies.loadFeatureContext ?? features.loadFeatureContext;
const loggerImpl = dependencies.logger ?? logger;
const readJsonFileImpl = dependencies.readJsonFile ?? readJsonFile;
const resolveProjectSelectionImpl = dependencies.resolveSyncWorkloadAddProjectSelection ?? resolveSyncWorkloadAddProjectSelection;
const writeJsonFileImpl = dependencies.writeJsonFile ?? writeJsonFile;
const writeTextFileImpl = dependencies.writeTextFile ?? writeTextFile;
let spinner;
try {
const projectSelection = await resolveProjectSelectionImpl(options, dependencies, cwd);
const scaffoldOptions = await resolveSyncWorkloadScaffoldOptions(options, projectSelection, dependencies);
ensureSyncConfigSectionImpl(cwd, {
environment: projectSelection.environment ?? undefined,
projectId: projectSelection.projectId
});
const context = await loadFeatureContextImpl('sync', options, {
cwd,
resolveProjectSelectionImpl: async () => projectSelection
});
const artifacts = createSyncWorkloadScaffoldArtifacts(scaffoldOptions);
const syncDirectory = path.dirname(context.configPath);
const workloadFilePath = path.join(syncDirectory, artifacts.workloadPath);
const mapperFilePath = path.join(syncDirectory, artifacts.mapperPath);
const currentWorkload = readJsonFileImpl(workloadFilePath, {
allowMissing: true
});
const mapperWritePlan = resolveMapperWritePlan(mapperFilePath, artifacts.mapperSource);
const warnings = [...artifacts.warnings, ...(Array.isArray(scaffoldOptions.scaffoldWarnings) ? scaffoldOptions.scaffoldWarnings : [])];
spinner = loggerImpl.spinner(options.dryRun === true ? 'Preparing Atlas sync workload scaffolding dry run...' : 'Adding workload to Atlas sync config...');
assertSyncWorkloadDoesNotDrift({
artifacts,
context,
currentWorkload,
workloadPath: workloadFilePath
});
const projectResult = ensureProjectEntry(context.rootConfig ?? readJsonFileImpl(context.configPath), {
environment: projectSelection.environment,
projectId: projectSelection.projectId
});
const patchedRootResult = patchSyncRootConfigWithWorkload(projectResult.rootConfig, artifacts.workloadPath);
const nextProjectConfig = clone(projectResult.projectConfig);
const sourceBindingResult = artifacts.sourceBinding ? mergeExistingBinding({
bindingLabel: `source binding "${artifacts.sourceName}"`,
existingValue: nextProjectConfig.sourceBindings?.[artifacts.sourceName] ?? null,
nextValue: artifacts.sourceBinding[artifacts.sourceName],
warnings
}) : {
action: 'unchanged',
value: null
};
const destinationBindingResult = mergeExistingBinding({
bindingLabel: `destination binding "${artifacts.destinationName}"`,
existingValue: nextProjectConfig.destinationBindings?.[artifacts.destinationName] ?? null,
nextValue: artifacts.destinationBinding[artifacts.destinationName],
warnings
});
const workloadBindingResult = mergeExistingBinding({
bindingLabel: `workload binding "${artifacts.workloadName}"`,
existingValue: nextProjectConfig.workloadBindings?.[artifacts.workloadName] ?? null,
nextValue: artifacts.workloadBinding[artifacts.workloadName],
warnings
});
nextProjectConfig.sourceBindings = {
...clone(nextProjectConfig.sourceBindings ?? {}),
...(artifacts.sourceBinding && sourceBindingResult.value ? {
[artifacts.sourceName]: sourceBindingResult.value
} : {})
};
nextProjectConfig.destinationBindings = {
...clone(nextProjectConfig.destinationBindings ?? {}),
[artifacts.destinationName]: destinationBindingResult.value
};
nextProjectConfig.workloadBindings = {
...clone(nextProjectConfig.workloadBindings ?? {}),
[artifacts.workloadName]: workloadBindingResult.value
};
const nextRootConfig = validateSyncRootSection({
...clone(patchedRootResult.config),
projects: {
...clone(patchedRootResult.config.projects ?? {}),
[projectSelection.projectId]: nextProjectConfig
}
});
const rootConfigChanged = !isEqual(context.rootConfig ?? readJsonFileImpl(context.configPath), nextRootConfig);
const workloadAction = currentWorkload === null ? options.dryRun === true ? 'would-create' : 'created' : 'unchanged';
const mapperAction = mapperWritePlan.action === 'create' ? options.dryRun === true ? 'would-create' : 'created' : mapperWritePlan.action === 'keep-existing' ? 'kept-existing' : 'unchanged';
const rootConfigAction = rootConfigChanged ? options.dryRun === true ? 'would-update' : 'updated' : 'unchanged';
if (mapperWritePlan.reason) {
warnings.push(mapperWritePlan.reason);
}
if (options.dryRun !== true) {
if (currentWorkload === null) {
writeJsonFileImpl(workloadFilePath, artifacts.workload);
}
if (rootConfigChanged) {
writeJsonFileImpl(context.configPath, nextRootConfig);
}
if (mapperWritePlan.write) {
writeTextFileImpl(mapperFilePath, artifacts.mapperSource);
}
}
spinner.succeed(options.dryRun === true ? 'Atlas sync workload scaffolding dry run is ready.' : 'Atlas sync workload scaffolding completed.');
const result = createSyncWorkloadAddResult({
artifacts,
configPath: context.configPath,
destinationBindingAction: destinationBindingResult.action,
environment: context.environment,
mapperAction,
mapperFilePath,
projectId: context.projectId,
rootConfigAction,
sourceBindingAction: sourceBindingResult.action,
status: options.dryRun === true ? 'dry-run' : 'updated',
warnings,
workloadAction,
workloadBindingAction: workloadBindingResult.action,
workloadFilePath
});
logSyncWorkloadAddSummary(loggerImpl, result, {
dryRun: options.dryRun === true
});
for (const warning of warnings) {
loggerImpl.warning(warning);
}
if (options.dryRun === true) {
loggerImpl.info('Dry run requested: no files were changed.');
}
return result;
} catch (error) {
spinner?.fail('Failed to scaffold Atlas sync workload.');
loggerImpl.error(error.message, false);
return {
status: 'failed'
};
}
};
export default async options => runSyncWorkloadAdd(options);