@puls-atlas/cli/dist/cmd/search/sources/addFirestore.js

The Puls Atlas CLI tool for managing Atlas projects

import fs from 'fs';
import path from 'path';
import inquirer from 'inquirer';
import { isEqual } from 'es-toolkit';
import * as features from '../../../utils/feature.js';
import { spreadIf } from '../../../utils/value.js';
import { readJsonFile, writeJsonFile, writeTextFile } from '../../../utils/file.js';
import { inferSchemaFromDocuments, logger, mapInferredSchemaToSearchFields, normalizeFirestoreCollectionPath, normalizeOptionalString, sampleFirestoreDocuments } from '../../../utils/index.js';
import { validateSearchRootSection, validateSearchWorkloadConfig } from '../config/searchValidation.js';
const DEFAULT_SAMPLE_LIMIT = 50;
const DEFAULT_SEARCH_SOURCE_TYPE = 'firestore';
const SUPPORTED_SEARCH_SOURCE_TYPES = [DEFAULT_SEARCH_SOURCE_TYPE];
const IDENTIFIER_PATTERN = /^[A-Za-z_$][A-Za-z0-9_$]*$/;
const normalizePathSeparators = targetPath => targetPath.split(path.sep).join('/');
const sanitizeNameSegment = value => value.toLowerCase().replace(/[^a-z0-9_]/g, '_').replace(/_{2,}/g, '_').replace(/^_+|_+$/g, '');
const isValidIdentifierName = value => IDENTIFIER_PATTERN.test(value);
const toPascalCase = value => value.split(/[^A-Za-z0-9]+/).map(segment => segment.trim()).filter(Boolean).map(segment => segment[0].toUpperCase() + segment.slice(1)).join('');
const toCollectionSegments = collectionPath => normalizeFirestoreCollectionPath(collectionPath).split('/').filter((segment, index) => segment.length > 0 && index % 2 === 0);
export const createSearchFirestoreSourceName = collectionPath => {
  const sanitizedName = sanitizeNameSegment(toCollectionSegments(collectionPath).join('_'));
  if (!sanitizedName) {
    throw new Error(`Could not derive a stable Atlas search source name from ${collectionPath}.`);
  }
  return sanitizedName;
};
const createMapperName = sourceName => sourceName.endsWith('s') && sourceName.length > 1 ? sourceName.slice(0, -1) : sourceName;
const ensureIdField = fields => {
  if (fields.some(field => field.name === 'id')) {
    return fields;
  }
  return [{
    name: 'id',
    optional: false,
    type: 'string'
  }, ...fields];
};
const createQueryBy = fields => {
  const preferredFieldNames = ['displayName', 'title', 'name', 'email', 'fname', 'lname'];
  const stringFieldNames = fields.filter(field => field.type === 'string' || field.type === 'string[]').map(field => field.name).filter(fieldName => fieldName !== 'id' && fieldName !== 'sourcePath');
  const preferred = preferredFieldNames.filter(fieldName => stringFieldNames.includes(fieldName));
  const remaining = stringFieldNames.filter(fieldName => !preferred.includes(fieldName));
  const selected = [...preferred, ...remaining].slice(0, 4);
  return selected.length > 0 ? selected : ['id'];
};
const createDefaultSort = fields => {
  for (const candidate of ['updatedAt', 'createdAt']) {
    if (fields.some(field => field.name === candidate && (field.type === 'int64' || field.type === 'int32' || field.type === 'float'))) {
      return [{
        direction: 'desc',
        field: candidate
      }];
    }
  }
  return [];
};
const createFilterBy = fields => fields.filter(field => /status|role|state|type/i.test(field.name)).map(field => field.name).filter(fieldName => fieldName !== 'id');
const createGroupBy = fields => fields.filter(field => /organization|tenant|team|group/i.test(field.name)).map(field => field.name).filter(fieldName => fieldName !== 'id');
const createMapperPropertyAssignment = fieldName => {
  const targetProperty = isValidIdentifierName(fieldName) ? fieldName : `[${JSON.stringify(fieldName)}]`;
  const sourceAccessor = isValidIdentifierName(fieldName) ? `source.${fieldName}` : `source[${JSON.stringify(fieldName)}]`;
  if (fieldName === 'id') {
    return `${targetProperty}: typeof source.id === 'string' ? source.id : String(source.id ?? source.uid ?? source.documentId ?? '')`;
  }
  return `${targetProperty}: ${sourceAccessor} ?? null`;
};
const createMapperStub = mapperExportName => {
  const sourceTypeName = 'SourceDocument';
  return `${[`type ${sourceTypeName} = Record<string, unknown>;`, '', '// TODO: Ensure id maps to a stable source identifier for this collection.', `export const ${mapperExportName} = (source: ${sourceTypeName}) => ({`, "    id: typeof source.id === 'string' ? source.id : String(source.id ?? source.uid ?? source.documentId ?? '')", '});', '', `export default ${mapperExportName};`].join('\n')}\n`;
};
const createMapperStubWithFields = (mapperExportName, fieldNames) => {
  const assignmentLines = fieldNames.map(fieldName => `    ${createMapperPropertyAssignment(fieldName)}`);
  return `${['type SourceDocument = Record<string, unknown>;', '', '// TODO: Ensure id maps to a stable source identifier for this collection.', `export const ${mapperExportName} = (source: SourceDocument) => ({`, assignmentLines.join(',\n'), '});', '', `export default ${mapperExportName};`].join('\n')}\n`;
};
export const createSearchFirestoreWorkloadArtifacts = (collectionPath, sampledDocuments, dependencies = {}) => {
  const inferSchemaFromDocumentsImpl = dependencies.inferSchemaFromDocuments ?? inferSchemaFromDocuments;
  const mapInferredSchemaToSearchFieldsImpl = dependencies.mapInferredSchemaToSearchFields ?? mapInferredSchemaToSearchFields;
  const normalizedCollectionPath = normalizeFirestoreCollectionPath(collectionPath);
  const collectionSegments = toCollectionSegments(normalizedCollectionPath);
  const sourceName = createSearchFirestoreSourceName(normalizedCollectionPath);
  const mapperName = createMapperName(sourceName);
  const mapperExportName = `map${toPascalCase(mapperName)}ToSearchDocument`;
  const indexName = `${sourceName}_search`;
  const inferredSchema = inferSchemaFromDocumentsImpl(sampledDocuments ?? []);
  const fields = ensureIdField(mapInferredSchemaToSearchFieldsImpl(inferredSchema));
  const queryBy = createQueryBy(fields);
  const defaultSort = createDefaultSort(fields);
  const filterBy = createFilterBy(fields);
  const groupBy = createGroupBy(fields);
  const facets = [...new Set([...filterBy, ...groupBy])];
  const routeCollectionSegment = collectionSegments.at(-1) ?? sourceName;
  const workloadRelativePath = normalizePathSeparators(path.join('workloads', `${sourceName}.json`));
  const mapperRelativePath = normalizePathSeparators(path.join('mappers', `${mapperName}.ts`));
  const mapperStub = fields.length <= 1 ? createMapperStub(mapperExportName) : createMapperStubWithFields(mapperExportName, [...new Set(fields.map(field => field.name))]);
  const workload = validateSearchWorkloadConfig({
    index: {
      ...spreadIf(defaultSort.length > 0, {
        defaultSort
      }),
      ...spreadIf(facets.length > 0, {
        facets
      }),
      fields,
      ...spreadIf(filterBy.length > 0, {
        filterBy
      }),
      ...spreadIf(groupBy.length > 0, {
        groupBy
      }),
      name: indexName,
      queryBy
    },
    mapper: {
      export: mapperExportName,
      name: mapperName,
      source: mapperRelativePath
    },
    source: {
      enabled: true,
      firestore: {
        documentPathPattern: `${normalizedCollectionPath}/*`
      },
      name: sourceName,
      routeTemplate: `/${routeCollectionSegment}/:id`,
      syncClass: 'delta-merge',
      type: 'firestore'
    }
  }, workloadRelativePath);
  const warnings = [];
  if (!Array.isArray(sampledDocuments) || sampledDocuments.length === 0) {
    warnings.push('No source documents were sampled. Atlas generated a conservative mapper and index schema; review and adjust before running apply.');
  }
  return {
    indexName,
    mapperName,
    mapperRelativePath,
    mapperStub,
    sourceName,
    warnings,
    workload,
    workloadRelativePath
  };
};
export const patchSearchRootConfigWithWorkload = (rootConfig, workloadRelativePath) => {
  const validatedRootConfig = validateSearchRootSection(rootConfig);
  const normalizedWorkloadPath = normalizePathSeparators(workloadRelativePath);
  const workloadEntries = Array.isArray(validatedRootConfig.workloads) ? validatedRootConfig.workloads : [];
  if (workloadEntries.includes(normalizedWorkloadPath)) {
    return {
      config: rootConfig,
      updated: false
    };
  }
  return {
    updated: true,
    config: {
      ...rootConfig,
      workloads: [...workloadEntries, normalizedWorkloadPath]
    }
  };
};
const resolveSampleLimit = options => {
  const parsedValue = Number.parseInt(options.sampleLimit ?? DEFAULT_SAMPLE_LIMIT, 10);
  if (!Number.isInteger(parsedValue) || parsedValue <= 0) {
    throw new Error('Atlas search source add --type firestore requires --sample-limit to be a positive integer.');
  }
  return parsedValue;
};
export const resolveSearchFirestoreSourceType = (options = {}) => {
  const rawSourceType = normalizeOptionalString(options.type)?.toLowerCase() ?? DEFAULT_SEARCH_SOURCE_TYPE;
  if (!SUPPORTED_SEARCH_SOURCE_TYPES.includes(rawSourceType)) {
    throw new Error(`Atlas search source add supports only --type ${DEFAULT_SEARCH_SOURCE_TYPE} right now.`);
  }
  return rawSourceType;
};
export const resolveSearchFirestoreSourcePath = async (options = {}, dependencies = {}) => {
  const configuredCollectionPath = normalizeOptionalString(options.path);
  if (configuredCollectionPath) {
    return normalizeFirestoreCollectionPath(configuredCollectionPath);
  }
  if (options.interactive !== true) {
    throw new Error('Atlas search source add --type firestore requires --path <sourcePath> or --interactive.');
  }
  const prompt = dependencies.prompt ?? inquirer.prompt;
  const {
    collectionPath
  } = await prompt([{
    message: 'Which Firestore source path should Atlas onboard?',
    name: 'collectionPath',
    type: 'input'
  }]);
  return normalizeFirestoreCollectionPath(collectionPath);
};
const resolveMapperWritePlan = (mapperFilePath, mapperSource) => {
  if (!fs.existsSync(mapperFilePath)) {
    return {
      action: 'create',
      reason: null,
      write: true
    };
  }
  const existingSource = fs.readFileSync(mapperFilePath, 'utf-8');
  if (existingSource === mapperSource) {
    return {
      action: 'unchanged',
      reason: null,
      write: false
    };
  }
  return {
    action: 'keep-existing',
    reason: 'Mapper file already exists with custom content. Atlas kept the existing mapper unchanged.',
    write: false
  };
};
const logFirestoreSourceAddSummary = (loggerImpl, payload, options = {}) => {
  const detailOnly = options.detailOnly ?? false;
  loggerImpl.summary(options.dryRun === true ? 'Firestore source add dry-run summary' : 'Firestore source add summary', [{
    label: 'Project',
    value: payload.projectId
  }, payload.environment ? {
    label: 'Environment',
    value: payload.environment
  } : null, {
    label: 'Config',
    value: payload.configPath
  }, {
    label: 'Source path',
    value: payload.collectionPath
  }, {
    label: 'Source name',
    value: payload.sourceName
  }, {
    label: 'Index name',
    value: payload.indexName
  }, {
    label: 'Mapper name',
    value: payload.mapperName
  }, {
    label: 'Sampled documents',
    value: payload.sampledDocumentCount
  }], {
    detailOnly
  });
  loggerImpl.summary('Planned file actions', [{
    label: payload.workloadPath,
    value: payload.workloadAction
  }, {
    label: payload.mapperPath,
    value: payload.mapperAction
  }, {
    label: payload.configPath,
    value: payload.rootConfigAction
  }], {
    detailOnly
  });
};
const assertFirestoreWorkloadDoesNotDrift = ({
  artifacts,
  context,
  currentWorkload,
  workloadPath
}) => {
  const existingCollection = context.config.collections?.[artifacts.sourceName] ?? null;
  const existingSource = context.config.sources?.[artifacts.sourceName] ?? null;
  const existingMapper = context.config.mappers?.[artifacts.mapperName] ?? null;
  const existingIndex = context.config.indexes?.[artifacts.indexName] ?? null;
  if ((existingCollection || existingSource) && currentWorkload === null) {
    throw new Error(`Atlas search source "${artifacts.sourceName}" already exists in another loaded workload. Update it manually instead of re-running onboarding.`);
  }
  if (existingMapper && currentWorkload === null) {
    throw new Error(`Atlas search mapper "${artifacts.mapperName}" already exists in another loaded workload. Update it manually instead of re-running onboarding.`);
  }
  if (existingIndex && currentWorkload === null) {
    throw new Error(`Atlas search index "${artifacts.indexName}" already exists in another loaded workload. Update it manually instead of re-running onboarding.`);
  }
  if (currentWorkload && !isEqual(currentWorkload, artifacts.workload)) {
    throw new Error(`Atlas search workload "${artifacts.sourceName}" already exists in ${workloadPath} with different content. Update it manually instead of re-running onboarding.`);
  }
};
export const runSearchFirestoreSourceAdd = async (options = {}, dependencies = {}, cwd = process.cwd()) => {
  const loadFeatureContextImpl = dependencies.loadFeatureContext ?? features.loadFeatureContext;
  const loggerImpl = dependencies.logger ?? logger;
  const readJsonFileImpl = dependencies.readJsonFile ?? readJsonFile;
  const sampleFirestoreDocumentsImpl = dependencies.sampleFirestoreDocuments ?? sampleFirestoreDocuments;
  const writeJsonFileImpl = dependencies.writeJsonFile ?? writeJsonFile;
  const writeTextFileImpl = dependencies.writeTextFile ?? writeTextFile;
  let spinner;
  try {
    const sourceType = resolveSearchFirestoreSourceType(options);
    const context = await loadFeatureContextImpl('search', options, {
      cwd
    });
    const collectionPath = await resolveSearchFirestoreSourcePath(options, {
      prompt: dependencies.prompt
    });
    const sampleLimit = resolveSampleLimit(options);
    spinner = loggerImpl.spinner(options.dryRun === true ? `Preparing Atlas search ${sourceType} workload onboarding dry run...` : `Adding ${sourceType} workload to Atlas search config...`);
    const sampledDocuments = await sampleFirestoreDocumentsImpl({
      collectionPath,
      limit: sampleLimit,
      projectId: context.projectId
    }, {
      createFirestoreClient: dependencies.createFirestoreClient,
      firestoreClient: dependencies.firestoreClient
    });
    const artifacts = createSearchFirestoreWorkloadArtifacts(collectionPath, sampledDocuments, {
      inferSchemaFromDocuments: dependencies.inferSchemaFromDocuments,
      mapInferredSchemaToSearchFields: dependencies.mapInferredSchemaToSearchFields
    });
    const searchDirectory = path.dirname(context.configPath);
    const workloadPath = path.join(searchDirectory, artifacts.workloadRelativePath);
    const mapperPath = path.join(searchDirectory, artifacts.mapperRelativePath);
    const rootConfig = readJsonFileImpl(context.configPath);
    const currentWorkload = readJsonFileImpl(workloadPath, {
      allowMissing: true
    });
    const rootConfigPatch = patchSearchRootConfigWithWorkload(rootConfig, artifacts.workloadRelativePath);
    const mapperWritePlan = resolveMapperWritePlan(mapperPath, artifacts.mapperStub);
    const warnings = [...artifacts.warnings];
    assertFirestoreWorkloadDoesNotDrift({
      artifacts,
      context,
      currentWorkload,
      workloadPath
    });
    if (sampledDocuments.length === sampleLimit) {
      warnings.push(`Atlas sampled ${sampledDocuments.length} documents (the --sample-limit cap). Consider rerunning with a higher --sample-limit before apply to improve optional-field inference.`);
    }
    if (mapperWritePlan.reason) {
      warnings.push(mapperWritePlan.reason);
    }
    const workloadAction = currentWorkload === null ? options.dryRun ? 'would-create' : 'created' : 'unchanged';
    let mapperAction = 'unchanged';
    if (mapperWritePlan.action === 'create') {
      mapperAction = options.dryRun ? 'would-create' : 'created';
    } else if (mapperWritePlan.action === 'keep-existing') {
      mapperAction = 'kept-existing';
    }
    const rootConfigAction = rootConfigPatch.updated ? options.dryRun ? 'would-update' : 'updated' : 'unchanged';
    if (options.dryRun !== true) {
      if (currentWorkload === null) {
        writeJsonFileImpl(workloadPath, artifacts.workload);
      }
      if (rootConfigPatch.updated) {
        writeJsonFileImpl(context.configPath, rootConfigPatch.config);
      }
      if (mapperWritePlan.write) {
        writeTextFileImpl(mapperPath, artifacts.mapperStub);
      }
    }
    spinner.succeed(options.dryRun === true ? 'Atlas search Firestore workload onboarding dry run is ready.' : 'Atlas search Firestore workload onboarding completed.');
    const result = {
      collectionPath,
      configPath: context.configPath,
      environment: context.environment,
      indexName: artifacts.indexName,
      mapperAction,
      mapperName: artifacts.mapperName,
      mapperPath,
      projectId: context.projectId,
      rootConfigAction,
      sampledDocumentCount: sampledDocuments.length,
      sourceName: artifacts.sourceName,
      sourceType,
      status: options.dryRun === true ? 'dry-run' : 'updated',
      warnings,
      workloadAction,
      workloadPath
    };
    logFirestoreSourceAddSummary(loggerImpl, result, {
      dryRun: options.dryRun === true
    });
    for (const warning of warnings) {
      loggerImpl.warning(warning);
    }
    if (options.dryRun === true) {
      loggerImpl.info('Dry run requested: no files were changed.');
    }
    return result;
  } catch (error) {
    if (spinner) {
      spinner.fail('Failed to onboard Firestore source into Atlas search config.');
    }
    loggerImpl.error(error.message, false);
    return {
      status: 'failed'
    };
  }
};
export default async options => runSearchFirestoreSourceAdd(options);