@stackbit/sdk/src/config/config-loader.ts

Stackbit SDK

import path from 'path';
import fse from 'fs-extra';
import chokidar from 'chokidar';
import semver from 'semver';
import _ from 'lodash';

import * as StackbitTypes from '@stackbit/types';
import {
    ModelsSource,
    Field,
    ModelExtension,
    FieldExtension,
    FieldType,
    DistributePartialListItems,
    FieldListItems,
    CustomActionDocument,
    CustomActionObjectModel,
    Experimental
} from '@stackbit/types';
import { append, fieldPathToString, getFirstExistingFile, omitByNil, prepend, rename } from '@stackbit/utils';

import { ConfigValidationResult, validateBaseConfig, validateConfig, validateContentModels } from './config-validator';
import { ConfigError, ConfigLoadError, ModelLoadError, ConfigValidationError, StackbitConfigNotFoundError } from './config-errors';
import { loadStackbitConfigFromJs } from './config-loader-esbuild';
import {
    assignLabelFieldIfNeeded,
    getListItemsOrSelf,
    getModelFieldForModelKeyPath,
    isCustomModelField,
    isDataModel,
    isListDataModel,
    isModelField,
    isObjectField,
    isObjectListItems,
    isPageModel,
    isReferenceField,
    mapModelFieldsRecursively,
    normalizeListFieldInPlace,
    mapListItemsPropsOrSelfSpecificProps,
    Logger
} from '../utils';
import type { Config, StackbitConfigWithPaths, SSGRunOptions, Model, PageModel, DataModel, ObjectModel, DataModelSingle } from './config-types';
import { extendModelsWithPresetsIds, loadPresets } from './presets-loader';
import {
    LoadStackbitConfigResult,
    WithReloadAndDestroy,
    LATEST_STACKBIT_VERSION,
    STACKBIT_CONFIG_JS_FILES,
    STACKBIT_CONFIG_YAML_FILES,
    loadStackbitYamlFromDir,
    loadYamlModelsFromFiles,
    mergeConfigModelsWithModelsFromFiles
} from './config-loader-utils';

const CONTENT_ENGINE_DEFAULT_PORT = 8000;

export type ConfigWithModelsPresetsResult = {
    valid: boolean;
    config: Config | null;
    errors: ConfigError[];
};

export type ConfigWithModelsPresetsResultWithReloadDestroy = WithReloadAndDestroy<ConfigWithModelsPresetsResult>;

export async function loadConfigWithModelsPresetsAndValidate({
    dirPath,
    modelsSource,
    stackbitConfigESBuildOutDir,
    watchCallback,
    logger,
    isForcedGitCSI
}: {
    dirPath: string;
    modelsSource?: ModelsSource;
    stackbitConfigESBuildOutDir?: string;
    watchCallback?: (result: ConfigWithModelsPresetsResult) => void;
    logger?: Logger;
    isForcedGitCSI?: boolean;
}): Promise<ConfigWithModelsPresetsResultWithReloadDestroy> {
    const configResult = await loadConfigWithModels({
        dirPath,
        stackbitConfigESBuildOutDir,
        watchCallback: watchCallback
            ? async (configResult) => {
                  const configLoaderResult = await processConfigLoaderResult({ configResult, dirPath, modelsSource, logger, isForcedGitCSI });
                  watchCallback(configLoaderResult);
              }
            : undefined,
        logger
    });

    const configLoaderResult = await processConfigLoaderResult({ configResult, dirPath, modelsSource, logger, isForcedGitCSI });
    const reload = configResult.reload;
    return {
        ...configLoaderResult,
        destroy: configResult.destroy,
        reload: reload
            ? async () => {
                  const configResult = await reload();
                  return processConfigLoaderResult({ configResult, dirPath, modelsSource, logger, isForcedGitCSI });
              }
            : undefined
    };
}

export type LoadConfigWithModelsResult = {
    config: Config | null;
    errors: (ConfigLoadError | StackbitConfigNotFoundError | ModelLoadError | ConfigValidationError)[];
};

export type LoadConfigWithModelsResultWithReloadDestroy = WithReloadAndDestroy<LoadConfigWithModelsResult>;

export async function loadConfigWithModels({
    dirPath,
    stackbitConfigESBuildOutDir,
    watchCallback,
    logger
}: {
    dirPath: string;
    stackbitConfigESBuildOutDir?: string;
    watchCallback?: (result: LoadConfigWithModelsResult) => void;
    logger?: Logger;
}): Promise<LoadConfigWithModelsResultWithReloadDestroy> {
    const wrapConfigResult = async (configResult: LoadConfigResult): Promise<LoadConfigWithModelsResult> => {
        if (!configResult.config) {
            return {
                config: null,
                errors: configResult.errors
            };
        }
        return await loadAndMergeModelsFromFiles(configResult.config);
    };

    const rawConfigResult = await loadConfig({
        dirPath,
        stackbitConfigESBuildOutDir,
        logger,
        watchCallback: watchCallback
            ? async (configResult: LoadConfigResult) => {
                  const wrappedResult = await wrapConfigResult(configResult);
                  watchCallback(wrappedResult);
              }
            : undefined
    });

    const wrappedResult = await wrapConfigResult(rawConfigResult);
    const reload = rawConfigResult.reload;

    return {
        ...wrappedResult,
        destroy: rawConfigResult.destroy,
        reload: reload
            ? async () => {
                  const configResult = await reload();
                  return wrapConfigResult(configResult);
              }
            : undefined
    };
}

export type LoadConfigResult =
    | {
          config: Config;
          errors: ConfigValidationError[];
      }
    | {
          config: null;
          errors: (ConfigLoadError | StackbitConfigNotFoundError)[];
      };

export type LoadConfigResultWithReloadDestroy = WithReloadAndDestroy<LoadConfigResult>;

export async function loadConfig({
    dirPath,
    stackbitConfigESBuildOutDir,
    watchCallback,
    logger
}: {
    dirPath: string;
    stackbitConfigESBuildOutDir?: string;
    watchCallback?: (result: LoadConfigResult) => void;
    logger?: Logger;
}): Promise<LoadConfigResultWithReloadDestroy> {
    const normalizeConfigResult = (rawConfigResult: LoadConfigFromDirResult): LoadConfigResult => {
        if (!rawConfigResult.config) {
            return {
                config: null,
                errors: [rawConfigResult.error]
            };
        }

        const validationResult = validateBaseConfig(rawConfigResult.config);
        const config = normalizeConfig(rawConfigResult.config);

        return {
            config: config,
            errors: validationResult.errors
        };
    };

    const rawConfigResult = await loadConfigFromDir({
        dirPath,
        stackbitConfigESBuildOutDir,
        watchCallback: watchCallback
            ? async (rawConfigResult: LoadConfigFromDirResult) => {
                  const normalizedResult = await normalizeConfigResult(rawConfigResult);
                  watchCallback(normalizedResult);
              }
            : undefined,
        logger
    });

    const normalizedResult = await normalizeConfigResult(rawConfigResult);
    const reload = rawConfigResult.reload;

    return {
        ...normalizedResult,
        destroy: rawConfigResult.destroy,
        reload: reload
            ? async () => {
                  const rawConfigResult = await reload();
                  return normalizeConfigResult(rawConfigResult);
              }
            : undefined
    };
}

async function processConfigLoaderResult({
    configResult,
    dirPath,
    modelsSource,
    logger,
    isForcedGitCSI
}: {
    configResult: LoadConfigWithModelsResult;
    dirPath: string;
    modelsSource?: ModelsSource;
    logger?: Logger;
    isForcedGitCSI?: boolean;
}): Promise<ConfigWithModelsPresetsResult> {
    const { config, errors: configLoadErrors } = configResult;

    if (!config) {
        return {
            valid: false,
            config: null,
            errors: configLoadErrors
        };
    }

    if (isForcedGitCSI) {
        const normalizedResult = validateAndNormalizeConfig(config, isForcedGitCSI);
        return {
            valid: normalizedResult.valid,
            config: {
                ...normalizedResult.config,
                models: normalizedResult.config.models,
                presets: {}
            },
            errors: [...configLoadErrors, ...normalizedResult.errors]
        };
    }

    const { models: externalModels, errors: externalModelsLoadErrors } = await loadModelsFromExternalSource(config, dirPath, modelsSource);

    const mergedModels =
        externalModels.length === 0
            ? config.models
            : mergeConfigModelsWithExternalModels({ configModels: config.models as ModelExtension[], externalModels, logger });
    const mergedConfig: Config = {
        ...config,
        models: mergedModels
    };

    const normalizedResult = validateAndNormalizeConfig(mergedConfig);

    const presetsResult = await loadPresets({ config: normalizedResult.config });

    const modelsWithPresetIds = extendModelsWithPresetsIds({
        models: normalizedResult.config.models,
        presets: presetsResult.presets
    });

    const configWithPresets = {
        ...normalizedResult.config,
        models: modelsWithPresetIds,
        presets: presetsResult.presets
    };

    return {
        valid: normalizedResult.valid,
        config: configWithPresets,
        errors: [...configLoadErrors, ...externalModelsLoadErrors, ...normalizedResult.errors, ...presetsResult.errors]
    };
}

export async function loadAndMergeModelsFromFiles(config: Config): Promise<{ config: Config; errors: (ModelLoadError | ConfigValidationError)[] }> {
    const { models: modelsFromFiles, errors: modelLoadErrors } = await loadYamlModelsFromFiles(config);
    const { models: mergedModels, errors: mergeModelErrors } = mergeConfigModelsWithModelsFromFiles(config.models, modelsFromFiles);

    const extendedConfig: Config = {
        ...config,
        models: mergedModels
    };

    return {
        config: extendedConfig,
        errors: [...modelLoadErrors, ...mergeModelErrors]
    };
}

export function validateAndNormalizeConfig(config: Config, isForcedGitCSI?: boolean): ConfigValidationResult {
    // validate the "contentModels" and extend config models with "contentModels"
    // this must be done before main config validation to make it independent of "contentModels".
    const { config: configWithContentModels, errors: contentModelsErrors } = validateAndExtendContentModels(config);

    // normalize config - backward compatibility updates, adding extra fields like "markdown_content", "type" and "layout",
    // and setting other default values.
    const configWithNormalizedModels = normalizeModels(configWithContentModels, isForcedGitCSI);

    // validate config
    const { config: validatedConfig, errors: validationErrors } = validateConfig(configWithNormalizedModels);

    const errors = [...contentModelsErrors, ...validationErrors];

    return normalizeValidationResult({
        valid: _.isEmpty(errors),
        config: validatedConfig,
        errors: errors
    });
}

export type LoadConfigFromDirResult =
    | {
          config: StackbitConfigWithPaths;
          error: null;
      }
    | {
          config: null;
          error: ConfigLoadError | StackbitConfigNotFoundError;
      };

export type LoadConfigFromDirResultWithReloadDestroy = WithReloadAndDestroy<LoadConfigFromDirResult>;

export async function loadConfigFromDir({
    dirPath,
    stackbitConfigESBuildOutDir,
    watchCallback,
    logger
}: {
    dirPath: string;
    stackbitConfigESBuildOutDir?: string;
    watchCallback?: (result: LoadConfigFromDirResult) => void;
    logger?: Logger;
}): Promise<LoadConfigFromDirResultWithReloadDestroy> {
    function wrapResult(result: LoadStackbitConfigResult, configFilePath: string): LoadConfigFromDirResult {
        if (result.error) {
            return {
                config: null,
                error: result.error
            };
        } else {
            return {
                config: {
                    ...result.config,
                    dirPath: dirPath,
                    filePath: configFilePath
                },
                error: null
            };
        }
    }

    // try to load stackbit config from YAML files
    try {
        const stackbitYamlPath = await getFirstExistingFile(STACKBIT_CONFIG_YAML_FILES, dirPath);
        if (stackbitYamlPath) {
            logger?.debug(`loading Stackbit configuration from ${stackbitYamlPath}`);
            const result = await loadStackbitYamlFromDir(dirPath);
            let close: () => Promise<void> = async () => void 0;
            let stopped = false;

            if (watchCallback) {
                const watcher = chokidar.watch([...STACKBIT_CONFIG_YAML_FILES, '.stackbit/models'], {
                    cwd: dirPath,
                    persistent: true,
                    ignoreInitial: true
                });
                const throttledFileChange = _.throttle(async () => {
                    const result = await loadStackbitYamlFromDir(dirPath);
                    watchCallback(wrapResult(result, stackbitYamlPath));
                }, 1000);
                const handleFileChange = (path: string) => {
                    logger?.debug(`identified change in stackbit config file: ${path}, reloading config...`);
                    throttledFileChange();
                };
                watcher.on('add', handleFileChange);
                watcher.on('change', handleFileChange);
                watcher.on('unlink', handleFileChange);
                watcher.on('addDir', handleFileChange);
                watcher.on('unlinkDir', handleFileChange);
                watcher.on('error', (error) => {
                    watchCallback({
                        config: null,
                        error: new ConfigLoadError(`Error loading Stackbit configuration: ${error.message}`, { originalError: error })
                    });
                });
                close = async () => {
                    if (stopped) {
                        return;
                    }
                    stopped = true;
                    throttledFileChange.cancel();
                    watcher.close();
                };
            }

            return {
                ...wrapResult(result, stackbitYamlPath),
                destroy: close,
                reload: async () => {
                    const result = await loadStackbitYamlFromDir(dirPath);
                    return wrapResult(result, stackbitYamlPath);
                }
            };
        }
    } catch (error: any) {
        return {
            config: null,
            error: new ConfigLoadError(`Error loading Stackbit configuration: ${error.message}`, { originalError: error })
        };
    }

    // try to load stackbit config from JavaScript files
    try {
        const configFilePath = await getFirstExistingFile(STACKBIT_CONFIG_JS_FILES, dirPath);
        if (configFilePath) {
            logger?.debug(`loading Stackbit configuration from: ${configFilePath}`);
            const configResult = await loadStackbitConfigFromJs({
                configPath: configFilePath,
                outDir: stackbitConfigESBuildOutDir ?? '.stackbit/cache',
                watch: !!watchCallback,
                logger: logger,
                callback: watchCallback
                    ? (result: LoadStackbitConfigResult) => {
                          watchCallback(wrapResult(result, configFilePath));
                      }
                    : undefined
            });
            const reload = configResult.reload;
            return {
                ...wrapResult(configResult, configFilePath),
                destroy: configResult.destroy,
                reload: reload
                    ? async () => {
                          const result = await reload();
                          return wrapResult(result, configFilePath);
                      }
                    : undefined
            };
        }
    } catch (error: any) {
        return {
            config: null,
            error: new ConfigLoadError(`Error loading Stackbit configuration: ${error.message}`, { originalError: error })
        };
    }

    return {
        config: null,
        error: new StackbitConfigNotFoundError()
    };
}

async function loadModelsFromExternalSource(
    config: Config,
    dirPath: string,
    modelsSource?: ModelsSource
): Promise<{ models: Model[]; errors: ModelLoadError[] }> {
    modelsSource = _.assign({}, modelsSource, config.modelsSource);
    const sourceType = _.get(modelsSource, 'type', 'files');
    if (sourceType === 'files') {
        // we already loaded models from files inside loadConfigFromDir function
        return { models: [], errors: [] };
    } else if (sourceType === 'contentful') {
        const contentfulModule = _.get(modelsSource, 'module', '@stackbit/cms-contentful');
        const modulePath = path.resolve(dirPath, 'node_modules', contentfulModule);
        const module = await import(modulePath);
        try {
            const { models } = await module.fetchAndConvertSchema(_.omit(modelsSource, ['type', 'module']));
            return {
                models: models,
                errors: []
            };
        } catch (error: any) {
            return {
                models: [],
                errors: [new ModelLoadError(`Error fetching and converting Contentful schema, error: ${error.message}`, { originalError: error })]
            };
        }
    }
    return {
        models: [],
        errors: [new ModelLoadError(`modelsSource ${modelsSource} is unsupported`)]
    };
}

// eslint-disable-next-line @typescript-eslint/no-unused-vars
async function loadConfigFromDotStackbit(dirPath: string) {
    const stackbitDotPath = path.join(dirPath, '.stackbit');
    const stackbitDotExists = await fse.pathExists(stackbitDotPath);
    if (!stackbitDotExists) {
        return null;
    }

    const config = {};

    const themeYaml = path.join(stackbitDotPath, 'theme.yaml');
    const themeYamlExists = await fse.readFile(themeYaml);
    if (themeYamlExists) {
        const themeConfig = await fse.readFile(themeYaml);
        _.assign(config, themeConfig);
    }

    const studioYaml = path.join(stackbitDotPath, 'studio.yaml');
    const studioYamlExists = await fse.readFile(themeYaml);
    if (studioYamlExists) {
        const studioConfig = await fse.readFile(studioYaml);
        _.assign(config, studioConfig);
    }

    const schemaYaml = path.join(stackbitDotPath, 'schema.yaml');
    const schemaYamlExists = await fse.readFile(themeYaml);
    if (schemaYamlExists) {
        const schemaConfig = await fse.readFile(schemaYaml);
        _.assign(config, schemaConfig);
    }

    return _.isEmpty(config) ? null : config;
}

export function mergeConfigModelsWithExternalModels({
    configModels,
    externalModels,
    logger
}: {
    configModels: ModelExtension[];
    externalModels: Model[];
    logger?: Logger;
}): Model[] {
    if (configModels.length === 0) {
        return externalModels;
    }

    const mergedModelsByName: Record<string, Model> = _.keyBy(externalModels, 'name');

    for (const configModel of configModels) {
        const externalModel = mergedModelsByName[configModel.name];
        if (!externalModel) {
            logger?.warn(`Can't remap model '${configModel.name}', the model doesn't exist in content source`);
            continue;
        }

        const modelType = configModel.type ? (configModel.type === 'config' ? 'data' : configModel.type) : externalModel.type ?? 'object';

        let mergedModel: Model = Object.assign(
            {},
            externalModel,
            _.pick(configModel, [
                '__metadata',
                'urlPath',
                'label',
                'description',
                'thumbnail',
                'singleInstance',
                'readOnly',
                'canDelete',
                'labelField',
                'preview',
                'permissions',
                'localized'
            ]),
            { type: modelType }
        );

        if (mergedModel.type === 'page' && !mergedModel.urlPath) {
            mergedModel.urlPath = '/{slug}';
        }

        const externalFieldGroups = externalModel.fieldGroups && Array.isArray(externalModel.fieldGroups) ? externalModel.fieldGroups : [];
        const configFieldGroups = configModel.fieldGroups && Array.isArray(configModel.fieldGroups) ? configModel.fieldGroups : [];
        if (externalFieldGroups.length || configFieldGroups.length) {
            mergedModel.fieldGroups = _.uniqBy([...externalFieldGroups, ...configFieldGroups], 'name');
        }

        const externalActions = 'actions' in externalModel && Array.isArray(externalModel.actions) ? externalModel.actions : [];
        const configActions = 'actions' in configModel && Array.isArray(configModel.actions) ? configModel.actions : [];
        if (externalActions.length || configActions.length) {
            (mergedModel as PageModel | DataModel | ObjectModel).actions = _.uniqBy([...externalActions, ...configActions], 'name') as
                | CustomActionDocument[]
                | CustomActionObjectModel[];
        }

        mergedModel = mapModelFieldsRecursively(mergedModel, (externalField, modelKeyPath): Field => {
            const stackbitField = getModelFieldForModelKeyPath(configModel as Model, modelKeyPath) as FieldExtension;
            if (!stackbitField) {
                return externalField;
            }

            const FieldRemapMatrix: Record<string, FieldType[]> = {
                string: ['text', 'html', 'markdown', 'slug', 'url', 'color', 'date', 'datetime', 'enum', 'image', 'json', 'style', 'cross-reference'],
                text: ['html', 'markdown', 'image', 'json', 'style', 'cross-reference'],
                json: ['image', 'style', 'cross-reference']
            };

            let stackbitFieldProps: FieldExtension | DistributePartialListItems<FieldListItems>;
            if (externalField.type === 'list') {
                if (!stackbitField.type || stackbitField.type === 'list') {
                    stackbitFieldProps = 'items' in stackbitField ? stackbitField.items ?? {} : {};
                } else {
                    stackbitFieldProps = {};
                    logger?.warn(
                        `Can't remap field of model '${mergedModel.name}' ` +
                            `at path '${fieldPathToString(modelKeyPath)}' ` +
                            `from 'list' type to '${stackbitField.type}' type. ` +
                            'List fields can only be mapped to list fields'
                    );
                }
            } else if (stackbitField.type === 'list' || 'items' in stackbitField) {
                stackbitFieldProps = {};
                logger?.warn(
                    `Can't remap field of model '${mergedModel.name}' ` +
                        `at path '${fieldPathToString(modelKeyPath)}' ` +
                        `from '${externalField.type}' type to 'list' type. ` +
                        'List fields can only be mapped from list fields'
                );
            } else {
                stackbitFieldProps = stackbitField;
            }

            externalField = mapListItemsPropsOrSelfSpecificProps(externalField, (externalFieldProps) => {
                // override field type if allowed, otherwise show a warning message
                let fieldType: FieldType = externalFieldProps.type;
                const allowedOverrideTypes = FieldRemapMatrix[externalFieldProps.type];
                if (stackbitFieldProps.type && stackbitFieldProps.type !== externalFieldProps.type) {
                    if (allowedOverrideTypes && allowedOverrideTypes.includes(stackbitFieldProps.type)) {
                        fieldType = stackbitFieldProps.type;
                    } else {
                        logger?.warn(
                            `Can't remap field of model '${mergedModel.name}' at path '${fieldPathToString(modelKeyPath)}' ` +
                                `from '${externalFieldProps.type}' type to '${stackbitFieldProps.type}' type. ` +
                                `The '${externalFieldProps.type}' fields can be only mapped to ` +
                                (!allowedOverrideTypes
                                    ? 'the same type.'
                                    : allowedOverrideTypes.length === 1
                                    ? `'${allowedOverrideTypes[0]}' type`
                                    : `one of [${allowedOverrideTypes.join(', ')}] types.`)
                        );
                    }
                }

                // add field specific properties
                let fieldSpecificProps = {};
                switch (fieldType) {
                    case 'number':
                        fieldSpecificProps = _.pick(stackbitFieldProps, ['subtype', 'min', 'max', 'step', 'unit']);
                        break;
                    case 'enum':
                        fieldSpecificProps = _.pick(stackbitFieldProps, ['options']);
                        break;
                    case 'image':
                        fieldSpecificProps = _.pick(stackbitFieldProps, ['source']);
                        break;
                    case 'style':
                        fieldSpecificProps = _.pick(stackbitFieldProps, ['styles']);
                        break;
                    case 'object': {
                        fieldSpecificProps = _.pick(stackbitFieldProps, ['labelField', 'preview', 'thumbnail']);
                        const externalFieldGroups =
                            'fieldGroups' in externalFieldProps && Array.isArray(externalFieldProps.fieldGroups) ? externalFieldProps.fieldGroups : [];
                        const configFieldGroups =
                            'fieldGroups' in stackbitFieldProps && Array.isArray(stackbitFieldProps.fieldGroups) ? stackbitFieldProps.fieldGroups : [];
                        if (externalFieldGroups.length || configFieldGroups.length) {
                            fieldSpecificProps = {
                                ...fieldSpecificProps,
                                fieldGroups: _.uniqBy([...externalFieldGroups, ...configFieldGroups], 'name')
                            };
                        }
                        break;
                    }
                    case 'model':
                        fieldSpecificProps = _.pick(stackbitFieldProps, ['models']);
                        break;
                    case 'reference':
                        fieldSpecificProps = _.pick(stackbitFieldProps, ['models']);
                        break;
                    case 'cross-reference':
                        fieldSpecificProps = _.pick(stackbitFieldProps, ['models']);
                        break;
                }

                const validations = mergeFieldValidations(externalFieldProps, stackbitFieldProps);

                return Object.assign({}, externalFieldProps, fieldSpecificProps, { type: fieldType }, validations ? { validations } : undefined);
            });

            const mergedField = Object.assign(
                {},
                externalField,
                _.pick(stackbitField, [
                    'label',
                    'description',
                    'required',
                    'default',
                    'group',
                    'const',
                    'hidden',
                    'readOnly',
                    'controlType',
                    'controlUrl',
                    'controlFilePath'
                ])
            );

            // Merge list field specific properties that weren't merged in mapListItemsPropsOrSelfSpecificProps
            if (externalField.type === 'list') {
                const validations = mergeFieldValidations(externalField, stackbitField);
                if (validations) {
                    mergedField.validations = validations;
                }
            }

            const externalActions = 'actions' in externalField && Array.isArray(externalField.actions) ? externalField.actions : [];
            const configActions = 'actions' in stackbitField && Array.isArray(stackbitField.actions) ? stackbitField.actions : [];
            if (externalActions.length || configActions.length) {
                mergedField.actions = _.uniqBy([...externalActions, ...configActions], 'name');
            }

            return mergedField;
        });

        mergedModelsByName[configModel.name] = mergedModel;
    }

    return Object.values(mergedModelsByName);
}

function mergeFieldValidations<T extends StackbitTypes.FieldSpecificProps>(
    externalFieldProps: T,
    stackbitFieldProps: FieldExtension | DistributePartialListItems<FieldListItems>
): T['validations'] {
    const externalFieldValidations = externalFieldProps.validations;
    const configFieldValidation = stackbitFieldProps.validations;
    if (externalFieldValidations && configFieldValidation) {
        const extendedValidations: StackbitTypes.FieldValidationsCustom = {};
        // concatenate custom validation functions
        if (
            'validate' in externalFieldValidations &&
            externalFieldValidations.validate &&
            'validate' in configFieldValidation &&
            configFieldValidation.validate
        ) {
            const externalFileTypes = Array.isArray(externalFieldValidations.validate)
                ? externalFieldValidations.validate
                : [externalFieldValidations.validate];
            const stackbitFileTypes = Array.isArray(configFieldValidation.validate) ? configFieldValidation.validate : [configFieldValidation.validate];
            extendedValidations.validate = [...externalFileTypes, ...stackbitFileTypes];
        }
        return Object.assign({}, externalFieldValidations, configFieldValidation, extendedValidations);
    } else if (configFieldValidation) {
        return configFieldValidation;
    } else {
        return externalFieldValidations;
    }
}

function normalizeConfig(rawConfig: StackbitConfigWithPaths): Config {
    const stackbitVersion = String(_.get(rawConfig, 'stackbitVersion', LATEST_STACKBIT_VERSION));
    const ver = semver.coerce(stackbitVersion);
    const isGTEStackbitYamlV5 = ver ? semver.satisfies(ver, '>=0.5.0') : false;

    const { logicFields, models: modelMap, sitemap, ...restConfig } = rawConfig;

    // in stackbit.yaml 'models' are defined as object where keys are the model names,
    // convert 'models' to array of objects and set their 'name' property to the model name
    const models = _.reduce(modelMap, (accum: Model[], model, modelName) => accum.concat(Object.assign({ name: modelName }, model)), []);

    return {
        ...restConfig,
        stackbitVersion: stackbitVersion,
        models: models,
        ...(sitemap ? { siteMap: sitemap } : {}),
        noEncodeFields: logicFields,
        hcrHandled: !stackbitVersion || _.get(rawConfig, 'customContentReload', _.get(rawConfig, 'hcrHandled', !isGTEStackbitYamlV5)),
        contentEngine: rawConfig?.contentEngine || { port: CONTENT_ENGINE_DEFAULT_PORT },
        internalStackbitRunnerOptions: getInternalStackbitRunnerOptions(rawConfig)
    };
}

function normalizeModels(config: Config, isForcedGitCSI?: boolean): Config {
    const pageLayoutKey = config.pageLayoutKey ?? 'layout';
    const objectTypeKey = config.objectTypeKey ?? 'type';
    const stackbitYamlVersion = String(config.stackbitVersion ?? '');
    const ver = semver.coerce(stackbitYamlVersion);
    const isStackbitYamlV2 = ver ? semver.satisfies(ver, '<0.3.0') : false;
    const models = config.models;
    const modelsByName = _.keyBy(models, 'name');
    const gitCMS = isGitCMS(config);

    const mappedModels = models.map((model): Model => {
        // create shallow copy of the model to prevent mutation of original models
        model = { ...model };

        if (!_.has(model, 'type')) {
            model.type = 'object';
        }

        // add model label if not set
        if (!_.has(model, 'label')) {
            model.label = _.startCase(model.name);
        }

        if (_.has(model, 'fields') && !Array.isArray(model.fields)) {
            model.fields = [];
        }

        if (isPageModel(model)) {
            // rename old 'template' property to 'layout'
            rename(model, 'template', 'layout');

            updatePageUrlPath(model);

            if (gitCMS) {
                updatePageFilePath(model, config);
                addMarkdownContentField(model);
            }
        } else if (isDataModel(model) && gitCMS) {
            updateDataFilePath(model, config);
        }

        if (gitCMS && !isForcedGitCSI) {
            // TODO: do not add pageLayoutKey and objectTypeKey fields to models,
            //  The content validator should always assume these fields.
            //  And when new objects created from UI, it should add these fields automatically.
            if (isPageModel(model)) {
                addLayoutFieldToPageModel(model, pageLayoutKey, model.name);
            } else if (isDataModel(model) && !isListDataModel(model)) {
                addObjectTypeKeyField(model, objectTypeKey, model.name);
            }
        }

        if (isListDataModel(model)) {
            // 'items.type' of list model defaults to 'string', set it explicitly
            normalizeListFieldInPlace(model);
            if (isObjectListItems(model.items)) {
                assignLabelFieldIfNeeded(model.items);
            }
        } else if (!_.has(model, 'labelField')) {
            assignLabelFieldIfNeeded(model);
        }

        return mapModelFieldsRecursively(model, (field: Field): Field => {
            // create shallow copy of the field to prevent mutation of original field
            field = { ...field };

            // add field label if label is not set
            if (!_.has(field, 'label')) {
                field.label = _.startCase(field.name);
            }

            return mapListItemsPropsOrSelfSpecificProps(field, (fieldSpecificProps) => {
                if (isObjectField(fieldSpecificProps)) {
                    assignLabelFieldIfNeeded(fieldSpecificProps);
                } else if (isCustomModelField(fieldSpecificProps, modelsByName)) {
                    // stackbit v0.2.0 compatibility
                    // convert the old custom model field type: { type: 'action' }
                    // to the new 'model' field type: { type: 'model', models: ['action'] }
                    fieldSpecificProps = {
                        ...fieldSpecificProps,
                        type: 'model',
                        models: [fieldSpecificProps.type]
                    };
                } else if ((fieldSpecificProps as any).type === 'models') {
                    // stackbit v0.2.0 compatibility
                    // convert the old 'models' field type: { type: 'models', models: ['link', 'button'] }
                    // to the new 'model' field type: { type: 'model', models: ['link', 'button'] }
                    fieldSpecificProps = {
                        ...fieldSpecificProps,
                        type: 'model',
                        models: _.get(fieldSpecificProps, 'models', [])
                    };
                } else if (fieldSpecificProps.type === 'model' && _.has(fieldSpecificProps, 'model')) {
                    // stackbit v0.2.0 compatibility
                    // convert the old 'model' field type: { type: 'model', model: 'link' }
                    // to the new 'model' field type: { type: 'model', models: ['link'] }
                    const { model, ...rest } = fieldSpecificProps as any;
                    fieldSpecificProps = {
                        ...rest,
                        models: [model]
                    };
                }

                if (isStackbitYamlV2) {
                    // in stackbit.yaml v0.2.x, the 'reference' field was what we have today as 'model' field:
                    if (isReferenceField(fieldSpecificProps)) {
                        fieldSpecificProps = {
                            ...fieldSpecificProps,
                            type: 'model',
                            models: _.get(fieldSpecificProps, 'models', [])
                        };
                    }
                }

                return fieldSpecificProps;
            });
        });
    });

    return {
        ...config,
        models: mappedModels
    };
}

function updatePageUrlPath(model: PageModel) {
    // set default urlPath if not set
    if (!model.urlPath) {
        model.urlPath = '/{slug}';
    }
}

/**
 * Sets the page model's filePath pattern.
 * If the model has `filePath` property, it is prefixed with `pagesDir` and returned.
 * If the model has no `filePath` property, then `filePath` is naively inferred by
 * prefixing `urlPath` with `pagesDir` and appending the `.md` extension.
 */
function updatePageFilePath(model: PageModel, config: Config) {
    let filePath: string;
    if (model.filePath) {
        if (typeof model.filePath === 'function') {
            return;
        }
        filePath = model.filePath;
    } else if (model.file) {
        filePath = model.file;
    } else {
        const urlPath = model.urlPath;
        if (urlPath === '/') {
            filePath = 'index.md';
        } else if (_.trim(urlPath, '/') === 'posts/{slug}' && config.ssgName === 'jekyll') {
            filePath = '_posts/{moment_format("YYYY-MM-DD")}-{slug}.md';
        } else {
            filePath = _.trim(urlPath, '/') + '.md';
        }
    }
    const parentDir = _.trim(config.pagesDir ?? '', '/');
    model.filePath = path.join(parentDir, filePath);
}

function updateDataFilePath(model: DataModel, config: Config) {
    let filePath;
    if (model.filePath) {
        if (typeof model.filePath === 'function') {
            return;
        }
        filePath = model.filePath;
    } else if (model.file) {
        filePath = model.file;
    } else {
        const folder = _.trim(_.get(model, 'folder'), '/');
        filePath = _.trim(`${folder}/{slug}.json`, '/');
    }
    const parentDir = _.trim(config.dataDir ?? '', '/');
    model.filePath = path.join(parentDir, filePath);
}

function addMarkdownContentField(model: PageModel) {
    if (model.hideContent) {
        return;
    }
    const hasMarkdownContent = _.find(_.get(model, 'fields'), { name: 'markdown_content' });
    if (hasMarkdownContent) {
        return;
    }
    append(model, 'fields', {
        type: 'markdown',
        name: 'markdown_content',
        label: 'Content',
        description: 'Page content'
    });
}

function addLayoutFieldToPageModel(model: PageModel, pageLayoutKey: string, modelName: string) {
    if (_.intersection(_.keys(model), ['file', 'folder', 'match', 'exclude']).length === 0 && !_.get(model, 'layout')) {
        model.layout = modelName;
    }
    const modelLayout = _.get(model, 'layout');
    if (!modelLayout) {
        return;
    }
    const hasLayoutField = _.find(_.get(model, 'fields'), { name: pageLayoutKey });
    if (hasLayoutField) {
        return;
    }
    prepend(model, 'fields', {
        type: 'string',
        name: pageLayoutKey,
        label: _.startCase(pageLayoutKey),
        const: modelLayout,
        hidden: true
    });
}

function addObjectTypeKeyField(model: DataModelSingle, objectTypeKey: string, modelName: string) {
    const hasObjectTypeField = _.find(_.get(model, 'fields'), { name: objectTypeKey });
    if (hasObjectTypeField) {
        return;
    }
    prepend(model, 'fields', {
        type: 'string',
        name: objectTypeKey,
        label: 'Object Type',
        description: 'The type of the object',
        const: modelName,
        hidden: true
    });
}

/**
 * Returns model names referenced by polymorphic 'model' and 'reference' fields.
 *
 * @param field
 */
function getReferencedModelNames(field: Field) {
    const fieldSpecificProps = getListItemsOrSelf(field);
    // TODO: add type field to model fields inside container update/create object logic rather adding type to schema
    // 'object' models referenced by 'model' fields should have 'type' field
    // if these fields have than 1 model.
    // 'data' models referenced by 'reference' fields should always have 'type' field.
    let referencedModelNames: string[] = [];
    if (isModelField(fieldSpecificProps) && fieldSpecificProps.models?.length > 1) {
        const modelNames = fieldSpecificProps.models;
        referencedModelNames = _.union(referencedModelNames, modelNames);
    } else if (isReferenceField(fieldSpecificProps) && fieldSpecificProps.models?.length > 0) {
        const modelNames = fieldSpecificProps.models;
        referencedModelNames = _.union(referencedModelNames, modelNames);
    }
    return referencedModelNames;
}

function validateAndExtendContentModels(config: Config): {
    config: Config;
    errors: ConfigValidationError[];
} {
    const contentModels = config.contentModels ?? {};
    const models = config.models ?? [];

    // external models already merged in mergeConfigModelsWithExternalModels function
    const externalModels = !isGitCMS(config);
    const emptyContentModels = _.isEmpty(contentModels);

    if (externalModels || emptyContentModels) {
        return {
            config: config,
            errors: []
        };
    }

    const validationResult = validateContentModels(contentModels, models);

    if (_.isEmpty(models)) {
        return {
            config: config,
            errors: validationResult.errors
        };
    }

    const extendedModels = models.map((model): Model => {
        const contentModel = validationResult.contentModels![model.name];
        if (!contentModel) {
            return model;
        }
        if (_.get(contentModel, '__metadata.invalid')) {
            return model;
        }
        const { isPage, newFilePath, ...restContentModel } = contentModel;
        const { type, ...restModel } = model;
        if (isPage && (!type || ['object', 'page'].includes(type))) {
            return {
                type: 'page',
                ...(newFilePath ? { filePath: newFilePath } : {}),
                ...restContentModel,
                ...restModel
            } as PageModel;
        } else if (!isPage && (!type || ['object', 'data'].includes(type))) {
            return {
                type: 'data',
                ...(newFilePath ? { filePath: newFilePath } : {}),
                ...restContentModel,
                ...restModel
            } as DataModel;
        } else {
            return model;
        }
    });

    return {
        config: {
            ...config,
            models: extendedModels
        },
        errors: validationResult.errors
    };
}

function normalizeValidationResult(validationResult: ConfigValidationResult): ConfigValidationResult {
    validationResult = filterAndOrderConfigFields(validationResult);
    return convertModelGroupsToModelListInPlace(validationResult);
}

function filterAndOrderConfigFields(validationResult: ConfigValidationResult): ConfigValidationResult {
    // TODO: check if we can move filtering and sorting to Joi
    return {
        ...validationResult,
        config: _.pick(validationResult.config, [
            'stackbitVersion',
            'ssgName',
            'ssgVersion',
            'cmsName',
            'import',
            'buildCommand',
            'publishDir',
            'nodeVersion',
            'postGitCloneCommand',
            'preInstallCommand',
            'postInstallCommand',
            'installCommand',
            'devCommand',
            'cacheDir',
            'staticDir',
            'uploadDir',
            'assets',
            'pagesDir',
            'dataDir',
            'pageLayoutKey',
            'objectTypeKey',
            'styleObjectModelName',
            'excludePages',
            'logicFields',
            'contentModels',
            'presetSource',
            'modelsSource',
            'mapModels',
            'presetReferenceBehavior',
            'nonDuplicatableModels',
            'duplicatableModels',
            'contentSources',
            'hcrHandled',
            'internalStackbitRunnerOptions',
            'dirPath',
            'filePath',
            'models',
            'presets',
            'sidebarButtons',
            'pageData', // obsolete, left for backward compatibility
            'pageModels', // obsolete, left for backward compatibility
            'encodedFieldTypes', // obsolete, left for backward compatibility
            'noEncodeFields', // obsolete, left for backward compatibility
            'omitFields' // obsolete, left for backward compatibility
        ]) as Config
    };
}

/**
 * Collects models groups and injects them into the `models` array of the
 * `reference` and `model` field types
 */
function convertModelGroupsToModelListInPlace(validationResult: ConfigValidationResult): ConfigValidationResult {
    const models = validationResult.config?.models ?? [];

    const groupMap = _.reduce(
        models,
        (groupMap, model) => {
            if (!model.groups) {
                return groupMap;
            }
            const key = model?.type === 'object' ? 'objectModels' : 'documentModels';
            _.forEach(model.groups, (groupName) => {
                append(groupMap, [groupName, key], model.name);
            });
            delete model.groups;
            return groupMap;
        },
        {} as Record<string, { objectModels?: string[]; documentModels?: string[] }>
    );

    // update groups to have unique model names
    _.forEach(groupMap, (group) => {
        _.forEach(group, (modelGroup, key) => {
            _.set(group, key, _.uniq(modelGroup));
        });
    });

    const mappedModels = models.map((model) => {
        return mapModelFieldsRecursively(model, (field): Field => {
            return mapListItemsPropsOrSelfSpecificProps(field, (fieldSpecificProps) => {
                if (!isModelField(fieldSpecificProps) && !isReferenceField(fieldSpecificProps)) {
                    return fieldSpecificProps;
                }
                const { ...cloned } = fieldSpecificProps;
                const key = isModelField(fieldSpecificProps) ? 'objectModels' : 'documentModels';
                const modelNames = _.reduce(
                    cloned.groups,
                    (modelNames, groupName) => {
                        const objectModelNames = _.get(groupMap, [groupName, key], []);
                        return _.uniq(modelNames.concat(objectModelNames));
                    },
                    fieldSpecificProps.models || []
                );
                delete cloned.groups;
                return Object.assign(cloned, { models: modelNames });
            });
        });
    });

    return {
        ...validationResult,
        config: {
            ...validationResult.config,
            models: mappedModels
        }
    };
}

function isGitCMS(config: Config) {
    return !config.contentSources && (!config.cmsName || config.cmsName === 'git');
}

function getInternalStackbitRunnerOptions(config: StackbitConfigWithPaths): SSGRunOptions | undefined {
    const experimentalSsgData = config?.experimental?.ssg || ((config as any).__unsafe_internal_stackbitRunnerOptions as Required<Experimental>['ssg']);

    if (!experimentalSsgData) {
        return;
    }

    let doneStart = experimentalSsgData.logPatterns?.up as string | string[] | undefined;
    if (typeof doneStart === 'string') {
        doneStart = [doneStart];
    }

    const ssgRunOptions = omitByNil({
        displayName: experimentalSsgData.name,
        triggerInstallFiles: experimentalSsgData.watch?.reinstallPackages,
        directPaths: experimentalSsgData.passthrough,
        directRoutes: experimentalSsgData.directRoutes,
        proxyWebsockets: experimentalSsgData.proxyWebsockets,
        ...(doneStart ? { patterns: { doneStart } } : undefined)
    });
    return _.isEmpty(ssgRunOptions) ? undefined : ssgRunOptions;
}