myst-cli/dist/config.js

Command line tools for MyST

import fs from 'node:fs';
import { dirname, relative, resolve } from 'node:path';
import yaml from 'js-yaml';
import { writeFileToFolder } from 'myst-cli-utils';
import { fileError, fileWarn, RuleId } from 'myst-common';
import { validateProjectConfig, validateSiteConfig } from 'myst-config';
import { fillProjectFrontmatter, fillSiteFrontmatter } from 'myst-frontmatter';
import { incrementOptions, validateObjectKeys, validationError, validateList, validateString, } from 'simple-validators';
import { VFile } from 'vfile';
import { prepareToWrite } from './frontmatter.js';
import { loadFrontmatterParts } from './process/file.js';
import { selectors } from './store/index.js';
import { config } from './store/reducers.js';
import { logMessagesFromVFile } from './utils/logging.js';
import { addWarningForFile } from './utils/addWarningForFile.js';
import { resolveToAbsolute } from './utils/resolveToAbsolute.js';
const VERSION = 1;
function emptyConfig() {
    return {
        version: VERSION,
    };
}
export function defaultConfigFile(session, path) {
    return resolve(path, session.configFiles[0]);
}
/**
 * Find config file at the specified path
 *
 * @param session with valid config filenames
 * @param path directory to look for config file
 * @returns the resolved path to the config file, or undefined if no config file is found
 * @throws an error if multiple config files are found in the path
 */
export function configFromPath(session, path) {
    const configs = session.configFiles
        .map((file) => {
        return resolve(path, file);
    })
        .filter((file) => {
        return fs.existsSync(file);
    });
    if (configs.length > 1)
        throw Error(`Multiple config files in ${path}`);
    if (configs.length === 0)
        return undefined;
    return configs[0];
}
/**
 * Load config yaml file and throw error if it fails
 */
function loadConfigYaml(file) {
    if (!fs.existsSync(file))
        throw Error(`Cannot find config file: ${file}`);
    let rawConf;
    try {
        rawConf = yaml.load(fs.readFileSync(file, 'utf-8'));
    }
    catch (err) {
        const suffix = err.message ? `\n\n${err.message}` : '';
        throw Error(`Unable to read config file ${file} as YAML${suffix}`);
    }
    return rawConf;
}
/**
 * Helper function to generate basic validation options
 */
function configValidationOpts(vfile, property, ruleId) {
    return {
        file: vfile.path,
        property,
        messages: {},
        errorLogFn: (message) => {
            fileError(vfile, message, { ruleId });
        },
        warningLogFn: (message) => {
            fileWarn(vfile, message, { ruleId });
        },
    };
}
/**
 * Function to add filler keys to base if the keys are not defined in base
 */
function fillSiteConfig(base, filler, opts) {
    return fillSiteFrontmatter(base, filler, opts, Object.keys(filler));
}
/**
 * Mutate config object to coerce deprecated frontmatter fields to valid schema
 */
export function handleDeprecatedFields(conf, file, vfile) {
    if (conf.site?.frontmatter) {
        fileWarn(vfile, `Frontmatter fields should be defined directly on site, not nested under "${file}#site.frontmatter"`, { ruleId: RuleId.configHasNoDeprecatedFields });
        const { frontmatter, ...rest } = conf.site;
        conf.site = { ...frontmatter, ...rest };
    }
    if (conf.project?.frontmatter) {
        fileWarn(vfile, `Frontmatter fields should be defined directly on project, not nested under "${file}#project.frontmatter"`, { ruleId: RuleId.configHasNoDeprecatedFields });
        const { frontmatter, ...rest } = conf.project;
        conf.project = { ...frontmatter, ...rest };
    }
    if (conf.project?.biblio) {
        fileWarn(vfile, `biblio is deprecated, please use first_page/last_page/volume/issue fields "${file}#project"`, { ruleId: RuleId.configHasNoDeprecatedFields });
        const { biblio, ...rest } = conf.project;
        conf.project = { ...biblio, ...rest };
    }
    if (conf.site?.logoText) {
        fileWarn(vfile, `logoText is deprecated, please use logo_text in "${file}#site"`, {
            ruleId: RuleId.configHasNoDeprecatedFields,
        });
        const { logoText, ...rest } = conf.site;
        conf.site = { logo_text: logoText, ...rest };
    }
}
/**
 * Load and validate a file as yaml config file
 *
 * @returns the validated site and project configs and list of extended config files
 * @throws an error if the config file is malformed or invalid
 */
async function getValidatedConfigsFromFile(session, file, projectPath, vfile, stack) {
    if (!vfile) {
        vfile = new VFile();
        vfile.path = file;
    }
    const opts = configValidationOpts(vfile, 'config', RuleId.validConfigStructure);
    const conf = validateObjectKeys(loadConfigYaml(file), {
        required: ['version'],
        optional: ['site', 'project', 'extend'],
        alias: { extends: 'extend' },
    }, opts);
    if (conf && conf.version !== VERSION) {
        validationError(`"${conf.version}" does not match ${VERSION}`, incrementOptions('version', opts));
    }
    logMessagesFromVFile(session, vfile);
    if (!conf || opts.messages.errors) {
        throw Error(`Please address invalid config file ${file}`);
    }
    // Keep original config object with extra keys, etc.
    handleDeprecatedFields(conf, file, vfile);
    let site;
    let project;
    const projectOpts = configValidationOpts(vfile, 'config.project', RuleId.validProjectConfig);
    let extend;
    if (conf.extend) {
        extend = await Promise.all((validateList(conf.extend, { coerce: true, ...incrementOptions('extend', opts) }, (item, index) => {
            return validateString(item, incrementOptions(`extend.${index}`, opts));
        }) ?? []).map(async (extendFile) => {
            const resolvedFile = await resolveToAbsolute(session, dirname(file), extendFile, {
                allowRemote: true,
            });
            return resolvedFile;
        }));
        stack = [...(stack ?? []), file];
        await Promise.all((extend ?? []).map(async (extFile) => {
            if (stack?.includes(extFile)) {
                fileError(vfile, 'Circular dependency encountered during "config.extend" resolution', {
                    ruleId: RuleId.validConfigStructure,
                    note: [...stack, extFile].map((f) => resolveToRelative(session, '.', f)).join(' > '),
                });
                return;
            }
            const { site: extSite, project: extProject } = await getValidatedConfigsFromFile(session, extFile, projectPath, vfile, stack);
            session.store.dispatch(config.actions.receiveConfigExtension({ file: extFile }));
            if (extSite) {
                site = site ? fillSiteConfig(extSite, site, incrementOptions('extend', opts)) : extSite;
            }
            if (extProject) {
                project = project ? fillProjectFrontmatter(extProject, project, projectOpts) : extProject;
            }
        }));
    }
    const { site: rawSite, project: rawProject } = conf ?? {};
    if (rawProject) {
        project = fillProjectFrontmatter(await validateProjectConfigAndThrow(session, projectPath, vfile, rawProject), project ?? {}, projectOpts);
    }
    if (project) {
        session.log.debug(`Loaded project config from ${file}`);
    }
    else {
        session.log.debug(`No project config defined in ${file}`);
    }
    if (rawSite) {
        site = fillSiteConfig(await validateSiteConfigAndThrow(session, projectPath, vfile, rawSite), site ?? {}, incrementOptions('extend', opts));
    }
    if (site) {
        session.log.debug(`Loaded site config from ${file}`);
    }
    else {
        session.log.debug(`No site config in ${file}`);
    }
    logMessagesFromVFile(session, vfile);
    return { site, project, extend };
}
/**
 * Load site/project config from local path to redux store
 *
 * Validates the loaded config and stores raw/validated values in redux.
 *
 * @param session with logging and redux store
 * @param path local directory to load config from
 * @param opts.reloadProject if true, reload the project config even if it already exists in the redux store
 * @returns the validated config, or undefined when no config file exists at the path
 * @throws an error if the config is invalid
 */
export async function loadConfig(session, path, opts) {
    const file = configFromPath(session, path);
    if (!file) {
        session.log.debug(`No config loaded from path: ${path}`);
        return;
    }
    const rawConf = loadConfigYaml(file);
    if (!opts?.reloadProject) {
        const existingConf = selectors.selectLocalRawConfig(session.store.getState(), path);
        if (existingConf && JSON.stringify(rawConf) === JSON.stringify(existingConf.raw)) {
            return existingConf.validated;
        }
    }
    const { extend, ...configs } = await getValidatedConfigsFromFile(session, file, path);
    const site = await loadAndResolveConfigParts(session, path, configs.site, file, 'site');
    const project = await loadAndResolveConfigParts(session, path, configs.project, file, 'project');
    const validated = { ...rawConf, site, project, extend };
    session.store.dispatch(config.actions.receiveRawConfig({
        path,
        file,
        raw: rawConf,
        validated,
    }));
    if (site)
        saveSiteConfig(session, path, site);
    if (project)
        saveProjectConfig(session, path, project);
    return validated;
}
/**
 * Resolve an absolute path to a relative path.
 *
 * Note: the absolute-path helper lives in `utils/resolveToAbsolute.ts`
 * We moved it there to avoid circular dependencies.
 */
function resolveToRelative(session, basePath, absPath, opts) {
    let message;
    try {
        if (opts?.allowNotExist || fs.existsSync(absPath)) {
            // If it is the same path, use a '.'
            return relative(basePath, absPath) || '.';
        }
        message = `Does not exist as local path: ${absPath}`;
    }
    catch {
        message = `Unable to resolve as relative path: ${absPath}`;
    }
    session.log.debug(message);
    return absPath;
}
/**
 * Resolve path-based fields in a site config.
 *
 * This function may be used for resolving relative local paths or resolving remote paths on a server,
 * depending on the implementation of the resolutionFn.
 *
 * @param session session with logging
 * @param path base path for config file directory, used for relative path resolution
 * @param siteConfig site config to resolve
 * @param resolutionFn function to resolve each path value
 * @returns copy of the site config with resolved path fields
 */
async function resolveSiteConfigPaths(session, path, siteConfig, resolutionFn) {
    const resolvedFields = {};
    if (siteConfig.projects) {
        resolvedFields.projects = await Promise.all(siteConfig.projects.map(async (proj) => {
            if (proj.path) {
                return { ...proj, path: await resolutionFn(session, path, proj.path) };
            }
            return proj;
        }));
    }
    return { ...siteConfig, ...resolvedFields };
}
/**
 * Resolve path-based fields in a project config.
 *
 * This function may be used for resolving relative local paths or resolving remote paths on a server,
 * depending on the implementation of the resolutionFn.
 *
 * Additionally, this function loads configured plugins immediately once the paths are resolved
 *
 * @param session session with logging
 * @param path base path for config file directory, used for relative path resolution
 * @param projectConfig project config to resolve
 * @param resolutionFn function used to resolve each path value
 * @returns copy of the project config with resolved path fields
 */
async function resolveProjectConfigPaths(session, path, projectConfig, resolutionFn) {
    const resolvedFields = {};
    if (projectConfig.bibliography) {
        resolvedFields.bibliography = await Promise.all(projectConfig.bibliography.map(async (f) => {
            const resolved = await resolutionFn(session, path, f);
            return resolved;
        }));
    }
    if (projectConfig.index) {
        resolvedFields.index = await resolutionFn(session, path, projectConfig.index);
    }
    if (projectConfig.plugins) {
        resolvedFields.plugins = await Promise.all(projectConfig.plugins.map(async (info) => {
            const resolved = await resolutionFn(session, path, info.path, {
                allowRemote: info.type !== 'executable',
            });
            if (fs.existsSync(resolved)) {
                return { ...info, path: resolved };
            }
            else {
                return info;
            }
        }));
        await session.loadPlugins(resolvedFields.plugins);
    }
    return { ...projectConfig, ...resolvedFields };
}
/**
 * Resolve `parts` entries in a site or project config.
 *
 * This will process markdown written directly in the config file and save it to the session store.
 * It will replace the `parts` entry in the config with a reference to the stored part.
 *
 * @param session session with logging
 * @param path base path for config file directory
 * @param configWithParts site/project config that may define `parts`
 * @param file config file path for error reporting
 * @param property config key used for validation messages (`site` or `project`)
 * @returns a shallow copy of the config with resolved parts, or undefined if no config is provided
 */
export async function loadAndResolveConfigParts(session, path, configWithParts, file, property) {
    if (!configWithParts)
        return undefined;
    if (!configWithParts.parts)
        return { ...configWithParts };
    const resolvedParts = await loadFrontmatterParts(session, file, `${property}.parts`, { parts: configWithParts.parts }, path);
    return { ...configWithParts, parts: resolvedParts };
}
async function validateSiteConfigAndThrow(session, path, vfile, rawSite) {
    const site = validateSiteConfig(rawSite, configValidationOpts(vfile, 'config.site', RuleId.validSiteConfig));
    logMessagesFromVFile(session, vfile);
    if (!site) {
        const errorSuffix = vfile.path ? ` in ${vfile.path}` : '';
        throw Error(`Please address invalid site config${errorSuffix}`);
    }
    return resolveSiteConfigPaths(session, path, site, resolveToAbsolute);
}
function saveSiteConfig(session, path, site) {
    session.store.dispatch(config.actions.receiveSiteConfig({ path, ...site }));
}
async function validateProjectConfigAndThrow(session, path, vfile, rawProject) {
    const project = validateProjectConfig(rawProject, configValidationOpts(vfile, 'config.project', RuleId.validProjectConfig));
    logMessagesFromVFile(session, vfile);
    if (!project) {
        const errorSuffix = vfile.path ? ` in ${vfile.path}` : '';
        throw Error(`Please address invalid project config${errorSuffix}`);
    }
    return resolveProjectConfigPaths(session, path, project, resolveToAbsolute);
}
function saveProjectConfig(session, path, project) {
    session.store.dispatch(config.actions.receiveProjectConfig({ path, ...project }));
}
/**
 * Write site config and project config to file
 *
 * If newConfigs are provided, the redux store will be updated with these
 * configs before writing.
 *
 * If a config file exists on the path, this will override only the new portions
 * of the config and leave the rest (i.e. if `newConfigs.siteConfig` is undefined,
 * everything under the `site` key in the existing config will be unchanged).
 *
 * @param session with logging and redux store
 * @param path directory to write config to
 * @param newConfigs site and project configs to write
 */
export async function writeConfigs(session, path, newConfigs) {
    // TODO: siteConfig -> rawSiteConfig before writing, don't lose extra keys in raw.
    //       also shouldn't need to re-readConfig...
    let { siteConfig, projectConfig } = newConfigs || {};
    const file = configFromPath(session, path) || defaultConfigFile(session, path);
    // Get site config to save
    const vfile = new VFile();
    vfile.path = file;
    // Get project config to save
    if (projectConfig) {
        saveProjectConfig(session, path, await validateProjectConfigAndThrow(session, path, vfile, projectConfig));
    }
    projectConfig = selectors.selectLocalProjectConfig(session.store.getState(), path);
    if (projectConfig) {
        projectConfig = prepareToWrite(projectConfig);
        projectConfig = await resolveProjectConfigPaths(session, path, projectConfig, resolveToRelative);
        projectConfig = await loadAndResolveConfigParts(session, path, projectConfig, file, 'project');
    }
    if (siteConfig) {
        saveSiteConfig(session, path, await validateSiteConfigAndThrow(session, path, vfile, siteConfig));
    }
    siteConfig = selectors.selectLocalSiteConfig(session.store.getState(), path);
    if (siteConfig) {
        siteConfig = await resolveSiteConfigPaths(session, path, siteConfig, resolveToRelative);
        siteConfig = await loadAndResolveConfigParts(session, path, siteConfig, file, 'site');
    }
    // Return early if nothing new to save
    if (!siteConfig && !projectConfig) {
        session.log.debug(`No new config to write to ${file}`);
        return;
    }
    // Get raw config to override
    const validatedRawConfig = (await loadConfig(session, path)) ?? emptyConfig();
    let logContent;
    if (siteConfig && projectConfig) {
        logContent = 'site and project configs';
    }
    else if (siteConfig) {
        logContent = 'site config';
    }
    else {
        logContent = 'project config';
    }
    session.log.debug(`Writing ${logContent} to ${file}`);
    // Combine site/project configs with
    const newConfig = { ...validatedRawConfig };
    if (siteConfig)
        newConfig.site = { ...validatedRawConfig.site, ...siteConfig };
    if (projectConfig)
        newConfig.project = { ...validatedRawConfig.project, ...projectConfig };
    writeFileToFolder(file, yaml.dump(newConfig), 'utf-8');
}
export async function findCurrentProjectAndLoad(session, path) {
    path = resolve(path);
    if (configFromPath(session, path)) {
        await loadConfig(session, path);
        const project = selectors.selectLocalProjectConfig(session.store.getState(), path);
        if (project) {
            session.store.dispatch(config.actions.receiveCurrentProjectPath({ path: path }));
            return path;
        }
    }
    if (dirname(path) === path) {
        return undefined;
    }
    return findCurrentProjectAndLoad(session, dirname(path));
}
export async function findCurrentSiteAndLoad(session, path) {
    path = resolve(path);
    if (configFromPath(session, path)) {
        await loadConfig(session, path);
        const site = selectors.selectLocalSiteConfig(session.store.getState(), path);
        if (site) {
            session.store.dispatch(config.actions.receiveCurrentSitePath({ path: path }));
            return path;
        }
    }
    if (dirname(path) === path) {
        return undefined;
    }
    return findCurrentSiteAndLoad(session, dirname(path));
}
export async function reloadAllConfigsForCurrentSite(session) {
    const state = session.store.getState();
    const sitePath = selectors.selectCurrentSitePath(state);
    const file = selectors.selectCurrentProjectFile(state) ?? defaultConfigFile(session, resolve('.'));
    if (!sitePath) {
        const message = 'Cannot (re)load site config. No configuration file found with "site" property.';
        addWarningForFile(session, file, message, 'error', { ruleId: RuleId.siteConfigExists });
        throw Error(message);
    }
    await loadConfig(session, sitePath);
    const siteConfig = selectors.selectCurrentSiteConfig(session.store.getState());
    if (!siteConfig?.projects)
        return;
    await Promise.all(siteConfig.projects
        .filter((project) => {
        return Boolean(project.path);
    })
        .map(async (project) => {
        try {
            await loadConfig(session, project.path);
        }
        catch (error) {
            // TODO: what error?
            session.log.debug(`Failed to find or load project config from "${project.path}"`);
        }
    }));
}