myst-cli
Version:
Command line tools for MyST
303 lines (302 loc) • 14.8 kB
JavaScript
import path from 'node:path';
import { tic } from 'myst-cli-utils';
import { fileError, fileWarn, RuleId, slugToUrl } from 'myst-common';
import { SourceFileKind } from 'myst-spec-ext';
import { basicTransformationsPlugin, htmlPlugin, footnotesPlugin, MultiPageReferenceResolver, resolveLinksAndCitationsTransform, resolveReferencesTransform, mathPlugin, codePlugin, keysTransform, linksTransform, MystTransformer, SphinxTransformer, WikiTransformer, GithubTransformer, RRIDTransformer, RORTransformer, DOITransformer, glossaryPlugin, abbreviationPlugin, reconstructHtmlPlugin, inlineMathSimplificationPlugin, checkLinkTextTransform, indexIdentifierPlugin, buildTocTransform, } from 'myst-transforms';
import { unified } from 'unified';
import { select, selectAll } from 'unist-util-select';
import { VFile } from 'vfile';
import { processPageFrontmatter, updateFileInfoFromFrontmatter } from '../frontmatter.js';
import { selectors } from '../store/index.js';
import { castSession } from '../session/cache.js';
import { checkLinksTransform, embedTransform, importMdastFromJson, includeFilesTransform, liftCodeMetadataToBlock, transformCitations, transformImageFormats, transformLinkedDOIs, transformLinkedRORs, transformOutputsToCache, transformRenderInlineExpressions, transformThumbnail, StaticFileTransformer, propagateBlockDataToCode, transformBanner, reduceOutputs, transformPlaceholderImages, transformDeleteBase64UrlSource, transformWebp, transformOutputsToFile, transformImagesToEmbed, transformImagesWithoutExt, transformImagesToDisk, transformFilterOutputStreams, transformLiftCodeBlocksInJupytext, transformMystXRefs, transformWidgetStaticAssetsToDisk, } from '../transforms/index.js';
import { logMessagesFromVFile } from '../utils/logging.js';
import { combineCitationRenderers } from './citations.js';
import { bibFilesInDir, selectFile } from './file.js';
import { parseMyst } from './myst.js';
import { kernelExecutionTransform, LocalDiskCache } from 'myst-execute';
import { rawDirectiveTransform } from '../transforms/raw.js';
import { addEditUrl } from '../utils/addEditUrl.js';
import { indexFrontmatterFromProject, manifestPagesFromProject, manifestTitleFromProject, } from '../build/utils/projectManifest.js';
const LINKS_SELECTOR = 'link,card,linkBlock';
const pluginUtils = { select, selectAll };
const htmlHandlers = {
comment(h, node) {
// Prevents HTML comments from showing up as text in web
// TODO: Remove once this is landed in myst-parser
const result = h(node, 'comment');
result.value = node.value;
return result;
},
};
export async function transformMdast(session, opts) {
const { file, projectPath, pageSlug, projectSlug, imageExtensions, extraTransforms, watchMode = false, minifyMaxCharacters, index, titleDepth, // Related to title set in markdown, rather than frontmatter
offset, // Related to multi-page nesting
execute, } = opts;
const toc = tic();
const { store, log } = session;
const cache = castSession(session);
if (!cache.$getMdast(file))
return;
const { mdast: mdastPre, kind, frontmatter: preFrontmatter, location, identifiers, widgets, } = cache.$getMdast(file)?.pre ?? {};
if (!mdastPre || !kind || !location)
throw new Error(`Expected mdast to be parsed for ${file}`);
log.debug(`Processing "${file}"`);
const vfile = new VFile(); // Collect errors on this file
vfile.path = file;
const mdast = structuredClone(mdastPre);
const frontmatter = processPageFrontmatter(session, preFrontmatter ?? {}, {
property: 'frontmatter',
file,
messages: {},
errorLogFn: (message) => {
fileError(vfile, message, { ruleId: RuleId.validPageFrontmatter });
},
warningLogFn: (message) => {
fileWarn(vfile, message, { ruleId: RuleId.validPageFrontmatter });
},
}, projectPath);
if (offset) {
if (!frontmatter.numbering)
frontmatter.numbering = {};
if (!frontmatter.numbering.title)
frontmatter.numbering.title = {};
if (frontmatter.numbering.title.offset == null)
frontmatter.numbering.title.offset = offset;
}
await addEditUrl(session, frontmatter, file);
const references = {
cite: { order: [], data: {} },
};
// Import additional content from mdast or other files
importMdastFromJson(session, file, mdast);
await includeFilesTransform(session, file, mdast, frontmatter, vfile);
rawDirectiveTransform(mdast, vfile);
// This needs to come before basic transformations since it may add labels to blocks
liftCodeMetadataToBlock(session, vfile, mdast);
if (execute && !frontmatter.execute?.skip) {
const cachePath = path.join(session.buildPath(), 'execute');
const fileName = path.basename(file);
session.log.debug(`⏳ Waiting for execution slot: ${fileName}`);
await session.executionSemaphore.runExclusive(async () => {
session.log.debug(`▶️ Executing: ${fileName}`);
await kernelExecutionTransform(mdast, vfile, {
basePath: session.sourcePath(),
cache: new LocalDiskCache(cachePath),
sessionFactory: () => session.jupyterSessionManager(),
frontmatter: frontmatter,
ignoreCache: false,
errorIsFatal: false,
log: session.log,
});
session.log.debug(`✅ Completed execution: ${fileName}`);
});
}
const pipe = unified()
.use(reconstructHtmlPlugin) // We need to group and link the HTML first
.use(htmlPlugin, { htmlHandlers }) // Some of the HTML plugins need to operate on the transformed html, e.g. figure caption transforms
.use(basicTransformationsPlugin, {
parser: (content) => parseMyst(session, content, file),
firstDepth: (titleDepth ?? 1) + (frontmatter.content_includes_title ? 0 : 1),
})
.use(inlineMathSimplificationPlugin, { replaceSymbol: false })
.use(mathPlugin, { macros: frontmatter.math });
// Load custom transform plugins
session.plugins?.transforms.forEach((t) => {
if (t.stage !== 'document')
return;
pipe.use(t.plugin, undefined, pluginUtils);
});
pipe
.use(glossaryPlugin) // This should be before the enumerate plugins
.use(abbreviationPlugin, { abbreviations: frontmatter.abbreviations })
.use(indexIdentifierPlugin);
await pipe.run(mdast, vfile);
// This needs to come after basic transformations since meta tags are added there
propagateBlockDataToCode(session, vfile, mdast);
// Initialize citation renderers for this (non-bib) file
cache.$citationRenderers[file] = await transformLinkedDOIs(session, vfile, mdast, cache.$doiRenderers, file);
const rendererFiles = [file];
if (projectPath) {
rendererFiles.unshift(projectPath);
}
else {
const localFiles = (await bibFilesInDir(session, path.dirname(file))) || [];
rendererFiles.push(...localFiles);
}
// Combine file-specific citation renderers with project renderers from bib files
const fileCitationRenderer = combineCitationRenderers(cache, ...rendererFiles);
transformRenderInlineExpressions(mdast, vfile);
await transformOutputsToCache(session, mdast, kind, { minifyMaxCharacters });
transformFilterOutputStreams(mdast, vfile, frontmatter.settings);
transformCitations(session, file, mdast, fileCitationRenderer, references);
await unified()
.use(codePlugin, { lang: frontmatter?.kernelspec?.language })
.use(footnotesPlugin) // Needs to happen near the end
.run(mdast, vfile);
transformImagesToEmbed(mdast);
transformImagesWithoutExt(session, mdast, file, { imageExtensions });
const isJupytext = frontmatter.kernelspec || frontmatter.jupytext;
if (isJupytext)
transformLiftCodeBlocksInJupytext(mdast);
const sha256 = selectors.selectFileInfo(store.getState(), file).sha256;
const useSlug = pageSlug !== index;
let url;
let dataUrl;
if (pageSlug && projectSlug) {
url = `/${projectSlug}/${useSlug ? pageSlug : ''}`;
dataUrl = `/${projectSlug}/${pageSlug}.json`;
}
else if (pageSlug) {
url = `/${useSlug ? pageSlug : ''}`;
dataUrl = `/${pageSlug}.json`;
}
url = slugToUrl(url);
updateFileInfoFromFrontmatter(session, file, frontmatter, url, dataUrl);
const data = {
kind: isJupytext ? SourceFileKind.Notebook : kind,
file,
location,
sha256,
slug: pageSlug,
dependencies: [],
frontmatter,
mdast,
references,
identifiers,
widgets,
};
const cachedMdast = cache.$getMdast(file);
if (cachedMdast)
cachedMdast.post = data;
if (extraTransforms) {
await Promise.all(extraTransforms.map(async (transform) => {
await transform(session, opts);
}));
}
logMessagesFromVFile(session, vfile);
if (!watchMode)
log.info(toc(`📖 Built ${file} in %s.`));
}
export async function postProcessMdast(session, { file, checkLinks, pageReferenceStates, extraLinkTransformers, site, }) {
const toc = tic();
const { log } = session;
const cache = castSession(session);
const mdastPost = selectFile(session, file);
if (!mdastPost)
return;
const vfile = new VFile(); // Collect errors on this file
vfile.path = file;
const { mdast, dependencies, frontmatter } = mdastPost;
const state = new MultiPageReferenceResolver(pageReferenceStates, file, vfile);
const externalReferences = Object.values(cache.$externalReferences);
const storeState = session.store.getState();
const projectPath = selectors.selectCurrentProjectPath(storeState);
const siteConfig = selectors.selectCurrentSiteConfig(storeState);
const projectSlug = siteConfig?.projects?.find((proj) => proj.path === projectPath)?.slug;
if (site) {
buildTocTransform(mdast, vfile, projectPath
? [
{
title: manifestTitleFromProject(session, projectPath),
level: 1,
slug: '',
enumerator: indexFrontmatterFromProject(session, projectPath).enumerator,
},
...(await manifestPagesFromProject(session, projectPath)),
]
: undefined, projectSlug, mdastPost.slug);
}
// NOTE: This is doing things in place, we should potentially make this a different state?
const transformers = [
...(extraLinkTransformers || []),
new WikiTransformer(),
new GithubTransformer(),
new RRIDTransformer(),
new RORTransformer(),
new DOITransformer(), // This also is picked up in the next transform
new MystTransformer(externalReferences),
new SphinxTransformer(externalReferences),
new StaticFileTransformer(session, file), // Links static files and internally linked files
];
resolveLinksAndCitationsTransform(mdast, { state, transformers });
linksTransform(mdast, vfile, {
transformers,
selector: LINKS_SELECTOR,
});
await transformLinkedRORs(session, vfile, mdast, file);
resolveReferencesTransform(mdast, vfile, { state, transformers });
await transformMystXRefs(session, vfile, mdast, frontmatter);
await embedTransform(session, mdast, file, dependencies, state);
const pipe = unified();
session.plugins?.transforms.forEach((t) => {
if (t.stage !== 'project')
return;
pipe.use(t.plugin, undefined, pluginUtils);
});
await pipe.run(mdast, vfile);
// Ensure there are keys on every node after post processing
keysTransform(mdast);
checkLinkTextTransform(mdast, externalReferences, vfile);
logMessagesFromVFile(session, vfile);
log.debug(toc(`Transformed mdast cross references and links for "${file}" in %s`));
if (checkLinks)
await checkLinksTransform(session, file, mdast);
}
export async function finalizeMdast(session, mdast, frontmatter, file, { imageWriteFolder, useExistingImages, imageAltOutputFolder, imageExtensions, optimizeWebp, simplifyFigures, processThumbnail, maxSizeWebp, }) {
const vfile = new VFile(); // Collect errors on this file
vfile.path = file;
if (simplifyFigures) {
// Transform output nodes to images / text
reduceOutputs(session, mdast, file, imageWriteFolder, {
altOutputFolder: simplifyFigures ? undefined : imageAltOutputFolder,
});
}
transformOutputsToFile(session, mdast, imageWriteFolder, {
altOutputFolder: simplifyFigures ? undefined : imageAltOutputFolder,
vfile,
});
if (!useExistingImages) {
await transformWidgetStaticAssetsToDisk(session, mdast, file, imageWriteFolder, simplifyFigures ? imageWriteFolder : (imageAltOutputFolder ?? imageWriteFolder));
await transformImagesToDisk(session, mdast, file, imageWriteFolder, {
altOutputFolder: imageAltOutputFolder,
imageExtensions,
});
// Must happen after transformImages
await transformImageFormats(session, mdast, file, imageWriteFolder, {
altOutputFolder: imageAltOutputFolder,
imageExtensions,
});
if (optimizeWebp) {
await transformWebp(session, { file, imageWriteFolder, maxSizeWebp });
}
if (processThumbnail) {
// Note, the thumbnail transform must be **after** images, as it may read the images
await transformThumbnail(session, mdast, file, frontmatter, imageWriteFolder, {
altOutputFolder: imageAltOutputFolder,
webp: optimizeWebp,
maxSizeWebp,
});
await transformBanner(session, file, frontmatter, imageWriteFolder, {
altOutputFolder: imageAltOutputFolder,
webp: optimizeWebp,
maxSizeWebp,
});
}
}
await transformDeleteBase64UrlSource(mdast);
if (simplifyFigures) {
// This must happen after embedded content is resolved so all children are present on figures
transformPlaceholderImages(mdast, { imageExtensions });
}
const cache = castSession(session);
const postData = cache.$getMdast(file)?.post;
if (postData) {
postData.frontmatter = frontmatter;
postData.mdast = mdast;
// TODO out-of-band widgets?
postData.widgets = cache.$getMdast(file)?.pre.widgets;
updateFileInfoFromFrontmatter(session, file, frontmatter);
}
logMessagesFromVFile(session, vfile);
}