myst-cli
Version:
Command line tools for MyST
146 lines (145 loc) • 6.18 kB
JavaScript
import { NotebookCell, RuleId, fileWarn } from 'myst-common';
import { selectAll } from 'unist-util-select';
import { nanoid } from 'nanoid';
import { CELL_TYPES, ensureString } from 'nbtx';
import { VFile } from 'vfile';
import { logMessagesFromVFile } from '../utils/logging.js';
import { BASE64_HEADER_SPLIT } from '../transforms/images.js';
import { parseMyst } from './myst.js';
import { findExpression, metadataSection } from '../transforms/inlineExpressions.js';
import { frontmatterValidationOpts } from '../frontmatter.js';
import { filterKeys } from 'simple-validators';
import { validatePageFrontmatter, PAGE_FRONTMATTER_KEYS, FRONTMATTER_ALIASES, } from 'myst-frontmatter';
function blockParent(cell, children) {
const kind = cell.cell_type === CELL_TYPES.code ? NotebookCell.code : NotebookCell.content;
return { type: 'block', kind, data: JSON.parse(JSON.stringify(cell.metadata)), children };
}
/**
* mdast transform to move base64 cell attachments directly to image nodes
*
* The image transform subsequently handles writing this in-line base64 to file.
*/
function replaceAttachmentsTransform(session, mdast, attachments, file) {
const vfile = new VFile();
vfile.path = file;
const imageNodes = selectAll('image', mdast);
imageNodes.forEach((image) => {
if (!image.url)
return;
const attachmentKey = image.url.match(/^attachment:(.*)$/)?.[1];
if (!attachmentKey)
return;
try {
const attachment = Object.entries(attachments[attachmentKey])[0];
const mimeType = attachment[0];
const attachmentVal = ensureString(attachment[1]);
if (!attachmentVal) {
fileWarn(vfile, `Unrecognized attachment name in ${file}: ${attachmentKey}`, {
ruleId: RuleId.notebookAttachmentsResolve,
});
}
else if (attachmentVal.includes(BASE64_HEADER_SPLIT)) {
image.url = attachmentVal;
}
else {
image.url = `data:${mimeType}${BASE64_HEADER_SPLIT}${attachmentVal}`;
}
}
catch {
fileWarn(vfile, `Unable to resolve attachment in ${file}: ${attachmentKey}`, {
ruleId: RuleId.notebookAttachmentsResolve,
});
}
});
logMessagesFromVFile(session, vfile);
}
export async function processNotebook(session, file, content) {
const { mdast } = await processNotebookFull(session, file, content);
return mdast;
}
/**
* Embed the Jupyter output data for a user expression into the AST
*/
function embedInlineExpressions(userExpressions, block) {
const inlineNodes = selectAll('inlineExpression', block);
inlineNodes.forEach((inlineExpression) => {
const data = findExpression(userExpressions, inlineExpression.value);
if (!data)
return;
inlineExpression.result = data.result;
});
}
export async function processNotebookFull(session, file, content) {
const { log } = session;
const { metadata, cells } = JSON.parse(content);
// notebook will be empty, use generateNotebookChildren, generateNotebookOrder here if we want to populate those
const language = metadata?.kernelspec?.language ?? 'python';
log.debug(`Processing Notebook: "${file}"`);
// Load frontmatter from notebook metadata
const vfile = new VFile();
vfile.path = file;
// Pull out only the keys we care about
const filteredMetadata = filterKeys(metadata ?? {}, [
...PAGE_FRONTMATTER_KEYS,
// Include aliased entries for page frontmatter keys
...Object.entries(FRONTMATTER_ALIASES)
.filter(([_, value]) => PAGE_FRONTMATTER_KEYS.includes(value))
.map(([key, _]) => key),
]);
const frontmatter = validatePageFrontmatter(filteredMetadata ?? {}, frontmatterValidationOpts(vfile));
// Load widgets from notebook metadata
// TODO validation / sanitation
const widgets = (metadata?.widgets ?? {});
let end = cells.length;
if (cells && cells.length > 1 && cells?.[cells.length - 1].source.length === 0) {
end = -1;
}
const items = await cells?.slice(0, end).reduce(async (P, cell, index) => {
const acc = await P;
if (cell.cell_type === CELL_TYPES.markdown) {
const cellContent = ensureString(cell.source);
// If the first cell is a frontmatter block, do not put a block break above it
const omitBlockDivider = index === 0 && cellContent.startsWith('---\n');
const cellMdast = parseMyst(session, cellContent, file, { ignoreFrontmatter: index > 0 });
if (cell.attachments) {
replaceAttachmentsTransform(session, cellMdast, cell.attachments, file);
}
if (omitBlockDivider) {
return acc.concat(...cellMdast.children);
}
const block = blockParent(cell, cellMdast.children);
embedInlineExpressions(block.data?.[metadataSection], block);
return acc.concat(block);
}
if (cell.cell_type === CELL_TYPES.raw) {
const raw = {
type: 'code',
lang: '',
value: ensureString(cell.source),
};
return acc.concat(blockParent(cell, [raw]));
}
if (cell.cell_type === CELL_TYPES.code) {
const code = {
type: 'code',
lang: language,
executable: true,
value: ensureString(cell.source),
};
const outputs = {
type: 'outputs',
id: nanoid(),
children: cell.outputs.map((output) => ({
type: 'output',
jupyter_data: output,
children: [],
})),
};
return acc.concat(blockParent(cell, [code, outputs]));
}
return acc;
}, Promise.resolve([]));
logMessagesFromVFile(session, vfile);
const mdast = { type: 'root', children: items };
return { mdast, frontmatter, widgets };
}