myst-cli
Version:
Command line tools for MyST
145 lines (144 loc) • 6.71 kB
JavaScript
import { NotebookCell, RuleId, fileWarn } from 'myst-common';
import { selectAll } from 'unist-util-select';
import { nanoid } from 'nanoid';
import { CELL_TYPES, ensureString } from 'nbtx';
import { VFile } from 'vfile';
import { logMessagesFromVFile } from '../utils/logging.js';
import { BASE64_HEADER_SPLIT } from '../transforms/images.js';
import { parseMyst } from './myst.js';
import { findExpression, metadataSection } from '../transforms/inlineExpressions.js';
import { frontmatterValidationOpts } from '../frontmatter.js';
import { filterKeys } from 'simple-validators';
import { validatePageFrontmatter, PAGE_FRONTMATTER_KEYS, FRONTMATTER_ALIASES, } from 'myst-frontmatter';
function blockParent(cell, children) {
const kind = cell.cell_type === CELL_TYPES.code ? NotebookCell.code : NotebookCell.content;
return { type: 'block', kind, data: JSON.parse(JSON.stringify(cell.metadata)), children };
}
/**
* mdast transform to move base64 cell attachments directly to image nodes
*
* The image transform subsequently handles writing this in-line base64 to file.
*/
function replaceAttachmentsTransform(session, mdast, attachments, file) {
const vfile = new VFile();
vfile.path = file;
const imageNodes = selectAll('image', mdast);
imageNodes.forEach((image) => {
var _a;
if (!image.url)
return;
const attachmentKey = (_a = image.url.match(/^attachment:(.*)$/)) === null || _a === void 0 ? void 0 : _a[1];
if (!attachmentKey)
return;
try {
const attachment = Object.entries(attachments[attachmentKey])[0];
const mimeType = attachment[0];
const attachmentVal = ensureString(attachment[1]);
if (!attachmentVal) {
fileWarn(vfile, `Unrecognized attachment name in ${file}: ${attachmentKey}`, {
ruleId: RuleId.notebookAttachmentsResolve,
});
}
else if (attachmentVal.includes(BASE64_HEADER_SPLIT)) {
image.url = attachmentVal;
}
else {
image.url = `data:${mimeType}${BASE64_HEADER_SPLIT}${attachmentVal}`;
}
}
catch {
fileWarn(vfile, `Unable to resolve attachment in ${file}: ${attachmentKey}`, {
ruleId: RuleId.notebookAttachmentsResolve,
});
}
});
logMessagesFromVFile(session, vfile);
}
export async function processNotebook(session, file, content) {
const { mdast } = await processNotebookFull(session, file, content);
return mdast;
}
export async function processNotebookFull(session, file, content) {
var _a, _b, _c;
const { log } = session;
const { metadata, cells } = JSON.parse(content);
// notebook will be empty, use generateNotebookChildren, generateNotebookOrder here if we want to populate those
const language = (_b = (_a = metadata === null || metadata === void 0 ? void 0 : metadata.kernelspec) === null || _a === void 0 ? void 0 : _a.language) !== null && _b !== void 0 ? _b : 'python';
log.debug(`Processing Notebook: "${file}"`);
// Load frontmatter from notebook metadata
const vfile = new VFile();
vfile.path = file;
// Pull out only the keys we care about
const filteredMetadata = filterKeys(metadata !== null && metadata !== void 0 ? metadata : {}, [
...PAGE_FRONTMATTER_KEYS,
// Include aliased entries for page frontmatter keys
...Object.entries(FRONTMATTER_ALIASES)
.filter(([_, value]) => PAGE_FRONTMATTER_KEYS.includes(value))
.map(([key, _]) => key),
]);
const frontmatter = validatePageFrontmatter(filteredMetadata !== null && filteredMetadata !== void 0 ? filteredMetadata : {}, frontmatterValidationOpts(vfile));
// Load widgets from notebook metadata
// TODO validation / sanitation
const widgets = ((_c = metadata === null || metadata === void 0 ? void 0 : metadata.widgets) !== null && _c !== void 0 ? _c : {});
let end = cells.length;
if (cells && cells.length > 1 && (cells === null || cells === void 0 ? void 0 : cells[cells.length - 1].source.length) === 0) {
end = -1;
}
const items = await (cells === null || cells === void 0 ? void 0 : cells.slice(0, end).reduce(async (P, cell, index) => {
var _a;
const acc = await P;
if (cell.cell_type === CELL_TYPES.markdown) {
const cellContent = ensureString(cell.source);
// If the first cell is a frontmatter block, do not put a block break above it
const omitBlockDivider = index === 0 && cellContent.startsWith('---\n');
const cellMdast = parseMyst(session, cellContent, file, { ignoreFrontmatter: index > 0 });
if (cell.attachments) {
replaceAttachmentsTransform(session, cellMdast, cell.attachments, file);
}
if (omitBlockDivider) {
return acc.concat(...cellMdast.children);
}
const block = blockParent(cell, cellMdast.children);
// Embed expression results into expression
const userExpressions = (_a = block.data) === null || _a === void 0 ? void 0 : _a[metadataSection];
const inlineNodes = selectAll('inlineExpression', block);
inlineNodes.forEach((inlineExpression) => {
const data = findExpression(userExpressions, inlineExpression.value);
if (!data)
return;
inlineExpression.result = data.result;
});
return acc.concat(block);
}
if (cell.cell_type === CELL_TYPES.raw) {
const raw = {
type: 'code',
lang: '',
value: ensureString(cell.source),
};
return acc.concat(blockParent(cell, [raw]));
}
if (cell.cell_type === CELL_TYPES.code) {
const code = {
type: 'code',
lang: language,
executable: true,
value: ensureString(cell.source),
};
// Embed outputs in an output block
const output = {
type: 'output',
id: nanoid(),
data: [],
};
if (cell.outputs && cell.outputs.length > 0) {
output.data = cell.outputs;
}
return acc.concat(blockParent(cell, [code, output]));
}
return acc;
}, Promise.resolve([])));
logMessagesFromVFile(session, vfile);
const mdast = { type: 'root', children: items };
return { mdast, frontmatter, widgets };
}