myst-cli
Version:
Command line tools for MyST
271 lines (270 loc) • 11.6 kB
JavaScript
import fs from 'node:fs';
import { dirname, join, relative } from 'node:path';
import { computeHash } from 'myst-cli-utils';
import { liftChildren, fileError, RuleId, fileWarn } from 'myst-common';
import { htmlTransform } from 'myst-transforms';
import stripAnsi from 'strip-ansi';
import { remove } from 'unist-util-remove';
import { selectAll } from 'unist-util-select';
import { ensureString, extFromMimeType, minifyCellOutput, walkOutputs } from 'nbtx';
import { castSession } from '../session/cache.js';
import { resolveOutputPath } from './images.js';
function getFilename(hash, contentType) {
return `${hash}${extFromMimeType(contentType)}`;
}
function getWriteDestination(hash, contentType, writeFolder) {
return join(writeFolder, getFilename(hash, contentType));
}
/**
* Traverse all output nodes, minify their content, and cache on the session
*/
export async function transformOutputsToCache(session, mdast, kind, opts) {
const outputs = selectAll('output', mdast);
if (!outputs.length)
return;
const cache = castSession(session);
await Promise.all(outputs
.filter((output) => output.visibility !== 'remove')
.map(async (output) => {
output.data = await minifyCellOutput(output.data, cache.$outputs, {
computeHash,
maxCharacters: opts === null || opts === void 0 ? void 0 : opts.minifyMaxCharacters,
});
}));
}
export function stringIsMatplotlibOutput(value) {
if (!value)
return false;
// We can add more when we find them...
const match = value.match(/<(Figure|Text|matplotlib)(.*)at ([0-9a-z]+)>/) ||
value.match(/^<(Figure|Text|AxesSubplot|module)(.*)>$/) ||
value.match(/^(Text)\((.*)\)$/);
return !!match;
}
/** Remove warnings and errors from outputs */
export function transformFilterOutputStreams(mdast, vfile, { output_stdout: stdout = 'show', output_stderr: stderr = 'show', output_matplotlib_strings: mpl = 'remove-warn', } = {}) {
const blocks = selectAll('block', mdast);
blocks.forEach((block) => {
var _a;
const tags = Array.isArray((_a = block.data) === null || _a === void 0 ? void 0 : _a.tags) ? block.data.tags : [];
const blockRemoveStderr = tags.includes('remove-stderr');
const blockRemoveStdout = tags.includes('remove-stdout');
const outputs = selectAll('output', block);
// There should be only one output in the block
outputs.forEach((output) => {
output.data = output.data.filter((data) => {
if ((stderr !== 'show' || blockRemoveStderr) &&
data.output_type === 'stream' &&
data.name === 'stderr') {
const doRemove = stderr.includes('remove') || blockRemoveStderr;
const doWarn = stderr.includes('warn');
const doError = stderr.includes('error');
if (doWarn || doError) {
(doError ? fileError : fileWarn)(vfile, doRemove ? 'Removing stderr from outputs' : 'Output contains stderr', {
node: output,
note: ensureString(data.text),
});
}
return !doRemove;
}
if ((stdout !== 'show' || blockRemoveStdout) &&
data.output_type === 'stream' &&
data.name === 'stdout') {
const doRemove = stdout.includes('remove') || blockRemoveStdout;
const doWarn = stdout.includes('warn');
const doError = stdout.includes('error');
if (doWarn || doError) {
(doError ? fileError : fileWarn)(vfile, doRemove ? 'Removing stdout from outputs' : 'Output contains stdout', {
node: output,
note: ensureString(data.text),
});
}
return !doRemove;
}
if (mpl !== 'show' &&
data.output_type === 'execute_result' &&
Object.keys(data.data).length === 1 &&
data.data['text/plain']) {
const content = data.data['text/plain'].content;
if (!stringIsMatplotlibOutput(content))
return true;
const doRemove = mpl.includes('remove');
const doWarn = mpl.includes('warn');
const doError = mpl.includes('error');
if (doWarn || doError) {
(doError ? fileError : fileWarn)(vfile, doRemove
? 'Removing matplotlib string from outputs'
: 'Output contains matplotlib string', {
node: output,
note: `${content}\n Fix: Put a semicolon on the last line of your cell or add "output_matplotlib_strings: remove" to the project settings.`,
});
}
return !doRemove;
}
return true;
});
});
});
}
function writeCachedOutputToFile(session, hash, cachedOutput, writeFolder, opts) {
const [content, { contentType, encoding }] = cachedOutput;
const filename = getFilename(hash, contentType);
const destination = getWriteDestination(hash, contentType, writeFolder);
if (fs.existsSync(destination)) {
session.log.debug(`Cached file found for notebook output: ${destination}`);
}
else {
try {
if (!fs.existsSync(writeFolder))
fs.mkdirSync(writeFolder, { recursive: true });
fs.writeFileSync(destination, content, { encoding: encoding });
session.log.debug(`Notebook output successfully written: ${destination}`);
}
catch {
if (opts === null || opts === void 0 ? void 0 : opts.vfile) {
fileError(opts.vfile, `Error writing notebook output: ${destination}`, {
node: opts.node,
ruleId: RuleId.notebookOutputCopied,
});
}
return undefined;
}
}
return resolveOutputPath(filename, writeFolder, opts === null || opts === void 0 ? void 0 : opts.altOutputFolder);
}
/**
* Write cached content from output nodes to file
*/
export function transformOutputsToFile(session, mdast, writeFolder, opts) {
const outputs = selectAll('output', mdast);
const cache = castSession(session);
outputs.forEach((node) => {
walkOutputs(node.data, (obj) => {
const { hash } = obj;
if (!hash || !cache.$outputs[hash])
return undefined;
obj.path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, {
...opts,
node,
});
});
});
}
/**
* Return if new type is preferred output content_type over existing type
*
* Since this is for static output, images are top preference, then
* html, then text.
*
* If the new and existing types are the same, always just keep existing.
*/
function isPreferredOutputType(newType, existingType) {
if (existingType.startsWith('image/'))
return false;
if (newType.startsWith('image'))
return true;
if (existingType === 'text/html')
return false;
if (newType === 'text/html')
return true;
return false;
}
/**
* Convert output nodes with minified content to image or code
*
* This writes outputs of type image to file, modifies outputs of type
* text to a code node, and removes other output types.
*/
export function reduceOutputs(session, mdast, file, writeFolder, opts) {
const outputs = selectAll('output', mdast);
const cache = castSession(session);
outputs.forEach((node) => {
var _a, _b, _c;
if (node.visibility === 'remove' || node.visibility === 'hide') {
// Hidden nodes should not show up in simplified outputs for static export
node.type = '__delete__';
return;
}
if (!((_a = node.data) === null || _a === void 0 ? void 0 : _a.length) && !((_b = node.children) === null || _b === void 0 ? void 0 : _b.length)) {
node.type = '__delete__';
return;
}
node.type = '__lift__';
if ((_c = node.children) === null || _c === void 0 ? void 0 : _c.length)
return;
const selectedOutputs = [];
node.data.forEach((output) => {
let selectedOutput;
walkOutputs([output], (obj) => {
const { output_type, content_type, hash } = obj;
if (!hash)
return undefined;
if (!selectedOutput || isPreferredOutputType(content_type, selectedOutput.content_type)) {
if (['error', 'stream'].includes(output_type)) {
selectedOutput = { content_type: 'text/plain', hash };
}
else if (typeof content_type === 'string') {
if (content_type.startsWith('image/') ||
content_type === 'text/plain' ||
content_type === 'text/html') {
selectedOutput = { content_type, hash };
}
}
}
});
if (selectedOutput)
selectedOutputs.push(selectedOutput);
});
const children = selectedOutputs
.map((output) => {
const { content_type, hash } = output !== null && output !== void 0 ? output : {};
if (!hash || !cache.$outputs[hash])
return undefined;
if (content_type === 'text/html') {
const htmlTree = {
type: 'root',
children: [
{
type: 'html',
value: cache.$outputs[hash][0],
},
],
};
htmlTransform(htmlTree);
if (selectAll('image', htmlTree).find((htmlImage) => !htmlImage.url)) {
return undefined;
}
return htmlTree.children;
}
else if (content_type.startsWith('image/')) {
const path = writeCachedOutputToFile(session, hash, cache.$outputs[hash], writeFolder, {
...opts,
node,
});
if (!path)
return undefined;
const relativePath = relative(dirname(file), path);
return {
type: 'image',
data: { type: 'output' },
url: relativePath,
urlSource: relativePath,
};
}
else if (content_type === 'text/plain' && cache.$outputs[hash]) {
const [content] = cache.$outputs[hash];
return {
type: 'code',
data: { type: 'output' },
value: stripAnsi(content),
};
}
return undefined;
})
.flat()
.filter((output) => !!output);
node.children = children;
});
remove(mdast, '__delete__');
liftChildren(mdast, '__lift__');
}