@torchlight-api/torchlight-cli
Version:
A CLI for Torchlight - the syntax highlighting API
731 lines (589 loc) • 20.8 kB
JavaScript
;
var commander = require('commander');
var axios = require('axios');
var md5 = require('md5');
var get = require('lodash.get');
var chunk = require('lodash.chunk');
var chalk = require('chalk');
var path = require('path');
var cheerio = require('cheerio');
var chokidar = require('chokidar');
var fs = require('fs-extra');
var events = require('events');
var inquirer = require('inquirer');
function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
var axios__default = /*#__PURE__*/_interopDefaultLegacy(axios);
var md5__default = /*#__PURE__*/_interopDefaultLegacy(md5);
var get__default = /*#__PURE__*/_interopDefaultLegacy(get);
var chunk__default = /*#__PURE__*/_interopDefaultLegacy(chunk);
var chalk__default = /*#__PURE__*/_interopDefaultLegacy(chalk);
var path__default = /*#__PURE__*/_interopDefaultLegacy(path);
var cheerio__default = /*#__PURE__*/_interopDefaultLegacy(cheerio);
var chokidar__default = /*#__PURE__*/_interopDefaultLegacy(chokidar);
var fs__default = /*#__PURE__*/_interopDefaultLegacy(fs);
var inquirer__default = /*#__PURE__*/_interopDefaultLegacy(inquirer);
let silent = false;
function log(fn, args) {
args = Array.from(args);
if (!silent) {
console.log(fn(args.shift()), ...args);
}
}
function error() {
log(chalk__default["default"].bold.bgRed, arguments);
}
function info() {
log(chalk__default["default"].green, arguments);
}
function silence(silence = true) {
silent = silence;
}
var log$1 = {
error,
info,
silence
};
let store = {};
function Memory() {//
}
/**
* Get an item from the cache.
*
* @param {string} key
* @param {*} def
* @return {*}
*/
Memory.prototype.get = function (key, def) {
if (!Object.prototype.hasOwnProperty.call(store, key)) {
return def;
}
const entry = store[key];
if (Date.now() / 1000 > entry.expires) {
this.delete(key);
return def;
}
return entry.value;
};
/**
* Set an item in the cache.
*
* @param {string} key
* @param {*} value
* @param {number} ttlSeconds
*/
Memory.prototype.set = function (key, value, ttlSeconds = 60 * 24 * 7) {
store[key] = {
expires: Date.now() / 1000 + ttlSeconds,
value: value
};
};
/**
* Remove a key from the cache.
*
* @param key
*/
Memory.prototype.delete = function (key) {
delete store[key];
};
/**
* Clear the cache.
*/
Memory.prototype.clear = function () {
store = {};
};
/**
* @constructor
*/
const Torchlight = function () {
this.initialized = false;
this.chunkSize = 30;
this.configuration = {};
};
/**
* @param config
* @return {Torchlight}
*/
Torchlight.prototype.init = function (config, cache) {
var _process, _process$env, _config;
if (this.initialized) {
return this;
}
config = config || {};
if ((_process = process) !== null && _process !== void 0 && (_process$env = _process.env) !== null && _process$env !== void 0 && _process$env.TORCHLIGHT_TOKEN && !((_config = config) !== null && _config !== void 0 && _config.token)) {
config.token = process.env.TORCHLIGHT_TOKEN;
}
this.initialized = true;
this.configuration = config;
this.cache = cache || new Memory();
return this;
};
/**
* Get a value out of the configuration.
*
* @param {string} key
* @param {*} def
* @return {*}
*/
Torchlight.prototype.config = function (key, def = undefined) {
return get__default["default"](this.configuration, key, def);
};
/**
* Hash of the Torchlight configuration.
*
* @return {string}
*/
Torchlight.prototype.configHash = function () {
return md5__default["default"](this.configuration);
};
/**
* @param blocks
* @return {Promise<*>}
*/
Torchlight.prototype.highlight = function (blocks) {
// Set the data from cache if it's there.
blocks.map(block => block.setResponseData(this.cache.get(block.hash(), {}))); // Reject the blocks that have already been highlighted from the cache.
const needed = blocks.filter(block => !block.highlighted); // Only send the un-highlighted blocks to the API.
return this.request(needed).then(highlighted => {
needed.forEach(block => {
// Look through the response and match em up by ID.
const found = highlighted.find(b => block.id === b.id);
if (!found || !found.highlighted) {
return;
} // Store it in the cache for later.
this.cache.set(block.hash(), {
highlighted: found.highlighted,
classes: found.classes,
styles: found.styles
}); // Set the info on the block.
block.setResponseData(found);
}); // Look for the blocks that weren't highlighted and add a default.
blocks.filter(block => !block.highlighted).forEach(block => {
log$1.error(`A block failed to highlight. The code was: \`${block.code.substring(0, 20)} [...]\``); // Add the `line` divs so everyone's CSS will work even on default blocks.
block.highlighted = block.code.split('\n').map(line => `<div class="line">${htmlEntities(line)}</div>`).join('');
block.classes = 'torchlight';
});
return blocks;
});
};
/**
* @param blocks
* @return {Promise<*[]>}
*/
Torchlight.prototype.request = function (blocks) {
if (!blocks.length) {
return Promise.resolve([]);
}
const token = this.config('token'); // For huge sites, we need to send blocks in chunks so
// that we don't send e.g. 500 blocks in one request.
if (blocks.length > this.chunkSize) {
return this.fan(blocks);
}
const host = this.config('host', 'https://api.torchlight.dev');
return axios__default["default"].post(`${host}/highlight`, {
blocks: blocks.map(block => block.toRequestParams()),
options: this.config('options', {})
}, {
headers: {
Authorization: `Bearer ${token}`,
'X-Torchlight-Client': 'Torchlight CLI'
}
}).then(response => response.data.blocks);
};
Torchlight.prototype.fan = function (blocks) {
const highlighted = [];
const errors = [];
const requests = chunk__default["default"](blocks, this.chunkSize).map(chunk => this.request(chunk)); // Let all of the promises settle, even if some of them fail.
return Promise.allSettled(requests).then(responses => {
responses.forEach(response => {
// For a successful request, add the blocks to the array.
if (response.status === 'fulfilled') {
highlighted.push(...response.value);
} // For an error, stash it as well.
if (response.status === 'rejected') {
errors.push(response.reason);
}
}); // We got some blocks...
if (highlighted.length) {
// ...and some errors. In this case we just log the
// error and go ahead and use the blocks.
if (errors.length) {
log$1.error(`${errors.length} fanned request(s) failed, but others succeeded. Error: ${errors[0]}.`);
}
return highlighted;
} // Errors only, throw a proper error.
if (errors.length) {
throw new Error(errors[0]);
}
return [];
});
};
function htmlEntities(str) {
return String(str).replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
}
var torchlight = new Torchlight();
function guid() {
const S4 = () => Math.floor((1 + Math.random()) * 0x10000).toString(16).substring(1);
return `${S4()}${S4()}-${S4()}-${S4()}-${S4()}-${S4()}${S4()}${S4()}`.toLowerCase();
}
function Block(opts = {}) {
opts = {
id: guid(),
theme: torchlight.config('theme', 'nord'),
...opts
};
this.id = opts.id;
this.code = opts.code;
this.language = opts.language;
this.theme = opts.theme;
this.highlighted = null;
this.classes = null;
this.styles = null;
}
Block.prototype.hash = function () {
return md5__default["default"]('' + this.language + this.theme + this.code + torchlight.config('bust', 0) + JSON.stringify(torchlight.config('options')));
};
Block.prototype.code = function (code) {
this.code = code;
return this;
};
Block.prototype.language = function (language) {
this.language = language;
return this;
};
Block.prototype.theme = function (theme) {
this.theme = theme;
return this;
};
Block.prototype.placeholder = function (extra = '') {
if (extra) {
extra = `-${extra}`;
}
return `__torchlight-block-[${this.id}]${extra}__`;
};
Block.prototype.setResponseData = function (data) {
if (data) {
this.highlighted = data.highlighted;
this.classes = data.classes;
this.styles = data.styles;
}
return this;
};
Block.prototype.toRequestParams = function () {
return {
id: this.id,
hash: this.hash,
language: this.language,
theme: this.theme,
code: this.code
};
};
const bus = new events.EventEmitter();
const FILE_WATCHING_COMPLETE = 1;
function highlight (torchlight, options) {
options = {
input: torchlight.config('highlight.input', ''),
output: torchlight.config('highlight.output', ''),
include: torchlight.config('highlight.includeGlobs', ['**/*.htm', '**/*.html']),
exclude: torchlight.config('highlight.excludePatterns', ['/node_modules/', '/vendor/']),
watch: false,
...options
};
if (options.watch) {
log$1.info(`
***************************************
* Torchlight is watching files... *
***************************************
`);
}
const input = path__default["default"].resolve(options.input);
const output = path__default["default"].resolve(options.output || options.input);
const watcher = chokidar__default["default"].watch(normalizeStringedArray(options.include), {
cwd: input,
ignored: path => normalizeStringedArray(options.exclude).some(s => path.includes(s)),
ignoreInitial: false
});
watcher.on('all', (event, file) => {
if (event !== 'add' && event !== 'change') {
return;
}
log$1.info('Highlighting %s', file);
const source = fs__default["default"].readFileSync(path__default["default"].join(input, file), 'utf-8');
highlight$1(torchlight, source).then(highlighted => {
const destination = path__default["default"].join(output, file);
fs__default["default"].ensureFileSync(destination);
if (highlighted === fs__default["default"].readFileSync(destination, 'utf-8')) {
return;
}
log$1.info('Writing to %s', destination);
fs__default["default"].writeFileSync(destination, highlighted, 'utf-8');
});
});
watcher.on('ready', function () {
if (!options.watch) {
watcher.close().then(() => bus.emit(FILE_WATCHING_COMPLETE));
}
});
}
function normalizeStringedArray(value) {
return (typeof value === 'string' ? value.split(',') : value).filter(x => x);
}
function highlight$1(torchlight, source) {
let highlighted = source;
const $ = cheerio__default["default"].load(source, {
sourceCodeLocationInfo: true
}, false);
const blocks = []; // Search for blocks that have not already been processed.
$('pre:not([data-torchlight-processed])').each((index, pre) => {
const $pre = $(pre);
$pre.children('code').each((index, code) => {
const $code = $(code);
const block = new Block({
// Using `text()` will re-encode entities like &lgt;
code: $code.text(),
language: decipherLanguage($pre, $code)
}); // Add our class placeholder as a class, so that we don't overwrite
// any classes that are already there.
$pre.addClass(block.placeholder('class')); // Add a fake style that we can replace later.
$pre.css(block.placeholder('style'), '0'); // Store the raw code as the developer wrote it, so we can re-highlight
// it later if we need to, or allow it to be copied to clipboard.
const raw = `<textarea data-torchlight-original='true' style='display: none !important;'>${$code.html()}</textarea>`; // Add the placeholder inside the code tag.
$code.html(block.placeholder('highlighted') + raw); // Give the developer an opportunity to add things to the placeholder
// element. Like copy to clipboard buttons, language indicators, etc.
if (torchlight.config('modifyPlaceholderElement')) {
torchlight.config('modifyPlaceholderElement')($, $pre, $code, block);
}
blocks.push(block);
}); // Add the options hash that this block will be highlighted with.
$pre.attr('data-torchlight-processed', torchlight.configHash()); // Cut out the *exact* pre element as it is in the file. Cheerio converts
// single quotes to double, normalizes whitespace, and otherwise "cleans
// up" the parsed document, so we can't simply modify the Cheerio dom and
// write it back to disk. Instead we're going to surgically cut out the
// pre tag and all its contents without touching anything else around it.
const pristinePreElement = source.substring(pre.sourceCodeLocation.startOffset, pre.sourceCodeLocation.endOffset); // Swap out the original tag with the outerHTML of our modified tag.
highlighted = highlighted.replace(pristinePreElement, $.html($pre));
});
if (!blocks.length) {
return Promise.resolve(source);
}
return torchlight.highlight(blocks).then(() => {
blocks.forEach(block => {
var _block$classes, _block$styles;
const swap = {
[block.placeholder('class')]: (_block$classes = block.classes) !== null && _block$classes !== void 0 ? _block$classes : '',
[block.placeholder('style') + ': 0;']: (_block$styles = block.styles) !== null && _block$styles !== void 0 ? _block$styles : '',
[block.placeholder('highlighted')]: block.highlighted
};
Object.keys(swap).forEach(key => {
highlighted = highlighted.replace(key, () => swap[key]);
});
});
return highlighted;
});
}
/**
* Given a <pre> element, figure out what language it is.
*
* @param $pre
* @return {string}
*/
function decipherLanguage($pre, $code) {
const custom = torchlight.config('highlight.decipherLanguageFromElement'); // Let the developer add their own deciphering mechanism.
if (custom) {
const lang = custom($pre);
if (lang) {
return lang;
}
}
const langs = [// Look first at the code element.
...decipherFromElement($code), // And then the pre element.
...decipherFromElement($pre)];
return langs.length ? langs[0] : 'text';
}
/**
* Given any element, figure out what language it might be.
*
* @param $el
* @return {*[]}
*/
function decipherFromElement($el) {
var _$el$data, _$el$data2;
if (!$el) {
return [];
}
const classes = ($el.attr('class') || '').split(' ') // These classes are commonly used to denote code languages.
.filter(c => c.startsWith('language-') || c.startsWith('lang-')).map(c => c.replace('language-', '').replace('lang-', ''));
return [// Data attributes get highest priority.
(_$el$data = $el.data()) === null || _$el$data === void 0 ? void 0 : _$el$data.language, (_$el$data2 = $el.data()) === null || _$el$data2 === void 0 ? void 0 : _$el$data2.lang, ...classes].filter(l => l);
}
function write(location) {
const source = path__default["default"].resolve(path__default["default"].join(__dirname, '../stubs/config.js'));
const stub = fs__default["default"].readFileSync(source, 'utf-8');
fs__default["default"].ensureFileSync(location);
fs__default["default"].writeFileSync(location, stub);
log$1.info('File written to %s', location);
}
function init (torchlight, options) {
options = {
path: 'torchlight.config.js',
...options
};
const location = path__default["default"].resolve(options.path);
if (!fs__default["default"].existsSync(location)) {
return write(location);
}
const questions = [{
type: 'confirm',
name: 'overwrite',
message: `Overwrite file at ${location}?`,
default: false
}];
inquirer__default["default"].prompt(questions).then(answers => {
if (answers.overwrite) {
write(location);
}
});
}
function cacheClear (torchlight, options) {
torchlight.cache.clear();
}
/**
* @param options
* @constructor
*/
function File(options = {}) {
if (!options.directory) {
throw new Error('No cache directory specified.');
}
this.directory = path__default["default"].resolve(options.directory);
fs__default["default"].ensureDirSync(this.directory);
}
/**
* Get an item from the cache.
*
* @param {string} key
* @param {*} def
* @return {*}
*/
File.prototype.get = function (key, def) {
if (!fs__default["default"].pathExistsSync(this.filename(key))) {
return def;
}
const entry = fs__default["default"].readJsonSync(this.filename(key));
if (Date.now() / 1000 > entry.expires) {
this.delete(key);
return def;
}
return entry.value;
};
/**
* Set an item in the cache.
*
* @param {string} key
* @param {*} value
* @param {number} ttlSeconds
*/
File.prototype.set = function (key, value, ttlSeconds = 60 * 24 * 7) {
fs__default["default"].writeJsonSync(this.filename(key), {
expires: Date.now() / 1000 + ttlSeconds,
value: value
});
};
/**
* Remove a key from the cache.
*
* @param key
*/
File.prototype.delete = function (key) {
fs__default["default"].removeSync(this.filename(key));
};
/**
* Clear the cache.
*/
File.prototype.clear = function () {
fs__default["default"].removeSync(this.directory);
};
/**
* @param {string} key
* @return {string}
*/
File.prototype.filename = function (key) {
return path__default["default"].join(this.directory, md5__default["default"](key) + '.json');
};
/**
* @param {string|object} config
* @return {*}
*/
function makeConfig(config) {
// By convention, look in the root directory for
// a torchlight.config.js file.
if (config === undefined || config === '') {
config = 'torchlight.config.js';
}
if (typeof config === 'string') {
config = fs__default["default"].pathExistsSync(path__default["default"].resolve(config)) ? require(path__default["default"].resolve(config)) : {};
}
return config || {};
}
/**
* Make a cache to hold highlighted blocks.
*
* @return {Cache}
*/
function makeCache(config) {
const cache = config === null || config === void 0 ? void 0 : config.cache; // Make a file cache if we're given a directory.
if (cache && typeof cache === 'string') {
return new File({
directory: cache
});
} // Use the cache they have provided, or default to an in-memory cache.
return cache || new Memory();
}
/**
* Configure the commander CLI application.
*
* @param options
* @return {Command}
*/
function makeProgram(options = {}) {
if (options !== null && options !== void 0 && options.testing) {
// Don't exit when there are errors, so we
// can catch them.
commander.program.exitOverride(); // Capture the output, so we can inspect it.
commander.program.configureOutput({
writeOut: () => {//
},
writeErr: () => {//
},
...((options === null || options === void 0 ? void 0 : options.configureOutput) || {})
});
} // Bootstrap the Torchlight singleton before every command.
commander.program.hook('preAction', thisCommand => {
const config = makeConfig(thisCommand.opts().config);
const cache = makeCache(config);
torchlight.init(config, cache);
});
makeCommand('_default_', highlight).description('Highlight code blocks in source files').option('-i, --input <directory>', 'Input directory. Defaults to current directory.').option('-o, --output <directory>', 'Output directory. Defaults to current directory.').option('-n, --include <patterns>', 'Glob patterns used to search for source files. Separate ' + 'multiple patterns with commas. Defaults to "**/*.htm,**/*.html".').option('-x, --exclude <patterns>', 'String patterns to ignore (not globs). Separate multiple ' + 'patterns with commas. Defaults to "/node_modules/,/vendor/".').option('-w, --watch', 'Watch source files for changes.');
makeCommand('init', init).description('Publish the Torchlight configuration file.').option('-p, --path <path>', 'Location for the configuration file.');
makeCommand('cache:clear', cacheClear).description('Clear the cache');
return commander.program;
}
/**
* @param name
* @return {Command}
*/
function makeCommand(name, handler) {
let cmd = commander.program;
if (name !== '_default_') {
// Name the other commands.
cmd = cmd.command(name);
} // Add a little shim around the handler so we can pass the
// torchlight variable in, just for convenience.
cmd.action(function (options) {
return handler(torchlight, options);
}); // Every command gets the -c option, so the developer can
// specify a path to a config file.
return cmd.option('-c, --config <file>', 'Path to the Torchlight configuration file.');
}
makeProgram().parse();
//# sourceMappingURL=torchlight.cjs.js.map