@blitzjs/file-pipeline
Version:
Display package for the Blitz CLI
1,000 lines (840 loc) • 26.2 kB
JavaScript
;
Object.defineProperty(exports, '__esModule', { value: true });
var path = require('path');
var display = require('@blitzjs/display');
var chalk = require('chalk');
var pipe = require('pump');
var through = require('through2');
require('parallel-transform');
var pumpify = require('pumpify');
var rimrafCallback = require('rimraf');
var fs = require('fs');
var micromatch = require('micromatch');
var chokidar = require('chokidar');
var mergeStream = require('merge-stream');
var File = require('vinyl');
var vinyl = require('vinyl-file');
var vfs = require('vinyl-fs');
var crypto = require('crypto');
var fsExtra = require('fs-extra');
var debounce = require('lodash/debounce');
var gulpIf = require('gulp-if');
function _interopDefault (e) { return e && e.__esModule ? e : { 'default': e }; }
var path__default = /*#__PURE__*/_interopDefault(path);
var chalk__default = /*#__PURE__*/_interopDefault(chalk);
var pipe__default = /*#__PURE__*/_interopDefault(pipe);
var through__default = /*#__PURE__*/_interopDefault(through);
var pumpify__default = /*#__PURE__*/_interopDefault(pumpify);
var rimrafCallback__default = /*#__PURE__*/_interopDefault(rimrafCallback);
var micromatch__default = /*#__PURE__*/_interopDefault(micromatch);
var chokidar__default = /*#__PURE__*/_interopDefault(chokidar);
var mergeStream__default = /*#__PURE__*/_interopDefault(mergeStream);
var File__default = /*#__PURE__*/_interopDefault(File);
var vinyl__default = /*#__PURE__*/_interopDefault(vinyl);
var vfs__default = /*#__PURE__*/_interopDefault(vfs);
var crypto__default = /*#__PURE__*/_interopDefault(crypto);
var debounce__default = /*#__PURE__*/_interopDefault(debounce);
var gulpIf__default = /*#__PURE__*/_interopDefault(gulpIf);
var IDLE = "IDLE";
var INIT = "INIT";
var FILE_WRITTEN = "FILE_WRITTEN";
var FILE_DELETED = "FILE_DELETED";
var ERROR_THROWN = "ERROR_THROWN";
var READY = "READY";
// The following are a loose collaction of stream
/*#__PURE__*/require("from2");
/*#__PURE__*/require("flush-write-stream");
// const pipeline = (pumpifyFn as any) as PumpifyFn & {obj: PumpifyFn}
var pipeline = /*#__PURE__*/pumpify__default['default'].ctor({
autoDestroy: false,
destroy: false,
objectMode: true,
highWaterMark: 10000
});
/**
* Display is a stream that converts build status events and prepares them for the console.
* A good way to think about this is as the root of the "view" component of the application.
*/
function createDisplay() {
var lastEvent = {
type: INIT,
payload: null
};
var spinner = display.log.spinner("Compiling").start();
var stream = through__default['default']({
objectMode: true
}, function (event, _, next) {
switch (event.type) {
case FILE_WRITTEN:
{
var filePath = event.payload.history[0].replace(process.cwd(), "");
spinner.text = filePath;
break;
}
case ERROR_THROWN:
{
// Tidy up if operational error is encountered
if (lastEvent.type === FILE_WRITTEN) {
spinner.fail("Uh oh something broke");
}
break;
}
case READY:
{
spinner.succeed(chalk__default['default'].green.bold("Compiled"));
break;
}
} // Capture last event incase we need to tidy up the console
lastEvent = event;
next();
});
return {
stream: stream
};
}
function rimraf(f, opts) {
if (opts === void 0) {
opts = {};
}
return new Promise(function (res, rej) {
rimrafCallback__default['default'](f, opts, function (error) {
if (error) {
rej(error);
} else {
res();
}
});
});
}
var watch = function watch(includePaths, options) {
function resolveFilepath(filepath) {
if (path.isAbsolute(filepath)) {
return path.normalize(filepath);
}
return path.resolve(options.cwd || process.cwd(), filepath);
}
var stream = through__default['default']({
objectMode: true
}, function (f, _, next) {
next(f);
});
function processEvent(evt) {
return async function (filepath, _stat) {
filepath = resolveFilepath(filepath);
var fileOpts = Object.assign({}, options, {
path: filepath
});
var file = evt === "unlink" || evt === "unlinkDir" ? new File__default['default'](fileOpts) : await vinyl__default['default'].read(filepath, fileOpts);
file.event = evt;
stream.push(file);
};
}
var fswatcher = chokidar__default['default'].watch(includePaths, options);
fswatcher.on("add", processEvent("add"));
fswatcher.on("change", processEvent("change"));
fswatcher.on("unlink", processEvent("unlink"));
fswatcher.on("error", function (error) {
return console.log("Watcher error: " + error);
});
return {
stream: stream,
fswatcher: fswatcher
};
};
function getWatcher(watching, cwd, include, ignore) {
if (watching) {
return watch(include, {
cwd: cwd,
ignored: ignore,
persistent: true,
ignoreInitial: true,
alwaysStat: true,
awaitWriteFinish: {
stabilityThreshold: 200,
pollInterval: 50
}
});
}
return {
stream: through__default['default'].obj(),
fswatcher: {
close: function close() {}
}
};
}
/**
* A stage that will provide agnostic file input based on a set of globs.
* Initially it will start as a vinyl stream and if the watch config is
* set to true it will also provide a file watcher.
* @param config Config object
*/
function agnosticSource(_ref) {
var ignore = _ref.ignore,
include = _ref.include,
cwd = _ref.cwd,
_ref$watch = _ref.watch,
watching = _ref$watch === void 0 ? false : _ref$watch;
var allGlobs = [].concat(include, ignore.map(function (a) {
return "!" + a;
}));
var vinylFsStream = vfs__default['default'].src(allGlobs, {
buffer: true,
read: true,
dot: true,
cwd: cwd,
allowEmpty: true
});
var watcher = getWatcher(watching, cwd, include, ignore);
var stream = mergeStream__default['default'](vinylFsStream, watcher.stream);
vinylFsStream.on("end", function () {
// Send ready event when our initial scan of the folder is done
stream.write("ready");
});
var close = function close() {
return watcher.fswatcher.close();
};
stream.on("end", async function () {
await close();
});
return {
stream: stream,
close: close
};
}
function isFile(file) {
return File__default['default'].isVinyl(file);
}
function isEvent(file) {
return typeof file === "string";
}
function hash(input) {
return crypto__default['default'].createHash("md5").update(input).digest("hex");
}
/**
* Stream API for utilizing stream functions
*
* @param push Function for pushing pipeline items to the stream
* @param next Function for passing errors or pushing pipeline items to the stream and then triggering the next ingestion
*/
var defaultStreamOptions = {
readableObjectMode: true,
writableObjectMode: true,
objectMode: true
};
var defaultTransformFn = function defaultTransformFn(f) {
return f;
};
function transform(transformFn, options) {
if (transformFn === void 0) {
transformFn = defaultTransformFn;
}
if (options === void 0) {
options = defaultStreamOptions;
}
var mergedOpts = Object.assign({}, defaultStreamOptions, options);
return through__default['default'](mergedOpts, async function (item, _, next) {
await processInput({
transformFn: transformFn,
next: next,
self: this,
item: item
});
});
}
transform.file = function transformFiles(transformFn, options) {
if (transformFn === void 0) {
transformFn = defaultTransformFn;
}
if (options === void 0) {
options = defaultStreamOptions;
}
var mergedOpts = Object.assign({}, defaultStreamOptions, options);
return through__default['default'](mergedOpts, async function (item, _, next) {
await processInput({
transformFn: transformFn,
next: next,
self: this,
filesOnly: true,
item: item
});
});
};
async function processInput(_ref) {
var transformFn = _ref.transformFn,
next = _ref.next,
self = _ref.self,
filesOnly = _ref.filesOnly,
item = _ref.item;
var push = self.push.bind(self); // Forward events without running transformFn
if (filesOnly && isEvent(item)) return next(null, item);
var transformed = await Promise.resolve(transformFn(item, {
push: push,
next: next
}));
if (transformed instanceof Error) {
return next(transformed);
}
if (transformed) {
next(null, transformed);
}
}
/**
* Returns a stage that prepares files coming into the stream
* with correct event information as well as hash information
* This is used by the work optimizer and elsewhere to manage the
* way files are handled and optimized
*/
function createEnrichFiles() {
var stream = transform.file(function (file, _ref) {
var next = _ref.next;
// Don't send directories
if (file.isDirectory()) {
next();
return;
}
if (!file.event) {
file.event = "add";
}
if (!file.hash) {
var _file$stat;
file.hash = hash(file.path + ((_file$stat = file.stat) == null ? void 0 : _file$stat.mtime.toString()));
}
return file;
});
return {
stream: stream
};
}
var FileCache = /*#__PURE__*/function () {
function FileCache() {
this.fileCache = {};
}
var _proto = FileCache.prototype;
_proto["delete"] = function _delete(file) {
delete this.fileCache[file.path];
};
_proto.add = function add(file) {
this.fileCache[file.path] = {
path: file.path
};
};
_proto.filterByPath = function filterByPath(filterFn) {
return Object.entries(this.fileCache).filter(function (_ref) {
var path = _ref[0];
return filterFn(path);
}).map(function (_ref2) {
_ref2[0];
var entry = _ref2[1];
return entry;
});
};
_proto.filter = function filter(filterFn) {
return Object.values(this.fileCache).filter(filterFn);
};
_proto.toString = function toString() {
return JSON.stringify(this.fileCache);
};
_proto.toPaths = function toPaths() {
return Object.keys(this.fileCache);
};
FileCache.create = function create() {
return new FileCache();
};
return FileCache;
}();
/**
* Provides a file cache of the files running through the stream
* The cache can be used elsewhere in the stream for dynamic analysis
* of multiple files.
*/
function createFileCache(filter) {
if (filter === void 0) {
filter = function filter() {
return true;
};
}
var cache = FileCache.create();
var stream = transform.file(function (file, _ref3) {
var next = _ref3.next;
if (isEvent(file)) return next(null, file); // Don't cache files that dont match the filter
if (!filter(file)) {
return next(null, file);
}
if (file.event === "unlink") {
cache["delete"](file);
} else {
cache.add(file);
}
next(null, file);
}, {
objectMode: true,
highWaterMark: 1
});
return {
stream: stream,
cache: cache
};
}
/**
* Idle handler will fire events when the stream is idle
* for a certain amount of time.
*
* The first time it fires it will also fire a ready event
*/
var createIdleHandler = function createIdleHandler(bus, delay) {
if (delay === void 0) {
delay = 500;
}
var timeout;
var handler = function handler() {
bus.write({
type: IDLE
});
};
function resetTimeout() {
destroyTimeout();
timeout = global.setTimeout(handler, delay);
}
function destroyTimeout() {
clearTimeout(timeout);
}
var stream = through__default['default']({
objectMode: true
}, function (f, _, next) {
if (isEvent(f) && f === "ready") {
bus.write({
type: READY
});
}
resetTimeout();
next(null, f);
});
stream.on("end", function () {
destroyTimeout();
handler();
});
return {
stream: stream
};
};
function _extends() {
_extends = Object.assign || function (target) {
for (var i = 1; i < arguments.length; i++) {
var source = arguments[i];
for (var key in source) {
if (Object.prototype.hasOwnProperty.call(source, key)) {
target[key] = source[key];
}
}
}
return target;
};
return _extends.apply(this, arguments);
}
var RouteCache = /*#__PURE__*/function () {
function RouteCache() {
this.routeCache = {};
this.lengthOfHTTPErrorURI = 4;
}
var _proto = RouteCache.prototype;
_proto.normalizePath = function normalizePath(input) {
if (path__default['default'].sep === path__default['default'].posix.sep) return input;
return input.split(path__default['default'].sep).join(path__default['default'].posix.sep);
};
_proto.getUrifromPath = function getUrifromPath(path) {
var uri = path;
var findStr = "/pages";
var findStrIdx = path.indexOf(findStr);
if (findStrIdx >= 0) {
uri = path.substring(findStrIdx + findStr.length, path.lastIndexOf("."));
} else {
findStr = "/api";
findStrIdx = path.indexOf(findStr);
uri = "/api" + path.substring(findStrIdx + findStr.length, path.lastIndexOf("."));
}
var uriWithoutIndex = uri.replace("/index", "");
return uriWithoutIndex.length > 0 ? uriWithoutIndex : "/";
};
_proto.getVerb = function getVerb(type) {
switch (type) {
case "api":
return "*";
case "rpc":
return "post";
default:
return "get";
}
};
_proto.isErrorCode = function isErrorCode(uri) {
if (uri.length === this.lengthOfHTTPErrorURI) {
// need better way to check HTTP error code
var regex = /^[1-5][0-9][0-9]$/;
return regex.test(uri.substring(1));
}
return false;
};
_proto.getType = function getType(file) {
var pagesPathRegex = /(pages[\\/][^_.].+(?<!\.test)\.(m?[tj]sx?|mdx))$/;
var rpcPathRegex = /(api[\\/].+[\\/](queries|mutations).+)$/;
var apiPathRegex = /(api[\\/].+\.[tj]s)$/;
if (rpcPathRegex.test(file.path)) {
return "rpc";
} else if (apiPathRegex.test(file.path)) {
return "api";
} else if (pagesPathRegex.test(file.path)) {
return "page";
}
return null;
};
_proto.add = function add(file) {
var _file$originalRelativ;
var srcPath = (_file$originalRelativ = file.originalRelative) != null ? _file$originalRelativ : file.relative;
if (this.routeCache[srcPath]) return;
var type = this.getType(file);
if (!type) {
return;
}
var uri = this.getUrifromPath(this.normalizePath(file.path));
var isErrorCode = this.isErrorCode(uri);
if (!isErrorCode) {
this.routeCache[srcPath] = {
path: srcPath,
uri: uri,
verb: this.getVerb(type),
type: type
};
}
};
_proto["delete"] = function _delete(file) {
var _file$originalRelativ2;
var srcPath = (_file$originalRelativ2 = file.originalRelative) != null ? _file$originalRelativ2 : file.relative;
delete this.routeCache[srcPath];
};
_proto.filterByPath = function filterByPath(filterFn) {
return Object.entries(this.routeCache).filter(function (_ref) {
var path = _ref[0];
return filterFn(path);
}).map(function (_ref2) {
_ref2[0];
var entry = _ref2[1];
return entry;
});
};
_proto.filter = function filter(filterFn) {
return Object.values(this.routeCache).filter(filterFn);
};
_proto.get = function get(key) {
var _key$originalRelative;
if (typeof key === "string") return this.routeCache[key];
var srcPath = (_key$originalRelative = key == null ? void 0 : key.originalRelative) != null ? _key$originalRelative : key == null ? void 0 : key.relative;
if (srcPath) return this.routeCache[srcPath];
return this.routeCache;
};
_proto.set = function set(key, value) {
var _value$path;
this.routeCache[key] = _extends({}, value, {
path: (_value$path = value.path) != null ? _value$path : "-"
});
};
_proto.toString = function toString() {
return JSON.stringify(this.routeCache, null, 2);
};
_proto.toArray = function toArray() {
return Object.values(this.routeCache);
};
RouteCache.create = function create() {
if (RouteCache.singleton) return RouteCache.singleton;
RouteCache.singleton = new RouteCache();
return RouteCache.singleton;
};
return RouteCache;
}();
/**
* Provides a route cache of the files running through the stream
*/
RouteCache.singleton = null;
function createRouteCache() {
var cache = RouteCache.create();
return {
cache: cache
};
}
// Mostly concerned with solving the Dirty Sync problem
var defaultSaveCache = /*#__PURE__*/debounce__default['default'](function (filePath, data) {
return fsExtra.writeFile(filePath, Buffer.from(JSON.stringify(data, null, 2))).then(function () {})["catch"](function () {});
}, 500);
var defaultReadCache = function defaultReadCache(filePath) {
// We need to do sync file reading here as this cache
// must be loaded before the stream is added to the pipeline
// or we end up with more complexity having to cache files as they come in
return fsExtra.existsSync(filePath) ? fsExtra.readFileSync(filePath).toString() : "";
};
/**
* Returns streams that help handling work optimisation in the file transform stream.
*/
function createWorkOptimizer(src, dest, overrideTriage, saveCache, readCache) {
if (overrideTriage === void 0) {
overrideTriage = function overrideTriage() {
return undefined;
};
}
if (saveCache === void 0) {
saveCache = defaultSaveCache;
}
if (readCache === void 0) {
readCache = defaultReadCache;
}
var getOriginalPathHash = function getOriginalPathHash(file) {
return hash(path.relative(src, file.history[0]));
};
var doneCacheLocation = path.resolve(dest, ".blitz.incache.json");
var doneStr = readCache(doneCacheLocation);
var todo = {};
var done = doneStr ? JSON.parse(doneStr) : {};
var stats = {
todo: todo,
done: done
};
var reportComplete = transform.file(async function (file) {
var pathHash = getOriginalPathHash(file);
delete todo[pathHash];
if (file.event === "add") {
done[pathHash] = file.hash;
}
if (file.event === "unlink") {
delete done[pathHash];
}
await saveCache(doneCacheLocation, done);
return file;
});
var triage = transform.file(function (file, _ref) {
var push = _ref.push,
next = _ref.next;
var pathHash = getOriginalPathHash(file);
switch (overrideTriage(file)) {
case "proceed":
push(file);
return next();
case "ignore":
return next();
}
if (!file.hash) {
display.log.debug("File does not have hash! " + file.path);
return next();
} // Dont send files that have already been done or have already been added
if (done[pathHash] === file.hash || todo[pathHash] === file.hash) {
display.log.debug("Rejecting because this job has been done before: " + file.path);
return next();
}
todo[pathHash] = file.hash;
push(file);
next();
});
return {
triage: triage,
reportComplete: reportComplete,
stats: stats
};
}
function getDestPath(folder, file) {
return path.resolve(folder, path.relative(file.cwd, file.path));
}
/**
* Deletes a file in the stream from the filesystem
* @param folder The destination folder
*/
function unlink(folder, unlinkFile) {
if (unlinkFile === void 0) {
unlinkFile = rimraf;
}
return transform.file(async function (file) {
if (file.event === "unlink" || file.event === "unlinkDir") {
var destPath = getDestPath(folder, file);
await unlinkFile(destPath, {
glob: false
});
}
return file;
});
}
var isUnlinkFile = function isUnlinkFile(file) {
return file.event === "unlink" || file.event === "unlinkDir";
};
/**
* Returns a Stage that writes files to the destination path
*/
var createWrite = function createWrite(destination, reporter, // Allow the writer to be overriden
writeStream, unlinkStream) {
if (writeStream === void 0) {
writeStream = vfs.dest(destination);
}
if (unlinkStream === void 0) {
unlinkStream = unlink(destination);
}
var splitToBus = transform.file(function (file) {
reporter.write({
type: isUnlinkFile(file) ? FILE_DELETED : FILE_WRITTEN,
payload: file
});
return file;
});
var stream = gulpIf__default['default'](isFile, gulpIf__default['default'](isUnlinkFile, pipeline(unlinkStream, splitToBus), pipeline(writeStream, splitToBus)));
return {
stream: stream
};
};
function isSourceFile(file) {
var _file$hash;
return ((_file$hash = file.hash) == null ? void 0 : _file$hash.indexOf(":")) === -1;
}
function createStageArgs(config, input, bus, fileCache, routeCache) {
var getInputCache = function getInputCache() {
return fileCache;
};
var getRouteCache = function getRouteCache() {
return routeCache;
};
function processNewFile(file) {
if (!file.stat) {
// Add a stats here so we can then generate a new ID
// during enrichment
var stat = new fs.Stats();
file.stat = stat;
file.event = "add";
}
input.write(file);
}
function processNewChildFile(_ref) {
var parent = _ref.parent,
child = _ref.child,
stageId = _ref.stageId,
subfileId = _ref.subfileId;
child.hash = [parent.hash, stageId, subfileId].join("|");
processNewFile(child);
}
return {
config: config,
input: input,
bus: bus,
getInputCache: getInputCache,
getRouteCache: getRouteCache,
processNewFile: processNewFile,
processNewChildFile: processNewChildFile
};
}
/**
* Creates a pipeline stream that transforms files.
* @param config Config object containing basic information for the file pipeline
* @param stages Array of stages to apply to each file
* @param errors Stream that takes care of all operational error rendering
* @param bus Stream to pipe events to
*/
function createPipeline(config, stages, bus, // Initialise source and writer here so we can inject them in testing
source, writer) {
if (source === void 0) {
source = agnosticSource(config);
}
if (writer === void 0) {
writer = createWrite(config.dest, bus);
}
// Helper streams don't account for business stages
var input = through__default['default'].obj();
var optimizer = createWorkOptimizer(config.src, config.dest, config.overrideTriage);
var enrichFiles = createEnrichFiles();
var srcCache = createFileCache(isSourceFile);
var routeCache = createRouteCache();
var idleHandler = createIdleHandler(bus); // Send this object to every stage
var api = createStageArgs(config, input, bus, srcCache.cache, routeCache.cache); // Initialize each stage
var initializedStages = stages.map(function (stage) {
return stage(api);
}); // Discard git ignored files
var ignorer = through__default['default'].obj(function (file, _, next) {
if (file && file.path) {
var match = micromatch__default['default'].isMatch(file.path, config.ignore);
if (match) {
return next(); // skip chunk
}
}
next(null, file);
});
var stream = pipeline.apply(void 0, [source.stream, // files come from file system
input, // files coming via internal API
// Preparing files
enrichFiles.stream, srcCache.stream, optimizer.triage, // Filter files
ignorer].concat(initializedStages.map(function (stage) {
return stage.stream;
}), [// Tidy up
writer.stream, optimizer.reportComplete, idleHandler.stream]));
var ready = Object.assign.apply(Object, [{}].concat(initializedStages.map(function (stage) {
return stage.ready;
})));
return {
stream: stream,
ready: ready
};
}
var defaultBus = /*#__PURE__*/through__default['default'].obj();
/**
* Assembles a file stranform pipeline to convert blitz source code to something that
* can run in NextJS.
* @param config Configuration object
*/
async function transformFiles(src, stages, dest, options) {
var _options$ignore = options.ignore,
ignore = _options$ignore === void 0 ? [] : _options$ignore,
_options$include = options.include,
include = _options$include === void 0 ? [] : _options$include,
_options$watch = options.watch,
watch = _options$watch === void 0 ? false : _options$watch,
_options$bus = options.bus,
bus = _options$bus === void 0 ? defaultBus : _options$bus,
source = options.source,
writer = options.writer,
requestClean = options.clean,
overrideTriage = options.overrideTriage;
if (requestClean) await rimraf(dest, {
glob: false
}); // Fix windows EPERM issues
if (process.platform === "win32") await rimraf(path.join(dest, ".next"), {
glob: false
});
var display = createDisplay();
return await new Promise(function (resolve, reject) {
var fileTransformPipeline = createPipeline({
cwd: src,
src: src,
dest: dest,
include: include,
ignore: ignore,
watch: watch,
overrideTriage: overrideTriage
}, stages, bus, source, writer);
bus.on("data", function (_ref) {
var type = _ref.type;
if (type === READY) {
resolve(fileTransformPipeline.ready);
}
}); // Send source to fileTransformPipeline
fileTransformPipeline.stream.on("error", function (err) {
bus.write({
type: ERROR_THROWN,
payload: err
});
if (err) reject(err);
}); // Send reporter events to display
pipe__default['default'](bus, display.stream, function (err) {
if (err) reject(err);
});
});
}
exports.ERROR_THROWN = ERROR_THROWN;
exports.FILE_DELETED = FILE_DELETED;
exports.FILE_WRITTEN = FILE_WRITTEN;
exports.FileCache = FileCache;
exports.IDLE = IDLE;
exports.INIT = INIT;
exports.READY = READY;
exports.RouteCache = RouteCache;
exports.rimraf = rimraf;
exports.transform = transform;
exports.transformFiles = transformFiles;