UNPKG

nodent

Version:

NoDent - Asynchronous Javascript language extensions

775 lines (707 loc) 26.6 kB
#!/usr/bin/env node 'use strict'; /** * NoDent - Asynchronous JavaScript Language extensions for Node * * AST transforms and node loader extension */ var stdJSLoader ; var fs = require('fs') ; var NodentCompiler = require('nodent-compiler') ; // Config options used to control the run-time behaviour of the compiler var config = { log:function(msg){ console.warn("Nodent: "+msg) }, // Where to print errors and warnings augmentObject:false, // Only one has to say 'yes' extension:'.njs', // The 'default' extension dontMapStackTraces:false, // Only one has to say 'no' asyncStackTrace:false, babelTree:false, dontInstallRequireHook:false } ; // Code generation options, which are determined by the "use nodent"; directive. Specifically, // the (optional) portion as in "use nodent<-option-set>"; is used to select a compilation // behaviour on a file-by-file basis. There are preset option sets called "es7", "promise(s)" // and "generators" (which cannot be over-written). Others can be specified via the // 'setCompileOptions(name,options)' call, or via the 'nodent:{directive:{'name':{...}}}' entry in the // current project's package.json. In the latter's case, the name 'default' is also reserved and // is used for a bare 'use nodent' driective with no project-specific extension. // Finally, the 'use nodent' directive can be followed by a JSON encoded set of options, for example: // 'use nodent-es7 {"wrapAwait":true}'; // 'use nodent {"wrapAwait":true,"promise":true}'; // 'use nodent-generators {"parser":{"sourceType":"module"}}'; // // The order of application of these options is: // initialCodeGenOpts (hard-coded) // named by the 'use nodent-OPTION', as read from the package.json // named by the 'use nodent-OPTION', as set by the setCompileOptions (or setDefaultCompileOptions) // set within the directive as a JSON-encoded extension function copyObj(a){ var o = {} ; a.forEach(function(b){ if (b && typeof b==='object') for (var k in b) o[k] = b[k] ; }) ; return o ; }; var defaultCodeGenOpts = Object.create(NodentCompiler.initialCodeGenOpts, {es7:{value:true,writable:true,enumerable:true}}) ; var optionSets = { default:defaultCodeGenOpts, es7:Object.create(defaultCodeGenOpts), promise:Object.create(defaultCodeGenOpts,{ promises:{value:true,writable:true,enumerable:true} }), generator:Object.create(defaultCodeGenOpts,{ generators:{value:true,writable:true,enumerable:true}, es7:{value:false,writable:true,enumerable:true} }), engine:Object.create(defaultCodeGenOpts,{ engine:{value:true,writable:true,enumerable:true}, promises:{value:true,writable:true,enumerable:true} }), host:Object.create(defaultCodeGenOpts,{ promises:{value:"host",writable:true,enumerable:true}, es6target:{value:"host",writable:true,enumerable:true}, engine:{value:"host",writable:true,enumerable:true} }) }; optionSets.promises = optionSets.promise ; optionSets.generators = optionSets.generator ; function globalErrorHandler(err) { throw err ; } /* Extract compiler options from code (either a string or AST) */ var useDirective = /^\s*['"]use\s+nodent-?([a-zA-Z0-9]*)?(\s*.*)?['"]\s*;/ var runtimes = require('nodent-runtime') ; var $asyncbind = runtimes.$asyncbind ; var $asyncspawn = runtimes.$asyncspawn ; var Thenable = $asyncbind.Thenable ; function isDirective(node){ return node.type === 'ExpressionStatement' && (node.expression.type === 'StringLiteral' || (node.expression.type === 'Literal' && typeof node.expression.value === 'string')) ; } var hostOptions = { promises:"Promise", es6target:"()=>0", engine:"(async ()=>0)", noRuntime:"Promise" } ; function parseCompilerOptions(code,log,filename) { if (!log) log = console.warn.bind(console) ; var regex, set, parseOpts = {} ; if (typeof code=="string") { if (regex = code.match(useDirective)) { set = regex[1] || 'default' ; } } else { // code is an AST for (var i=0; i<code.body.length; i++) { if (isDirective(code.body[i].type)) { var test = "'"+code.body[i].value+"'" ; if (regex = test.match(useDirective)) { set = regex[1] || 'default' ; break ; } } else { break ; // Directives should preceed all other statements } } } if (!regex) { if (!defaultCodeGenOpts.noUseDirective) return null ; set = "default" ; regex = [null,null,"{}"] ; } if (set) { try { if (!filename) filename = require('path').resolve('.') ; else if (!require('fs').lstatSync(filename).isDirectory()) filename = require('path').dirname(filename) ; var packagePath = require('resolve').sync('package.json',{ moduleDirectory:[''], extensions:[''], basedir:filename }) ; var packageOptions = JSON.parse(fs.readFileSync(packagePath)).nodent.directive[set] ; } catch(ex) { // Meh } } try { parseOpts = copyObj([optionSets[set],packageOptions,regex[2] && JSON.parse(regex[2])]); } catch(ex) { log("Invalid literal compiler option: "+((regex && regex[0]) || "<no options found>")); } Object.keys(hostOptions).forEach(function(k){ if (parseOpts[k] === 'host') { parseOpts[k] = (function(){ try { eval(hostOptions[k]) ; return true } catch (ex) { return false } })() } }) ; if (parseOpts.promises || parseOpts.es7 || parseOpts.generators || parseOpts.engine) { if ((((parseOpts.promises || parseOpts.es7) && parseOpts.generators))) { log("No valid 'use nodent' directive, assumed -es7 mode") ; parseOpts = optionSets.es7 ; } if (parseOpts.generators || parseOpts.engine) parseOpts.promises = true ; if (parseOpts.promises) parseOpts.es7 = true ; return parseOpts ; } return null ; // No valid nodent options } function stripBOM(content) { // Remove byte order marker. This catches EF BB BF (the UTF-8 BOM) // because the buffer-to-string conversion in `fs.readFileSync()` // translates it to FEFF, the UTF-16 BOM. if (content.charCodeAt(0) === 0xFEFF) { content = content.slice(1); } if (content.substring(0,2) === "#!") { content = "//"+content ; } return content; } function compileNodentedFile(nodent,log) { log = log || nodent.log ; return function(mod, filename, parseOpts) { var content = stripBOM(fs.readFileSync(filename, 'utf8')); var pr = nodent.parse(content,filename,parseOpts); parseOpts = parseOpts || parseCompilerOptions(pr.ast,log, filename) ; nodent.asynchronize(pr,undefined,parseOpts,log) ; nodent.prettyPrint(pr,parseOpts) ; mod._compile(pr.code, pr.filename); } }; // Things that DON'T depend on initOpts (or config, and therefore nodent) function asyncify(promiseProvider) { promiseProvider = promiseProvider || Thenable ; return function(obj,filter,suffix) { if (Array.isArray(filter)) { var names = filter ; filter = function(k,o) { return names.indexOf(k)>=0 ; } } else { filter = filter || function(k,o) { return (!k.match(/Sync$/) || !(k.replace(/Sync$/,"") in o)) ; }; } if (!suffix) suffix = "" ; var o = Object.create(obj) ; for (var j in o) (function(){ var k = j ; try { var pd ; for (var po = obj; po; po = po.__proto__) { if (pd = Object.getOwnPropertyDescriptor(po,k)) break ; } if (pd.value && typeof pd.value==='function' && (!o[k+suffix] || !o[k+suffix].isAsync) && filter(k,o)) { o[k+suffix] = function() { var a = Array.prototype.slice.call(arguments) ; var resolver = function($return,$error) { var cb = function(err,ok){ if (err) return $error(err) ; switch (arguments.length) { case 0: return $return() ; case 2: return $return(ok) ; default: return $return(Array.prototype.slice.call(arguments,1)) ; } } ; // If more args were supplied than declared, push the CB if (a.length > obj[k].length) { a.push(cb) ; } else { // Assume the CB is the final arg a.push(cb) ; } var ret = obj[k].apply(obj,a) ; } ; return new promiseProvider(resolver) ; } o[k+suffix].isAsync = true ; } } catch (ex) { // Log the fact that we couldn't augment this member?? } })() ; o["super"] = obj ; return o ; } }; function generateRequestHandler(path, matchRegex, options) { var cache = {} ; var compiler = this ; if (!matchRegex) matchRegex = /\.njs$/ ; if (!options) options = {compiler:{}} ; else if (!options.compiler) options.compiler = {} ; var compilerOptions = copyObj([NodentCompiler.initialCodeGenOpts,options.compiler]) ; return function (req, res, next) { if (req.url.indexOf('..')>=0) return next() ; if (cache[req.url]) { res.setHeader("Content-Type", cache[req.url].contentType); options.setHeaders && options.setHeaders(res) ; res.write(cache[req.url].output) ; res.end(); return ; } if (!req.url.match(matchRegex) && !(options.htmlScriptRegex && req.url.match(options.htmlScriptRegex))) { return next && next() ; } function sendException(ex) { res.statusCode = 500 ; res.write(ex.toString()) ; res.end() ; } var filename = path+req.url ; if (options.extensions && !fs.existsSync(filename)) { for (var i=0; i<options.extensions.length; i++) { if (fs.existsSync(filename+"."+options.extensions[i])) { filename = filename+"."+options.extensions[i] ; break ; } } } fs.readFile(filename,function(err,content){ if (err) { return sendException(err) ; } else { try { var pr,contentType ; if (options.htmlScriptRegex && req.url.match(options.htmlScriptRegex)) { pr = require('./htmlScriptParser')(compiler,content.toString(),req.url,options) ; contentType = "text/html" ; } else { if (options.runtime) { pr = "Function.prototype."+compilerOptions.$asyncbind+" = "+$asyncbind.toString()+";" ; if (compilerOptions.generators) pr += "Function.prototype."+compilerOptions.$asyncspawn+" = "+$asyncspawn.toString()+";" ; if (compilerOptions.wrapAwait && !compilerOptions.promises) pr += "Object."+compilerOptions.$makeThenable+" = "+Thenable.resolve.toString()+";" ; compilerOptions.mapStartLine = pr.split("\n").length ; pr += "\n"; } else { pr = "" ; } pr += compiler.compile(content.toString(),req.url,null,compilerOptions).code; contentType = "application/javascript" ; } res.setHeader("Content-Type", contentType); if (options.enableCache) cache[req.url] = {output:pr,contentType:contentType} ; options.setHeaders && options.setHeaders(res) ; res.write(pr) ; res.end(); } catch (ex) { return sendException(ex) ; } } }) ; }; }; function requireCover(cover,opts) { opts = opts || {} ; var key = cover+'|'+Object.keys(opts).sort().reduce(function(a,k){ return a+k+JSON.stringify(opts[k])},"") ; if (!this.covers[key]) { if (cover.indexOf("/")>=0) this.covers[key] = require(cover) ; else this.covers[key] = require(__dirname+"/covers/"+cover); } return this.covers[key](this,opts) ; } NodentCompiler.prototype.Thenable = Thenable ; NodentCompiler.prototype.EagerThenable = $asyncbind.EagerThenableFactory ; NodentCompiler.prototype.asyncify = asyncify ; NodentCompiler.prototype.require = requireCover ; NodentCompiler.prototype.generateRequestHandler = generateRequestHandler ; // Exported so they can be transported to a client NodentCompiler.prototype.$asyncspawn = $asyncspawn ; NodentCompiler.prototype.$asyncbind = $asyncbind ; NodentCompiler.prototype.parseCompilerOptions = parseCompilerOptions ; $asyncbind.call($asyncbind) ; function prepareMappedStackTrace(error, stack) { function mappedTrace(frame) { var source = frame.getFileName(); if (source && NodentCompiler.prototype.smCache[source]) { var position = NodentCompiler.prototype.smCache[source].smc.originalPositionFor({ line: frame.getLineNumber(), column: frame.getColumnNumber() }); if (position && position.line) { var desc = frame.toString() ; return '\n at ' + desc.substring(0,desc.length-1) + " => \u2026"+position.source+":"+position.line+":"+position.column + (frame.getFunctionName()?")":""); } } return '\n at '+frame; } return error + stack.map(mappedTrace).join(''); } // Set the 'global' references to the (backward-compatible) versions // required by the current version of Nodent function setGlobalEnvironment(initOpts) { var codeGenOpts = defaultCodeGenOpts ; /* Object.$makeThenable Object.prototype.isThenable Object.prototype.asyncify Function.prototype.noDentify // Moved to a cover as of v3.0.0 Function.prototype.$asyncspawn Function.prototype.$asyncbind Error.prepareStackTrace global[defaultCodeGenOpts.$error] */ var augmentFunction = {} ; augmentFunction[defaultCodeGenOpts.$asyncbind] = { value:$asyncbind, writable:true, enumerable:false, configurable:true }; augmentFunction[defaultCodeGenOpts.$asyncspawn] = { value:$asyncspawn, writable:true, enumerable:false, configurable:true }; try { Object.defineProperties(Function.prototype,augmentFunction) ; } catch (ex) { initOpts.log("Function prototypes already assigned: ",ex.messsage) ; } /** * We need a global to handle funcbacks for which no error handler has ever been defined. */ if (!(defaultCodeGenOpts[defaultCodeGenOpts.$error] in global)) { global[defaultCodeGenOpts[defaultCodeGenOpts.$error]] = globalErrorHandler ; } // "Global" options: // If anyone wants to augment Object, do it. The augmentation does not depend on the config options if (initOpts.augmentObject) { Object.defineProperties(Object.prototype,{ "asyncify":{ value:function(promiseProvider,filter,suffix){ return asyncify(promiseProvider)(this,filter,suffix) }, writable:true, configurable:true }, "isThenable":{ value:function(){ return Thenable.isThenable(this) }, writable:true, configurable:true } }) ; } Object[defaultCodeGenOpts.$makeThenable] = Thenable.resolve ; } /* Construct a 'nodent' object - combining logic and options */ var compiler ; function initialize(initOpts){ // Validate the options /* initOpts:{ * log:function(msg), * augmentObject:boolean, * extension:string? * dontMapStackTraces:boolean */ if (!initOpts) initOpts = {} ; else { // Throw an error for any options we don't know about for (var k in initOpts) { if (k==="use") continue ; // deprecated if (!config.hasOwnProperty(k)) throw new Error("NoDent: unknown option: "+k+"="+JSON.stringify(initOpts[k])) ; } } if (compiler) { compiler.setOptions(initOpts); } else { // Fill in any missing options with their default values Object.keys(config).forEach(function(k){ if (!(k in initOpts)) initOpts[k] = config[k] ; }) ; compiler = new NodentCompiler(initOpts) ; } // If anyone wants to mapStackTraces, do it. The augmentation does not depend on the config options if (!initOpts.dontMapStackTraces) { // This function is part of the V8 stack trace API, for more info see: // http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi Error.prepareStackTrace = prepareMappedStackTrace ; } setGlobalEnvironment(initOpts) ; /* If we've not done it before, create a compiler for '.js' scripts */ // Create a new compiler var nodentLoaders = []; function compareSemVer(a,b) { a = a.split('.') ; b = b.split('.') ; for (var i=0;i<3;i++) { if (a[i]<b[i]) return -1 ; if (a[i]>b[i]) return 1 ; } return 0 ; } function versionAwareNodentJSLoader(mod,filename) { if (filename.match(/nodent\/nodent\.js$/)) { var downLevel = {path:filename.replace(/\/node_modules\/nodent\/nodent\.js$/,"")} ; if (downLevel.path) { downLevel.version = JSON.parse(fs.readFileSync(filename.replace(/nodent\.js$/,"package.json"))).version ; // Load the specified nodent stdJSLoader(mod,filename) ; // If the version of nodent we've just loaded is lower than the // current (version-aware) version, hook the initialzer // so we can replace the JS loader after it's been run. if (compareSemVer(downLevel.version,NodentCompiler.prototype.version)<0) { downLevel.originalNodentLoader = mod.exports ; mod.exports = function(){ var previousJSLoader = require.extensions['.js'] ; var defaultNodentInstance = downLevel.originalNodentLoader.apply(this,arguments) ; downLevel.jsCompiler = require.extensions['.js'] ; require.extensions['.js'] = previousJSLoader ; setGlobalEnvironment(initOpts) ; return defaultNodentInstance ; } ; Object.keys(downLevel.originalNodentLoader).forEach(function(k){ mod.exports[k] = downLevel.originalNodentLoader[k] ; }) ; nodentLoaders.push(downLevel) ; nodentLoaders = nodentLoaders.sort(function(a,b){ return b.path.length - a.path.length ; }) ; } } } else if (filename.match(/node_modules\/nodent\/.*\.js$/)) { // Things inside nodent always use the standard loader return stdJSLoader(mod,filename) ; } else { // The the appropriate loader for this file for (var n=0; n<nodentLoaders.length; n++) { if (filename.slice(0,nodentLoaders[n].path.length)==nodentLoaders[n].path) { //console.log("Using nodent@",nodentLoaders[n].version,"to load",filename) ; if (!nodentLoaders[n].jsCompiler) { // The client app loaded, but never initialised nodent (so it can only // be using library/runtime functions, not the require-hook compiler) return stdJSLoader(mod,filename) ; } else { if (nodentLoaders[n].jsCompiler === versionAwareNodentJSLoader) break ; return nodentLoaders[n].jsCompiler.apply(this,arguments) ; } } } var content = stripBOM(fs.readFileSync(filename, 'utf8')); var parseOpts = parseCompilerOptions(content,initOpts.log,filename) ; if (parseOpts) { return stdCompiler(mod,filename,parseOpts) ; } return stdJSLoader(mod,filename) ; } } function registerExtension(extension) { if (Array.isArray(extension)) return extension.forEach(registerExtension) ; if (require.extensions[extension]) { var changedKeys = Object.keys(initOpts).filter(function(k){ return compiler[k] != initOpts[k]}) ; if (changedKeys.length) { initOpts.log("File extension "+extension+" already configured for async/await compilation.") ; } } require.extensions[extension] = compileNodentedFile(compiler,initOpts.log) ; } if (!initOpts.dontInstallRequireHook) { if (!stdJSLoader) { stdJSLoader = require.extensions['.js'] ; var stdCompiler = compileNodentedFile(compiler,initOpts.log) ; require.extensions['.js'] = versionAwareNodentJSLoader ; } /* If the initOpts specified a file extension, use this compiler for it */ if (initOpts.extension) { registerExtension(initOpts.extension) ; } } // Finally, load any required covers if (initOpts.use) { if (Array.isArray(initOpts.use)) { initOpts.log("Warning: nodent({use:[...]}) is deprecated. Use nodent.require(module,options)\n"+(new Error().stack).split("\n")[2]); if (initOpts.use.length) { initOpts.use.forEach(function(x){ compiler[x] = compiler.require(x) ; }) ; } } else { initOpts.log("Warning: nodent({use:{...}}) is deprecated. Use nodent.require(module,options)\n"+(new Error().stack).split("\n")[2]); Object.keys(initOpts.use).forEach(function(x){ compiler[x] = compiler.require(x,initOpts.use[x]) }) ; } } return compiler ; } ; /* Export these so that we have the opportunity to set the options for the default .js parser */ initialize.setDefaultCompileOptions = function(compiler,env) { if (compiler) { Object.keys(compiler).forEach(function(k){ if (!(k in defaultCodeGenOpts)) throw new Error("NoDent: unknown compiler option: "+k) ; defaultCodeGenOpts[k] = compiler[k] ; }) ; } env && Object.keys(env).forEach(function(k){ if (!(k in env)) throw new Error("NoDent: unknown configuration option: "+k) ; config[k] = env[k] ; }) ; return initialize ; }; initialize.setCompileOptions = function(set,compiler) { optionSet[set] = optionSet[set] || copyObj([defaultCodeGenOpts]); compiler && Object.keys(compiler).forEach(function(k){ if (!(k in defaultCodeGenOpts)) throw new Error("NoDent: unknown compiler option: "+k) ; optionSet[set][k] = compiler[k] ; }) ; return initialize ; }; initialize.asyncify = asyncify ; initialize.Thenable = $asyncbind.Thenable ; initialize.EagerThenable = $asyncbind.EagerThenableFactory ; module.exports = initialize ; function runFromCLI(){ function readStream(stream) { return new Thenable(function ($return, $error) { var buffer = [] ; stream.on('data',function(data){ buffer.push(data) }) ; stream.on('end',function(){ var code = buffer.map(function(b){ return b.toString()}).join("") ; return $return(code); }) ; stream.on('error',$error) ; }.$asyncbind(this)); } function getCLIOpts(start) { var o = [] ; for (var i=start || 2; i<process.argv.length; i++) { if (process.argv[i].slice(0,2)==='--') { var opt = process.argv[i].slice(2).split('=') ; o[opt[0]] = opt[1] || true ; } else o.push(process.argv[i]) ; } return o ; } function processInput(content,name){ try { var pr ; var parseOpts ; // Input options if (cli.fromast) { content = JSON.parse(content) ; pr = { origCode:"", filename:filename, ast: content } ; parseOpts = parseCompilerOptions(content,nodent.log) ; if (!parseOpts) { var directive = cli.use ? '"use nodent-'+cli.use+'";' : '"use nodent";' ; parseOpts = parseCompilerOptions(directive,nodent.log) ; console.warn("/* "+filename+": No 'use nodent*' directive, assumed "+directive+" */") ; } } else { parseOpts = parseCompilerOptions(cli.use?'"use nodent-'+cli.use+'";':content,nodent.log) ; if (!parseOpts) { parseOpts = parseCompilerOptions('"use nodent";',nodent.log) ; if (!cli.dest) console.warn("/* "+filename+": 'use nodent*' directive missing/ignored, assumed 'use nodent;' */") ; } pr = nodent.parse(content,filename,parseOpts); } // Processing options if (!cli.parseast && !cli.pretty) nodent.asynchronize(pr,undefined,parseOpts,nodent.log) ; // Output options nodent.prettyPrint(pr,parseOpts) ; if (cli.out || cli.pretty || cli.dest) { if (cli.dest && !name) throw new Error("Can't write unknown file to "+cli.dest) ; var output = "" ; if (cli.runtime) { output += ("Function.prototype.$asyncbind = "+Function.prototype.$asyncbind.toString()+";\n") ; output += ("global.$error = global.$error || "+global.$error.toString()+";\n") ; } output += pr.code ; if (name && cli.dest) { fs.writeFileSync(cli.dest+name,output) ; console.log("Compiled",cli.dest+name) ; } else { console.log(output); } } if (cli.minast || cli.parseast) { console.log(JSON.stringify(pr.ast,function(key,value){ return key[0]==="$" || key.match(/^(start|end|loc)$/)?undefined:value },2,null)) ; } if (cli.ast) { console.log(JSON.stringify(pr.ast,function(key,value){ return key[0]==="$"?undefined:value},0)) ; } if (cli.exec) { (new Function(pr.code))() ; } } catch (ex) { console.error(ex) ; } } var path = require('path') ; var initOpts = (process.env.NODENT_OPTS && JSON.parse(process.env.NODENT_OPTS)) || {}; var filename, cli = getCLIOpts() ; initialize.setDefaultCompileOptions({ sourcemap:cli.sourcemap, wrapAwait:cli.wrapAwait, lazyThenables:cli.lazyThenables, noRuntime:cli.noruntime, es6target:cli.es6target, parser:cli.noextensions?{noNodentExtensions:true}:undefined }); var nodent = initialize({ augmentObject:true }) ; if (!cli.fromast && !cli.parseast && !cli.pretty && !cli.out && !cli.dest && !cli.ast && !cli.minast && !cli.exec) { // No input/output options - just require the // specified module now we've initialized nodent try { var mod = path.resolve(cli[0]) ; return require(mod); } catch (ex) { ex && (ex.message = cli[0]+": "+ex.message) ; throw ex ; } } if (cli.length==0 || cli[0]==='-') { filename = "(stdin)" ; return readStream(process.stdin).then(processInput,globalErrorHandler) ; } else { for (var i=0; i<cli.length; i++) { filename = path.resolve(cli[i]) ; processInput(stripBOM(fs.readFileSync(filename, 'utf8')),cli[i]) ; } return ; } } if (require.main===module && process.argv.length>=3) runFromCLI() ;