UNPKG

shaka-player

Version:
868 lines (753 loc) 27.1 kB
#!/usr/bin/env node /*! @license * Shaka Player * Copyright 2016 Google LLC * SPDX-License-Identifier: Apache-2.0 */ /** * @fileoverview * * A node script that generates externs automatically from the uncompiled * source of a Closure project. Designed for Shaka Player, but may be usable * in other projects as well. Does not depend on the Closure compiler itself. * * We were not able to get our externs generated by the Closure compiler. There * were many issues with the Closure-generated externs, including the order of * the externs and the replacement of record types and enums with their * underlying types. * * This uses a node module called esprima to parse JavaScript, then explores the * abstract syntax tree from esprima. It finds exported symbols and generates * an appropriate extern definition for it. * * The generated externs are then topologically sorted according to the * goog.provide and goog.require calls in the sources. No sorting is done * within source files, and no sorting is done based on parameter types. * Circular deps between source files will not be resolved, and deps not * represented in goog.provide/goog.require will not be discovered. * * Arguments: --output <EXTERNS> <INPUT> [<INPUT> ...] */ // Load required modules. let assert = require('assert'); if (assert.strict) { // The "strict" mode was added in v9.9, use that if available. assert = assert.strict; } const esprima = require('esprima'); const fs = require('fs'); // The annotations we will consider "exporting" a symbol. const EXPORT_REGEX = /@(?:export|exportInterface|expose)\b/; // TODO: revisit this when Closure Compiler supports partially-exported classes. let partiallyExportedClassesDetected = false; /** * Topological sort of general objects using a DFS approach. * Will add a __mark field to each object as part of the sorting process. * @param {!Array.<T>} list * @param {function(T):!Array.<T>} getDeps * @return {!Array.<T>} * @template T * @see https://en.wikipedia.org/wiki/Topological_sorting#Depth-first_search */ function topologicalSort(list, getDeps) { const sorted = []; const NOT_VISITED = 0; const MID_VISIT = 1; const COMPLETELY_VISITED = 2; // Mark all objects as not visited. for (const object of list) { object.__mark = NOT_VISITED; } // Visit each object. for (const object of list) { visit(object); } // Return the sorted list. return sorted; /** * @param {T} object * @template T */ function visit(object) { if (object.__mark == MID_VISIT) { assert.fail('Dependency cycle detected!'); } else if (object.__mark == NOT_VISITED) { object.__mark = MID_VISIT; // Visit all dependencies. for (const dep of getDeps(object)) { visit(dep); } object.__mark = COMPLETELY_VISITED; // Push this object onto the list. All transitive dependencies have // already been added to the list. sorted.push(object); } } } /** * @param {ASTNode} node A node from the abstract syntax tree. * @return {boolean} true if this is a call node. */ function isCallNode(node) { // Example node: { // type: 'ExpressionStatement', // expression: { type: 'CallExpression', callee: {...}, arguments: [...] }, // } return node.type == 'ExpressionStatement' && node.expression.type == 'CallExpression'; } /** * Pretty-print a node via console.log. Useful for debugging and development * to see what the AST looks like. * @param {ASTNode} node A node from the abstract syntax tree. */ function dumpNode(node) { console.log(JSON.stringify(node, null, ' ')); } /** * @param {ASTNode} node A node from the abstract syntax tree. * @return {boolean} true if this is a call to goog.provide. */ function isProvideNode(node) { return isCallNode(node) && getIdentifierString(node.expression.callee) == 'goog.provide'; } /** * @param {ASTNode} node A node from the abstract syntax tree. * @return {boolean} true if this is a call to goog.require. */ function isRequireNode(node) { return isCallNode(node) && getIdentifierString(node.expression.callee) == 'goog.require'; } /** * @param {ASTNode} node A node from the abstract syntax tree. * @return {boolean} true if this is an exported symbol or property. */ function isExportNode(node) { const doc = getLeadingBlockComment(node); return doc && EXPORT_REGEX.test(doc); } /** * @param {ASTNode} node A node from the abstract syntax tree. * @return {boolean} true if this is a class assignment. */ function isClassAssignmentNode(node) { return node.type == 'ExpressionStatement' && node.expression.type == 'AssignmentExpression' && node.expression.right.type == 'ClassExpression'; } /** * @param {ASTNode} node A node from the abstract syntax tree. * @return {boolean} true if this is a class assignment with exported members. */ function isPartiallyExportedClassAssignmentNode(node) { if (!isClassAssignmentNode(node)) { return false; } const rightSide = node.expression.right; // Example code: foo.bar = class bar2 extends foo.baz { /* ... */ }; // Example right side: { // id: { name: 'bar' }, // or null // superClass: { type: 'MemberExpression', ... }, // or null // body: { body: [ ... ] }, // } for (const member of rightSide.body.body) { // Only look at exported members. Constructors are exported implicitly // when the class is exported. const comment = getLeadingBlockComment(member); if (EXPORT_REGEX.test(comment)) { return true; } } return false; } /** * @param {ASTNode} node A node from the abstract syntax tree. * @return {string} A reconstructed leading comment block for the node. * If there are multiple comments before this node, we will take the most * recent block comment, as that is the one that would contain any applicable * jsdoc/closure annotations for this symbol. */ function getLeadingBlockComment(node) { // Example code: /** @summary blah */ /** @export */ foo.bar = ...; // Example node: { // type: 'ExpressionStatement', // expression: { ... }, // leadingComments: [ // { type: 'Block', value: '* @summary blah ' }, // { type: 'Block', value: '* @export ' }, // ], // } if (!node.leadingComments || !node.leadingComments.length) { return null; } // Ignore non-block comments, since those are not jsdoc/Closure comments. const blockComments = node.leadingComments.filter((comment) => { return comment.type == 'Block'; }); if (!blockComments.length) { return null; } // In case there are multiple (for example, a file-level comment that also // preceeds the node), take the most recent one, which is closest to the node. const mostRecentComment = blockComments[blockComments.length - 1]; // Reconstruct the original block comment by adding back /* and */. return '/*' + mostRecentComment.value + '*/'; } /** * @param {number} idx An argument index from the call node. * @param {ASTNode} node A node from the abstract syntax tree. * @return {string} The argument value as a string. */ function getArgumentFromCallNode(idx, node) { // Example node: { // type: 'ExpressionStatement', // expression: { type: 'CallExpression', callee: {...}, arguments: [...] }, // } assert(isCallNode(node)); return node.expression.arguments[idx].value; } /** * @param {ASTNode} node An identifier or member node from the abstract syntax * tree. * @return {string} The identifier as a string. */ function getIdentifierString(node) { if (node.type == 'Identifier') { // Example code: foo // Example node: { type: 'Identifier', name: 'foo' } return node.name; } assert.equal(node.type, 'MemberExpression'); // Example code: foo.bar.baz // Example node: { // type: 'MemberExpression', // object: { // type: 'MemberExpression', // object: { type: 'Identifier', name: 'foo' }, // property: { type: 'Identifier', name: 'bar' }, // }, // property: { type: 'Identifier', name: 'baz' }, // } return getIdentifierString(node.object) + '.' + getIdentifierString(node.property); } /** * @param {ASTNode} node A function definition node from the abstract syntax * tree. * @return {!Array.<string>} a list of the parameter names. */ function getFunctionParameters(node) { assert(node.type == 'FunctionExpression' || node.type == 'ArrowFunctionExpression'); // Example code: function(x, y, z = null, ...varArgs) {...} // Example node: { // params: [ // { type: 'Identifier', name: 'x' }, // { type: 'Identifier', name: 'y' }, // { // type: 'AssignmentPattern', // left: { type: 'Identifier', name: 'z' }, // right: { type: 'Literal', raw: 'null' }, // }, // { // type: 'RestElement', // argument: { type: 'Identifier', name: 'varArgs' }, // }, // ], // body: {...}, // } return node.params.map((param) => { if (param.type == 'Identifier') { return param.name; } else if (param.type == 'AssignmentPattern') { return param.left.name; } else { assert.equal(param.type, 'RestElement'); return '...' + param.argument.name; } }); } /** * Take the original block comment and prep it for the externs by removing * export annotations and blank lines. * * @param {string} * @return {string} */ function removeExportAnnotationsFromComment(comment) { // Remove @export annotations. comment = comment.replace(EXPORT_REGEX, ''); // Split into lines, remove empty comment lines, then recombine. comment = comment.split('\n') .filter((line) => !/^ *\*? *$/.test(line)) .join('\n'); return comment; } /** * Recursively find all expression statements in all block nodes. * @param {ASTNode} node * @return {!Array.<ASTNode>} */ function getAllExpressionStatements(node) { assert(node.body && node.body.body); const expressionStatements = []; for (const childNode of node.body.body) { if (childNode.type == 'ExpressionStatement') { expressionStatements.push(childNode); } else if (childNode.body) { const childExpressions = getAllExpressionStatements(childNode); expressionStatements.push(...childExpressions); } } return expressionStatements; } /** * @param {!Set.<string>} names A set of the names of exported nodes. * @param {ASTNode} node An exported node from the abstract syntax tree. * @return {string} An extern string for this node. */ function createExternFromExportNode(names, node) { assert.equal(node.type, 'ExpressionStatement', 'Unknown node type'); let comment = getLeadingBlockComment(node); comment = removeExportAnnotationsFromComment(comment); let name; let assignment; switch (node.expression.type) { case 'AssignmentExpression': // Example code: /** @export */ foo.bar = function(...) { ... }; // Example node.expression: { // operator: '=', // left: { // type: 'MemberExpression', // object: { type: 'Identifier', name: 'foo' }, // property: { type: 'Identifier', name: 'bar' }, // }, right: { // type: 'FunctionExpression', params: [ ... ], body: {...} // } // } name = getIdentifierString(node.expression.left); assignment = createExternAssignment(name, node.expression.right, /* alwaysIncludeConstructor= */ true); break; case 'MemberExpression': // Example code: /** @export */ foo.bar; // Example node.expression: { // object: { type: 'Identifier', name: 'foo' }, // property: { type: 'Identifier', name: 'bar' }, // } name = getIdentifierString(node.expression); assignment = ''; break; default: assert.fail('Unexpected expression type: ' + node.expression.type); } // Keep track of the names we've externed. names.add(name); // Generate the actual extern string. let externString = comment + '\n' + name + assignment + ';\n'; // Find this.foo = bar in the constructor, and potentially generate externs // for that, too. if (node.expression.type == 'AssignmentExpression') { const rightSide = node.expression.right; if (rightSide.type == 'FunctionExpression' && comment.includes('@constructor')) { externString += createExternsFromConstructor(name, rightSide); } else if (rightSide.type == 'ClassExpression') { const ctor = getClassConstructor(node.expression.right); if (ctor) { externString += createExternsFromConstructor(name, ctor); } } } return externString; } /** * Some classes are not exported, but contain exported members. These need to * have externs generated, too. * * @param {!Set.<string>} names A set of the names of exported nodes. * @param {ASTNode} node An exported node from the abstract syntax tree. * @return {string} An extern string for this node. */ function createExternFromPartiallyExportedClassAssignmentNode(names, node) { assert.equal(node.type, 'ExpressionStatement', 'Unknown node type'); assert.equal(node.expression.type, 'AssignmentExpression', 'Should be assignment node'); assert.equal(node.expression.right.type, 'ClassExpression', 'Should be class assignment'); const name = getIdentifierString(node.expression.left); const assignment = createExternAssignment(name, node.expression.right, /* alwaysIncludeConstructor= */ false); let externString = name + assignment + ';\n'; // Find this.foo = bar in the constructor, and potentially generate externs // for that, too. const rightSide = node.expression.right; const ctor = getClassConstructor(node.expression.right); if (ctor) { externString += createExternsFromConstructor(name, ctor); } // Keep track of the names we've externed. names.add(name); return externString; } /** * @param {ASTNode} node A method node from the abstract syntax tree. * @return {string} The extern string for this method. */ function createExternMethod(node) { // Example code: foo.bar = class { // baz() { ... } // }; // Example node: { // leadingComments: [ ... ], // static: false, // key: Identifier, // value: FunctionExpression, // } const id = getIdentifierString(node.key); let comment = getLeadingBlockComment(node); if (!comment) { if (id == 'constructor') { // ES6 constructors don't necessarily need comments; a comment along the // lines of "Creates a Foo object." doesn't really add anything. comment = ''; } else { throw new Error('No leading block comment for: ' + id); } } comment = removeExportAnnotationsFromComment(comment); const params = getFunctionParameters(node.value); let methodString = (comment ? ' ' + comment + '\n' : '') + ' '; if (node.static) { methodString += 'static '; } methodString += id + '(' + params.join(', ') + ') {}'; return methodString; } /** * Find the constructor of an ES6 class, if it exists. * * @param {ASTNode} className * @return {ASTNode} */ function getClassConstructor(classNode) { // Example class node: { // type: 'ClassExpression', // body: { // type: 'ClassBody', // body: [ MethodDefinition, ... ], // } // } // // Example method node: { // type: 'MethodDefinition', // key: { type: 'Identifier', name: 'constructor' }, // value: { // type: 'FunctionExpression', // params: [ [Identifier], [Identifier], [Identifier] ], // body: { type: 'BlockStatement', body: [Array] }, // } assert.equal(classNode.type, 'ClassExpression'); for (const member of classNode.body.body) { if (member.type == 'MethodDefinition' && member.key.name == 'constructor') { return member.value; } } return null; } /** * @param {string} name The name of the thing we are assigning. * @param {ASTNode} node An assignment node from the abstract syntax tree. * @param {boolean} alwaysIncludeConstructor Include the constructor of a class * expression, even if there is no export annotation. * @return {string} The assignment part of the extern string for this node. */ function createExternAssignment(name, node, alwaysIncludeConstructor) { switch (node.type) { case 'ClassExpression': { // Example code: foo.bar = class bar2 extends foo.baz { /* ... */ }; // Example node: { // id: { name: 'bar' }, // or null // superClass: { type: 'MemberExpression', ... }, // or null // body: { body: [ ... ] }, // } let classString = ' = class '; if (node.id) { classString += getIdentifierString(node.id) + ' '; } if (node.superClass) { classString += 'extends ' + getIdentifierString(node.superClass) + ' '; } classString += '{\n'; for (const member of node.body.body) { const comment = getLeadingBlockComment(member); if (EXPORT_REGEX.test(comment)) { // This has an export annotation, so fall through and generate // externs. } else { // If there's no export annotation, we may make an exception for the // constructor in some situations. if (member.key.name == 'constructor' && alwaysIncludeConstructor) { // Fall through and generate externs. } else { // Skip extern generation. continue; } } assert.equal( member.type, 'MethodDefinition', 'Unexpected exported member type in exported class!'); classString += createExternMethod(member) + '\n'; } classString += '}'; return classString; } case 'ArrowFunctionExpression': case 'FunctionExpression': { // Example code: foo.square = function(x) { return x * x; }; // Example node: { params: [ { type: 'Identifier', name: 'x' } ] } const params = getFunctionParameters(node); return ' = function(' + params.join(', ') + ') {}'; } case 'ObjectExpression': { // Example code: foo.Bar = { 'ABC': 1, DEF: 2 }; // Example node: { // properties: [ { // kind: 'init', // key: { type: 'Literal', value: 'ABC' } // value: { type: 'Literal', value: 1 } // }, { // kind: 'init', // key: { type: 'Identifier', name: 'DEF' } // value: { type: 'Literal', value: 2 } // } ] // } const propertyStrings = node.properties.map((prop) => { assert.equal(prop.kind, 'init'); assert(prop.key.type == 'Literal' || prop.key.type == 'Identifier'); // Literal indicates a quoted name in the source, while Identifier is // an unquoted name. In the case of Literal, key.raw gets us the // unquoted name, we end up with an unquoted name in both cases. const name = prop.key.type == 'Literal' ? prop.key.raw : prop.key.name; assert.equal(prop.value.type, 'Literal'); return ' ' + name + ': ' + prop.value.raw; }); return ' = {\n' + propertyStrings.join(',\n') + '\n}'; } case 'Identifier': // Example code: /** @const {string} @export */ foo.version = VERSION; // Example extern: /** @const {string} */ foo.version; return ''; case 'Literal': // Example code: /** @const {string} @export */ foo.version = 'v1.0.0'; // Example extern: /** @const {string} */ foo.version; return ''; default: assert.fail('Unexpected export type: ' + node.type); return ''; // Shouldn't be hit, but linter wants a return statement. } } /** * Look for exports in a constructor body. If we don't do this, we may end up * with errors about classes not fully implementing their interfaces. In * reality, the interface is implemented by assigning members on "this". * * @param {string} className * @param {ASTNode} constructorNode * @return {string} */ function createExternsFromConstructor(className, constructorNode) { // Example code: // // /** @interface @exportInterface */ // FooLike = function() {}; // // /** @exportInterface @type {number} */ // FooLike.prototype.bar; // // /** @export @implements {FooLike} */ // class Foo { // constructor() { // /** @override @exportInterface */ // this.bar = 10; // } // }; // // Example externs: // // /** // * Generated by createExternFromExportNode: // * @implements {FooLike} // */ // class Foo { // constructor() {} // } // // /** // * Generated by createExternsFromConstructor: // * @override // */ // Foo.prototype.bar; const expressionStatements = getAllExpressionStatements(constructorNode); let externString = ''; for (const statement of expressionStatements) { const left = statement.expression.left; const right = statement.expression.right; // Skip anything that isn't an assignment to a member of "this". if (statement.expression.type != 'AssignmentExpression' || left.type != 'MemberExpression' || left.object.type != 'ThisExpression') { continue; } assert(left); assert(right); // Skip anything that isn't exported. let comment = getLeadingBlockComment(statement); if (!EXPORT_REGEX.test(comment)) { continue; } comment = removeExportAnnotationsFromComment(comment); assert.equal(left.property.type, 'Identifier'); const name = className + '.prototype.' + left.property.name; externString += comment + '\n' + name + ';\n'; } return externString; } /** * @param {!Set.<string>} names A set of the names of exported nodes. * @param {string} inputPath * @return {{ * path: string, * provides: !Array.<string>, * requires: !Array.<string>, * externs: string, * }} */ function generateExterns(names, inputPath) { // Load and parse the code, with comments attached to the nodes. const code = fs.readFileSync(inputPath, 'utf-8'); const program = esprima.parse(code, {attachComment: true}); assert.equal(program.type, 'Program'); const body = program.body; const provides = program.body.filter(isProvideNode) .map((node) => getArgumentFromCallNode(0, node)); const requires = program.body.filter(isRequireNode) .map((node) => getArgumentFromCallNode(0, node)); // Get all exported nodes and all classes, in order. const rawExterns = program.body.map((node) => { if (isExportNode(node)) { // Explicitly-exported nodes are handled here. return createExternFromExportNode(names, node); } else if (isPartiallyExportedClassAssignmentNode(node)) { // Some classes are not exported, but contain exported members. These // need to have externs generated, too. // But wait! The latest compiler won't actually export those correctly! // TODO: File a bug against the Closure Compiler. // In the mean time, log these now and throw an error at the end to make // sure we are generating usable releases. This tends to affect our // plugin registration APIs, and apps should definitely be able to use // those! if (!partiallyExportedClassesDetected) { partiallyExportedClassesDetected = true; console.log('The Closure Compiler does not handle partially-exported ' + 'classes correctly! The following classes need to be exported:'); } const name = getIdentifierString(node.expression.left); console.log(' * ' + name); return createExternFromPartiallyExportedClassAssignmentNode(names, node); } else { // Ignore anything else, and don't generate any externs. return ''; } }); const externs = rawExterns.join(''); return { path: inputPath, provides: provides, requires: requires, externs: externs, }; } /** * Generate externs from exported code. * Arguments: --output <EXTERNS> <INPUT> [<INPUT> ...] * * @param {!Array.<string>} args The args to this script, not counting node and * the script name itself. */ function main(args) { const inputPaths = []; let outputPath; for (let i = 0; i < args.length; ++i) { if (args[i] == '--output') { outputPath = args[i + 1]; ++i; } else { inputPaths.push(args[i]); } } assert(outputPath, 'You must specify output file with --output <EXTERNS>'); assert(inputPaths.length, 'You must specify at least one input file.'); // Generate externs for all input paths. const names = new Set(); const results = inputPaths.map((path) => generateExterns(names, path)); // TODO: revisit this when the compiler supports partially-exported classes. if (partiallyExportedClassesDetected) { throw new Error( 'Partially exported classes are not supported in the compiler!'); } // Sort them in dependency order. const sorted = topologicalSort(results, /* getDeps= */ (object) => { return object.requires.map((id) => { const dep = results.find((x) => x.provides.includes(id)); assert(dep, 'Cannot find dependency: ' + id); return dep; }); }); // Generate namespaces for all externs. For example, if we extern // foo.bar.baz, foo and foo.bar will both need to be declared first. const namespaces = new Set(); const namespaceDeclarations = []; for (const name of Array.from(names).sort()) { // Add the full name "foo.bar.baz" and its prototype ahead of time. We // should never generate these as namespaces. namespaces.add(name); namespaces.add(name + '.prototype'); // For name "foo.bar.baz", iterate over partialName "foo" and "foo.bar". const pieces = name.split('.'); for (let i = 1; i < pieces.length; ++i) { const partialName = pieces.slice(0, i).join('.'); if (!namespaces.has(partialName)) { let declaration; if (i == 1) { declaration = '/** @namespace */\n'; declaration += 'window.'; } else { declaration = '/** @const */\n'; } declaration += partialName + ' = {};\n'; namespaceDeclarations.push(declaration); namespaces.add(partialName); } } } // Get externs. const externs = sorted.map((x) => x.externs).join(''); // Get license header. const licenseHeader = fs.readFileSync(__dirname + '/license-header', 'utf-8'); // Output generated externs, with an appropriate header. fs.writeFileSync(outputPath, licenseHeader + '/**\n' + ' * @fileoverview Generated externs. DO NOT EDIT!\n' + ' * @externs\n' + ' * @suppress {duplicate} To prevent compiler errors with the\n' + ' * namespace being declared both here and by goog.provide in the\n' + ' * library.\n' + ' */\n\n' + namespaceDeclarations.join('') + '\n' + externs); } // Skip argv[0], which is the node binary, and argv[1], which is the script. main(process.argv.slice(2));