syntax-cli-prog
Version:
Syntactic analysis toolkit, language agnostic parsers generator.
465 lines (356 loc) • 14.1 kB
JavaScript
;
var _fs = require('fs');
var _fs2 = _interopRequireDefault(_fs);
var _colors = require('colors');
var _colors2 = _interopRequireDefault(_colors);
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
/**
* Rust tokenizer template.
*/
/**
* The MIT License (MIT)
* Copyright (c) 2015-present Dmitry Soshnikov <dmitry.soshnikov@gmail.com>
*/
var RUST_TOKENIZER_TEMPLATE = _fs2.default.readFileSync(__dirname + '/templates/tokenizer.template.rs', 'utf-8');
/**
* Default error handler for rust parser when encountered an error.
*/
var DEFAULT_ERROR_HANDLER = '\n if token.value == EOF && !self.tokenizer.has_more_tokens() {\n panic!("Unexpected end of input.");\n }\n self.tokenizer.panic_unexpected_token(token.value, token.start_line, token.start_column);\n';
/**
* The trait is used by parser generators (LL/LR) for Rust.
*/
var RustParserGeneratorTrait = {
/**
* Generates parsing table in Rust vector format.
*/
generateParseTable: function generateParseTable() {
this.writeData('TABLE', this._buildTable(this.generateParseTableData()));
},
/**
* Converts JS object into Rust HashMap.
*
* In Rust we represent a table as a vector, where index is a state number,
* and a value is a hashmap of LR entries (shift/reduce/etc).
*
* Example:
*
* vec![
* hashmap! { 1 => TE::Shift(4), 3 => TE::Reduce(1), ... },
* ]
*/
_buildTable: function _buildTable(table) {
var _this = this;
var entries = Object.keys(table).map(function (state) {
var row = table[state];
// Transform to Rust enum format: "s3" => TE::Shift(3), etc
Object.keys(row).forEach(function (key) {
var entry = row[key];
if (entry[0] === 's') {
row[key] = 'TE::Shift(' + entry.slice(1) + ')';
} else if (entry[0] === 'r') {
row[key] = 'TE::Reduce(' + entry.slice(1) + ')';
} else if (entry === 'acc') {
row[key] = 'TE::Accept';
} else {
row[key] = 'TE::Transit(' + entry + ')';
}
});
return _this._toRustHashMap(table[state], 'number');
});
return 'vec![\n ' + entries.join(',\n ') + '\n]';
},
/**
* Generates tokens table in Rust hashmap format.
*/
generateTokensTable: function generateTokensTable() {
this.writeData('TOKENS', this._toRustHashMap(this._tokens, 'string', 'number'));
},
/**
* Production handlers are implemented as methods on the parser class.
*/
buildSemanticAction: function buildSemanticAction(production) {
var originalAction = this.getSemanticActionCode(production);
var _extractDataTypes = this._extractDataTypes(originalAction),
action = _extractDataTypes.action,
types = _extractDataTypes.types;
action = this._actionFromHandler(action, '.tokenizer');
action = this._generateArgsPrologue(action, types,
// Total number of args.
production.isEpsilon() ? 0 : production.getRHS().length);
// Append return value.
var returnValue = types.hasOwnProperty('__') ? 'SV::_' + this._allTypes[types.__] + '(__)' : '__';
action = action + ('\n' + returnValue);
// Save the action, they are injected later.
this._productionHandlers.push({ args: '&mut self', action: action });
return null;
},
/**
* Builds SV (stack value) enum from all the used types in handlers.
*/
generateStackValueEnum: function generateStackValueEnum() {
var svEnum = Object.keys(this._allTypes).map(function (typeName, idx) {
return '_' + idx + '(' + typeName + ')';
});
this.writeData('SV_ENUM', svEnum.join(',\n '));
},
/**
* Generates prologue for fetching arguments from the parsing stack.
*/
_generateArgsPrologue: function _generateArgsPrologue(action, types, totalArgsCount) {
var argsPrologue = [];
for (var i = totalArgsCount; i > 0; i--) {
var arg = '_' + i;
if (!types.hasOwnProperty(arg)) {
// Just pop if arg is not used in the handler.
argsPrologue.push('self.values_stack.pop();');
} else {
var typeInfo = types[arg];
if (typeInfo) {
argsPrologue.push('let mut ' + arg + ' = pop!(self.values_stack, ' + ('_' + this._allTypes[typeInfo] + ');'));
} else {
argsPrologue.push('let mut ' + arg + ' = self.values_stack.pop().unwrap();');
}
}
}
return '// Semantic values prologue.\n' + argsPrologue.join('\n') + '\n\n' + action;
},
/**
* Extracts types of the used arguments, and return types.
* If a type is defined, a popped stack value is casted,
* otherwise, it's just popped (if doesn't participate in an operation
* and just propagated).
*/
_extractDataTypes: function _extractDataTypes(action) {
var _this2 = this;
var types = {};
if (!action) {
return { action: '', types: types };
}
var typesRe = /\s*\|([^|]*)\|\s*->\s*(\w+);/g;
var typesData = typesRe.exec(action);
if (typesData) {
// '$1:i32, $3:i32'
if (typesData[1].trim().length > 0) {
var argTypes = typesData[1].trim().split(/\s*[,;]\s*/);
argTypes.forEach(function (argData) {
var data = argData.split(/\s*:\s*/);
// types[$1] = 'i32';
types[data[0]] = data[1];
// Save the type to all types data to generate enum.
if (!_this2._allTypes.hasOwnProperty(data[1])) {
_this2._allTypes[data[1]] = _this2._allTypesIndex++;
}
});
}
// Result type.
if (typesData[2]) {
types.__ = typesData[2];
if (!this._allTypes.hasOwnProperty(typesData[2])) {
this._allTypes[typesData[2]] = this._allTypesIndex++;
}
}
}
// Strip the types info.
action = action.replace(typesRe, '');
// Extract other args, which do not use types.
var argsRe = /_\d+/g;
var usedArgs = action.match(argsRe);
if (usedArgs) {
usedArgs.forEach(function (usedArg) {
if (!types.hasOwnProperty(usedArg)) {
// Arg is used, but without a type.
types[usedArg] = null;
}
});
}
return { types: types, action: action };
},
/**
* Default format in the [ ] array notation.
*/
generateProductionsData: function generateProductionsData() {
return this.generateRawProductionsData().map(function (data) {
// Remove the semantic action handler, since in Rust
// we use a different structure to hold it.
delete data[2];
return '[' + data.join(', ') + ']';
});
},
/**
* Generates built-in tokenizer instance.
*/
generateBuiltInTokenizer: function generateBuiltInTokenizer() {
this.writeData('TOKENIZER', RUST_TOKENIZER_TEMPLATE);
},
/**
* Creates an action from raw handler.
*/
_actionFromHandler: function _actionFromHandler(handler) {
var context = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : '';
var action = (this._scopeVars(handler, context) || '').trim();
if (!action) {
return 'let __ = SV::Undefined;';
}
// From parser hooks, append ; at the end.
if (context === '.tokenizer' && !/;\s*$/.test(action)) {
action += ';';
}
return action;
},
/**
* Generates rules for tokenizer.
*/
generateLexRules: function generateLexRules() {
var _this3 = this;
var lexRulesArray = [];
var lexRules = this._grammar.getLexGrammar().getRules().map(function (rule, i) {
var action = _this3._actionFromHandler(rule.getRawHandler());
_this3._lexHandlers.push({ args: '&mut self', action: action });
var flags = [];
if (rule.isCaseInsensitive()) {
flags.push('i');
}
if (flags.length > 0) {
flags = '(?' + flags.join('') + ')';
} else {
flags = '';
}
lexRulesArray.push('Tokenizer::_lex_rule' + i);
var matcher = rule.getRawMatcher();
// there is no need for escape sequence for character '/'
// (actually you cannot have such escape sequence as '\/', whihc causes rust's regex parser to panic!)
matcher = matcher.replace("\\/", "/");
// well, maybe so many # is enough
// this is for handling character '"' correctly
// reference: rust's raw string literals https://rahul-thakoor.github.io/rust-raw-string-literals
return 'r##########"' + flags + matcher + '"##########';
});
this.writeData('LEX_RULE_HANDLERS_COUNT', lexRules.length);
this.writeData('LEX_RULE_HANDLERS_ARRAY', '[\n ' + lexRulesArray.join(',\n ') + '\n],');
this.writeData('LEX_RULES', '[&\'static str; ' + lexRules.length + '] = ' + ('[\n ' + lexRules.join(',\n ') + '\n]'));
},
generateLexRulesByStartConditions: function generateLexRulesByStartConditions() {
var lexGrammar = this._grammar.getLexGrammar();
var lexRulesByConditions = lexGrammar.getRulesByStartConditions();
var result = [];
for (var condition in lexRulesByConditions) {
result[condition] = lexRulesByConditions[condition].map(function (lexRule) {
return lexGrammar.getRuleIndex(lexRule);
});
}
this.writeData('LEX_RULES_BY_START_CONDITIONS', '' + this._toRustHashMap(result, 'string'));
},
/**
* Replaces global vars like `yytext`, `$$`, etc. to be
* referred from `yyparse`.
*/
_scopeVars: function _scopeVars(code) {
var context = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : '';
return code.replace(/yytext/g, 'self' + context + '.yytext').replace(/yyleng/g, 'self' + context + '.yyleng').replace(/__\s*=/g, 'let __ =').replace(/yyloc/g, 'Loc::from_tokens_range');
},
/**
* Type-converts a key of a Rust HashMap, e.g. string or number, etc.
*/
_hashKey: function _hashKey(key, keyType) {
switch (keyType) {
case 'string':
return '"' + key + '"';
case 'number':
return Number(key);
default:
throw new Error('_hashKey: Incorrect type ' + keyType);
}
},
/**
* Type-converts a value of a Rust hashmap, e.g. string or number, etc.
*/
_hashValue: function _hashValue(value, valueType) {
if (Array.isArray(value)) {
// Support only int arrays here for simplicity.
return 'vec! [ ' + value.join(', ') + ' ]';
}
switch (valueType) {
case 'string':
return '"' + value + '"';
case 'number':
return Number(value);
default:
return value;
}
},
/**
* Converts JS object to Rust HashMap representation.
*/
_toRustHashMap: function _toRustHashMap(object, keyType, valueType) {
var result = [];
for (var k in object) {
var value = object[k];
var key = k.replace(/"/g, '\\"');
result.push(this._hashKey(key, keyType) + ' => ' + ('' + this._hashValue(value, valueType)));
}
return 'hashmap! { ' + result.join(', ') + ' }';
},
/**
* Rust-specific lex rules handler declarations.
*/
generateLexHandlers: function generateLexHandlers() {
var handlers = this._generateHandlers(this._lexHandlers, '_lex_rule', "&'static str");
this.writeData('LEX_RULE_HANDLERS', handlers.join('\n\n'));
},
/**
* Rust-specific handler declarations.
*/
generateProductionHandlers: function generateProductionHandlers() {
var handlers = this._generateHandlers(this._productionHandlers, '_handler', 'SV');
this.writeData('PRODUCTION_HANDLERS_COUNT', handlers.length);
var handlersArray = [];
for (var i = 0; i < handlers.length; i++) {
handlersArray.push('Parser::_handler' + i);
}
this.writeData('PRODUCTION_HANDLERS_ARRAY', '[\n ' + handlersArray.join(',\n ') + '\n],');
this.writeData('PRODUCTION_HANDLERS', handlers.join('\n\n'));
},
/**
* Productions array in Rust format.
*/
generateProductions: function generateProductions() {
var productionsData = this.generateProductionsData();
var productionsCount = productionsData.length;
this.writeData('PRODUCTIONS', '[[i32; 2]; ' + productionsCount + '] = ' + ('[\n ' + productionsData.join(',\n ') + '\n]'));
},
/**
* Module include.
*/
generateModuleInclude: function generateModuleInclude() {
var moduleInclude = this._grammar.getModuleInclude();
var resultTypeData = /type\s+TResult\s*=\s*([^;]+);/.exec(moduleInclude);
if (!resultTypeData) {
throw new Error('\n\nRust plugin should provide module include, and define at least ' + ('result type:\n\n ' + _colors2.default.bold('type TResult = <...>;\n')));
}
// Result type.
var resultType = resultTypeData[1];
if (!this._allTypes.hasOwnProperty(resultType)) {
throw new Error('Result type ' + _colors2.default.bold(resultType) + ' is not found in ' + 'handled types. Make sure your productions return it.\n');
}
this.writeData('RESULT_TYPE', '_' + this._allTypes[resultType]);
// Parser hooks.
var onParseBegin = moduleInclude.indexOf('fn on_parse_begin') !== -1 ? 'on_parse_begin(self, string);' : '';
var onParseEnd = moduleInclude.indexOf('fn on_parse_end') !== -1 ? 'on_parse_end(self, &result);' : '';
var onParseError = moduleInclude.indexOf('fn on_parse_error') !== -1 ? 'on_parse_error(self, &token);' : DEFAULT_ERROR_HANDLER;
this.writeData('ON_PARSE_BEGIN_CALL', onParseBegin);
this.writeData('ON_PARSE_END_CALL', onParseEnd);
this.writeData('ON_PARSE_ERROR_CALL', onParseError);
this.writeData('MODULE_INCLUDE', moduleInclude);
},
/**
* Generates Rust function declarations for handlers.
*/
_generateHandlers: function _generateHandlers(handlers, name, returnType) {
return handlers.map(function (_ref, index) {
var args = _ref.args,
action = _ref.action;
return 'fn ' + name + index + ('(' + args + ') -> ' + returnType + ' {\n' + action + '\n}');
});
}
};
module.exports = RustParserGeneratorTrait;