eggtended-js

<!DOCTYPE html> <html> <head> <meta charset="utf-8"> <base data-ice="baseUrl" href="../../../"> <title data-ice="title">lib/parser/lexer.js | eggtended-js</title> <link type="text/css" rel="stylesheet" href="css/style.css"> <link type="text/css" rel="stylesheet" href="css/prettify-tomorrow.css"> <script src="script/prettify/prettify.js"></script> <script src="script/manual.js"></script> <meta name="description" content="An extended version of the Egg programming language"><meta property="og:type" content="website"><meta property="og:url" content="http://my-library.org"><meta property="og:site_name" content="eggtended-js"><meta property="og:title" content="eggtended-js"><meta property="og:image" content="https://i.imgur.com/9N4qe98.png"><meta property="og:description" content="An extended version of the Egg programming language"><meta property="og:author" content="https://github.com/Dibad"><meta property="twitter:card" content="summary"><meta property="twitter:title" content="eggtended-js"><meta property="twitter:description" content="An extended version of the Egg programming language"><meta property="twitter:image" content="https://i.imgur.com/9N4qe98.png"></head> <body class="layout-container" data-ice="rootContainer"> <header> <a href="./">Home</a> <a href="identifiers.html">Reference</a> <a href="source.html">Source</a> <a href="test.html" data-ice="testLink">Test</a> <div class="search-box"> <img src="./image/search.png"> <input class="search-input"> <ul class="search-result"></ul> </div> <a style="position:relative; top:3px;" href="https://github.com/Dibad/eggtended-js"><img width="20px" src="./image/github.png"></a></header> <nav class="navigation" data-ice="nav"><div> <ul> <li data-ice="doc">F<a href="function/index.html#static-function-checkIterable">checkIterable</a></li> <li data-ice="doc">F<a href="function/index.html#static-function-chunk">chunk</a></li> <li data-ice="doc">F<a href="function/index.html#static-function-getValidIndex">getValidIndex</a></li> <li data-ice="doc">F<a href="function/index.html#static-function-ins">ins</a></li> <li data-ice="doc"><a data-ice="dirPath" class="nav-dir-path" href="identifiers.html#interp">interp</a>C<a href="class/lib/interp/ast.js~Apply.html">Apply</a></li> <li data-ice="doc">C<a href="class/lib/interp/ast.js~Regex.html">Regex</a></li> <li data-ice="doc">C<a href="class/lib/interp/ast.js~Value.html">Value</a></li> <li data-ice="doc">C<a href="class/lib/interp/ast.js~Word.html">Word</a></li> <li data-ice="doc">C<a href="class/lib/interp/eggvm.js~Eggvm.html">Eggvm</a></li> <li data-ice="doc">C<a href="class/lib/interp/symboltable.js~SymbolTable.html">SymbolTable</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-SpecialForms">SpecialForms</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-TopEnv">TopEnv</a></li> <li data-ice="doc"><a data-ice="dirPath" class="nav-dir-path" href="identifiers.html#parser">parser</a>C<a href="class/lib/parser/lexer.js~Lexer.html">Lexer</a></li> <li data-ice="doc">C<a href="class/lib/parser/parse.js~Parser.html">Parser</a></li> <li data-ice="doc">C<a href="class/lib/parser/tokenRegex.js~TokenRegex.html">TokenRegex</a></li> <li data-ice="doc">F<a href="function/index.html#static-function-json2AST">json2AST</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-COMMA">COMMA</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-LP">LP</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-NEWLINE">NEWLINE</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-NUMBER">NUMBER</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-REGEX">REGEX</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-RP">RP</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-STRING">STRING</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-WHITES">WHITES</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-WORD">WORD</a></li> <li data-ice="doc"><a data-ice="dirPath" class="nav-dir-path" href="identifiers.html#repl">repl</a>F<a href="function/index.html#static-function-eggCompleter">eggCompleter</a></li> <li data-ice="doc">F<a href="function/index.html#static-function-getPromptLine">getPromptLine</a></li> <li data-ice="doc">F<a href="function/index.html#static-function-repl">repl</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-BLUE">BLUE</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-DEFAULT">DEFAULT</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-GREEN">GREEN</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-RED">RED</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-YELLOW">YELLOW</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-eggClear">eggClear</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-eggExit">eggExit</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-eggHelp">eggHelp</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-eggInfo">eggInfo</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-eggRun">eggRun</a></li> <li data-ice="doc">V<a href="variable/index.html#static-variable-eggVersion">eggVersion</a></li> </ul> </div> </nav> <div class="content" data-ice="content"><h1 data-ice="title">lib/parser/lexer.js</h1> <pre class="source-code line-number raw-source-code"><code class="prettyprint linenums" data-ice="content">const tokenRegex = require("./tokenRegex.js"); const fs = require("fs"); class Lexer { constructor() { this.lastIndex = 0; this.line = 1; } setProgram(program) { this.program = program; this.lastIndex = 0; this.line = 1; return this; } tokenize(program) { this.setProgram(program); return this._getTokens(); } static tokenize(program) { return new Lexer().tokenize(program); } tokenizeFromFile(file) { const program = fs.readFileSync(file, "utf-8"); return this.tokenize(program); } static tokenizeFromFile(file) { return new Lexer().tokenizeFromFile(file); } static setTokens(tokens) { Lexer.TOKENS = tokens; } getParBalance(program) { this.setProgram(program); let stack = 0; const tokens = this._getTokens(); for (const token of tokens) { if (token.type === "LP") { ++stack; } else if (token.type === "RP") { --stack; } } return stack; } _getTokens() { const tokens = []; let currentToken = this._getToken(); while (currentToken !== null) { tokens.push(currentToken); currentToken = this._getToken(); } return this.__transformTokens(tokens); } _getToken() { // Update lastIndex property for each expression used this.__updateLastIndices(); // Match and ignore whitespaces and newlines let whitespaces = Lexer.WHITES.exec(this.program); if (whitespaces !== null) { if (Lexer.NEWLINE.exec(whitespaces.value)) { ++this.line; } this.lastIndex = Lexer.WHITES.lastIndex; return this._getToken(); } // Iterate through each regex let token = null; for (let i = 0; i < Lexer.TOKENS.length; ++i) { let match = Lexer.TOKENS[i].exec(this.program); // When matching a expression, add the current line and set the token if (match !== null) { match.line = this.line; this.lastIndex = Lexer.TOKENS[i].lastIndex; token = match; break; } } // Return the matched token, or null if anything was found return token; } __transformTokens(tokens) { for (let i = 0; i < tokens.length; ++i) { // x: => "x", if (tokens[i].type === "WORD") { const nextToken = tokens[i + 1]; if (nextToken && nextToken.value === ":") { tokens[i].type = "STRING"; } } // Replace dots with parentehsis // a.b => a("b") // a.b.c => a("b")("c") // a.b(c, d) => a("b", c, d) if (tokens[i].type === "LP" && tokens[i].value === ".") { tokens[i].value = "("; const expr = tokens[i + 1]; const arg = tokens[i + 2]; if (expr && expr.type === "WORD") { expr.type = "STRING"; } if (arg && arg.type === "LP" && arg.value !== ".") { arg.type = "COMMA"; arg.value = ","; } else { tokens.splice(i + 2, 0, { type: "RP", value: ")" }); } } } return tokens; } __updateLastIndices() { Lexer.TOKENS.forEach(expr => { expr.lastIndex = this.lastIndex; }); Lexer.WHITES.lastIndex = this.lastIndex; Lexer.NEWLINE.lastIndex = this.lastIndex; } } Lexer.TOKENS = [ tokenRegex.NUMBER, tokenRegex.STRING, tokenRegex.REGEX, tokenRegex.WORD, tokenRegex.LP, tokenRegex.RP, tokenRegex.COMMA ]; Lexer.WHITES = tokenRegex.WHITES; Lexer.NEWLINE = tokenRegex.NEWLINE; module.exports = { Lexer }; </code></pre> </div> <footer class="footer"> Generated by <a href="https://esdoc.org">ESDoc(1.1.0)<img src="./image/esdoc-logo-mini-black.png"></a> </footer> <script src="script/search_index.js"></script> <script src="script/search.js"></script> <script src="script/pretty-print.js"></script> <script src="script/inherited-summary.js"></script> <script src="script/test-summary.js"></script> <script src="script/inner-link.js"></script> <script src="script/patch-for-local.js"></script> </body> </html>