itutor-mathlive

<!DOCTYPE html> <html lang="en"> <head> <meta charset="utf-8"><title>core/lexer.js - MathLive Docs</title><meta name="description" content="Beautifully typeset math made easy"><meta name="keywords" content="latex, tex, math, typesetting, documentation, docs"> <meta name="viewport" content="width=device-width, initial-scale=1"> <script src="scripts/prettify/prettify.js"></script> <script src="scripts/prettify/lang-css.js"></script>  <link href="https://fonts.googleapis.com/css?family=Source+Code+Pro:400,700|Source+Sans+Pro:400,400i,700,900" rel="stylesheet"> <style>pre.prettyprint{background: #35434e;}</style> <link type="text/css" rel="stylesheet" href="styles/prettify-tomorrow-night.css"> <link type="text/css" rel="stylesheet" href="styles/jsdoc-default.css"> <link type="text/css" rel="stylesheet" href="styles/custom.css"> </head> <body> <div class="forkme"><a href="https://github.com/arnog/mathlive"><img style="position: absolute; top: 0; right: 0; border: 0; z-index:1;" src="https://camo.githubusercontent.com/52760788cde945287fbb584134c4cbc2bc36f904/68747470733a2f2f73332e616d617a6f6e6177732e636f6d2f6769746875622f726962626f6e732f666f726b6d655f72696768745f77686974655f6666666666662e706e67" alt="Fork me on GitHub" data-canonical-src="https://s3.amazonaws.com/github/ribbons/forkme_right_white_ffffff.png"></a></div> <section role="navigation"> <input type="checkbox" id="nav-trigger" class="nav-trigger" /> <label for="nav-trigger" class="navicon-button x"> <div class="navicon"></div> </label> <label for="nav-trigger" class="overlay"></label> <nav id="nav"> <h3 class="group-title home"><a href="index.html">MathLive Docs</a></h3><input class="search" placeholder="Search" type="text"><div class="list"><h3 class="group-title">Tutorials</h3><ul><li><a href="tutorial-CONTRIBUTOR_GUIDE.html">Contributor Guide</a></li><li><a href="tutorial-MASTON.html">MASTON</a></li><li><a href="tutorial-USAGE_GUIDE.html">Usage Guide</a></li></ul><h3 class="group-title">Classes</h3><ul><li class="private"><a href="Context.html" class="className">Context</a><ul class='methods private'><li data-type='method' class='private'><a href="Context.html#clone" class="methodName">clone</a></li><li data-type='method' class='private'><a href="Context.html#cloneWith" class="methodName">cloneWith</a></li><li data-type='method' class='private'><a href="Context.html#getBackgroundColor" class="methodName">getBackgroundColor</a></li><li data-type='method' class='private'><a href="Context.html#getColor" class="methodName">getColor</a></li><li data-type='method' class='private'><a href="Context.html#setMathstyle" class="methodName">setMathstyle</a></li><li data-type='method' class='private'><a href="Context.html#withMathstyle" class="methodName">withMathstyle</a></li></ul></li><li><a href="EditableMathlist.html" class="className">EditableMathlist</a><ul class='methods'><li data-type='method'><a href="EditableMathlist.html#_addCell" class="methodName">_addCell</a></li><li data-type='method'><a href="EditableMathlist.html#_deleteAtoms" class="methodName">_deleteAtoms</a></li><li data-type='method'><a href="EditableMathlist.html#addColumnAfter_" class="methodName">addColumnAfter_</a></li><li data-type='method'><a href="EditableMathlist.html#addColumnBefore_" class="methodName">addColumnBefore_</a></li><li data-type='method'><a href="EditableMathlist.html#addRowAfter_" class="methodName">addRowAfter_</a></li><li data-type='method'><a href="EditableMathlist.html#addRowBefore_" class="methodName">addRowBefore_</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#ancestor" class="methodName">ancestor</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#anchor" class="methodName">anchor</a></li><li data-type='method'><a href="EditableMathlist.html#applyStyle" class="methodName">applyStyle</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#commandOffsets" class="methodName">commandOffsets</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#commitCommandStringBeforeInsertionPoint" class="methodName">commitCommandStringBeforeInsertionPoint</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#contains" class="methodName">contains</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#decorateCommandStringAroundInsertionPoint" class="methodName">decorateCommandStringAroundInsertionPoint</a></li><li data-type='method'><a href="EditableMathlist.html#delete" class="methodName">delete</a></li><li data-type='method'><a href="EditableMathlist.html#delete_" class="methodName">delete_</a></li><li data-type='method'><a href="EditableMathlist.html#deleteAll_" class="methodName">deleteAll_</a></li><li data-type='method'><a href="EditableMathlist.html#deleteNextChar_" class="methodName">deleteNextChar_</a></li><li data-type='method'><a href="EditableMathlist.html#deleteNextWord_" class="methodName">deleteNextWord_</a></li><li data-type='method'><a href="EditableMathlist.html#deletePreviousChar_" class="methodName">deletePreviousChar_</a></li><li data-type='method'><a href="EditableMathlist.html#deletePreviousWord_" class="methodName">deletePreviousWord_</a></li><li data-type='method'><a href="EditableMathlist.html#deleteToGroupEnd_" class="methodName">deleteToGroupEnd_</a></li><li data-type='method'><a href="EditableMathlist.html#deleteToGroupStart_" class="methodName">deleteToGroupStart_</a></li><li data-type='method'><a href="EditableMathlist.html#deleteToMathFieldEnd_" class="methodName">deleteToMathFieldEnd_</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#endOffset" class="methodName">endOffset</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#extend" class="methodName">extend</a></li><li data-type='method'><a href="EditableMathlist.html#extendDown_" class="methodName">extendDown_</a></li><li data-type='method'><a href="EditableMathlist.html#extendToGroupEnd_" class="methodName">extendToGroupEnd_</a></li><li data-type='method'><a href="EditableMathlist.html#extendToGroupStart_" class="methodName">extendToGroupStart_</a></li><li data-type='method'><a href="EditableMathlist.html#extendToMathFieldEnd_" class="methodName">extendToMathFieldEnd_</a></li><li data-type='method'><a href="EditableMathlist.html#extendToMathFieldStart_" class="methodName">extendToMathFieldStart_</a></li><li data-type='method'><a href="EditableMathlist.html#extendToNextBoundary_" class="methodName">extendToNextBoundary_</a></li><li data-type='method'><a href="EditableMathlist.html#extendToNextChar_" class="methodName">extendToNextChar_</a></li><li data-type='method'><a href="EditableMathlist.html#extendToNextWord_" class="methodName">extendToNextWord_</a></li><li data-type='method'><a href="EditableMathlist.html#extendToPreviousBoundary_" class="methodName">extendToPreviousBoundary_</a></li><li data-type='method'><a href="EditableMathlist.html#extendToPreviousChar_" class="methodName">extendToPreviousChar_</a></li><li data-type='method'><a href="EditableMathlist.html#extendToPreviousWord_" class="methodName">extendToPreviousWord_</a></li><li data-type='method'><a href="EditableMathlist.html#extendUp_" class="methodName">extendUp_</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#extractCharactersBeforeInsertionPoint" class="methodName">extractCharactersBeforeInsertionPoint</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#extractCommandStringAroundInsertionPoint" class="methodName">extractCommandStringAroundInsertionPoint</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#extractContents" class="methodName">extractContents</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#extractContentsOrdInGroupBeforeInsertionPoint" class="methodName">extractContentsOrdInGroupBeforeInsertionPoint</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#filter" class="methodName">filter</a></li><li data-type='method'><a href="EditableMathlist.html#insert" class="methodName">insert</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#insertFirstAtom" class="methodName">insertFirstAtom</a></li><li data-type='method'><a href="EditableMathlist.html#isCollapsed" class="methodName">isCollapsed</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#jump" class="methodName">jump</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#leap" class="methodName">leap</a></li><li data-type='method'><a href="EditableMathlist.html#moveAfterParent_" class="methodName">moveAfterParent_</a></li><li data-type='method'><a href="EditableMathlist.html#moveBeforeParent_" class="methodName">moveBeforeParent_</a></li><li data-type='method'><a href="EditableMathlist.html#moveDown_" class="methodName">moveDown_</a></li><li data-type='method'><a href="EditableMathlist.html#moveToGroupEnd_" class="methodName">moveToGroupEnd_</a></li><li data-type='method'><a href="EditableMathlist.html#moveToGroupStart_" class="methodName">moveToGroupStart_</a></li><li data-type='method'><a href="EditableMathlist.html#moveToMathFieldEnd_" class="methodName">moveToMathFieldEnd_</a></li><li data-type='method'><a href="EditableMathlist.html#moveToMathFieldStart_" class="methodName">moveToMathFieldStart_</a></li><li data-type='method'><a href="EditableMathlist.html#moveToNextChar_" class="methodName">moveToNextChar_</a></li><li data-type='method'><a href="EditableMathlist.html#moveToNextPlaceholder_" class="methodName">moveToNextPlaceholder_</a></li><li data-type='method'><a href="EditableMathlist.html#moveToNextWord_" class="methodName">moveToNextWord_</a></li><li data-type='method'><a href="EditableMathlist.html#moveToOpposite_" class="methodName">moveToOpposite_</a></li><li data-type='method'><a href="EditableMathlist.html#moveToPreviousChar_" class="methodName">moveToPreviousChar_</a></li><li data-type='method'><a href="EditableMathlist.html#moveToPreviousPlaceholder_" class="methodName">moveToPreviousPlaceholder_</a></li><li data-type='method'><a href="EditableMathlist.html#moveToPreviousWord_" class="methodName">moveToPreviousWord_</a></li><li data-type='method'><a href="EditableMathlist.html#moveToSubscript_" class="methodName">moveToSubscript_</a></li><li data-type='method'><a href="EditableMathlist.html#moveToSuperscript_" class="methodName">moveToSuperscript_</a></li><li data-type='method'><a href="EditableMathlist.html#moveUp_" class="methodName">moveUp_</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#next" class="methodName">next</a></li><li data-type='method'><a href="EditableMathlist.html#selectAll_" class="methodName">selectAll_</a></li><li data-type='method'><a href="EditableMathlist.html#selectGroup_" class="methodName">selectGroup_</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#setExtent" class="methodName">setExtent</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#setRange" class="methodName">setRange</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#setSelection" class="methodName">setSelection</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#sibling" class="methodName">sibling</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#siblings" class="methodName">siblings</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#skip" class="methodName">skip</a></li><li data-type='method'><a href="EditableMathlist.html#speakAll_" class="methodName">speakAll_</a></li><li data-type='method'><a href="EditableMathlist.html#speakAllWithSynchronizedHighlighting_" class="methodName">speakAllWithSynchronizedHighlighting_</a></li><li data-type='method'><a href="EditableMathlist.html#speakGroup_" class="methodName">speakGroup_</a></li><li data-type='method'><a href="EditableMathlist.html#speakLeftSibling_" class="methodName">speakLeftSibling_</a></li><li data-type='method'><a href="EditableMathlist.html#speakParent_" class="methodName">speakParent_</a></li><li data-type='method'><a href="EditableMathlist.html#speakRightSibling_" class="methodName">speakRightSibling_</a></li><li data-type='method'><a href="EditableMathlist.html#speakSelection_" class="methodName">speakSelection_</a></li><li data-type='method'><a href="EditableMathlist.html#speakSelectionWithSynchronizedHighlighting_" class="methodName">speakSelectionWithSynchronizedHighlighting_</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#startOffset" class="methodName">startOffset</a></li><li data-type='method' class='private'><a href="EditableMathlist.html#toString" class="methodName">toString</a></li><li data-type='method'><a href="EditableMathlist.html#transpose_" class="methodName">transpose_</a></li></ul></li><li class="private"><a href="Lexer.html" class="className">Lexer</a><ul class='methods private'><li data-type='method' class='private'><a href="Lexer.html#end" class="methodName">end</a></li><li data-type='method' class='private'><a href="Lexer.html#get" class="methodName">get</a></li><li data-type='method' class='private'><a href="Lexer.html#isWhiteSpace" class="methodName">isWhiteSpace</a></li><li data-type='method' class='private'><a href="Lexer.html#makeToken" class="methodName">makeToken</a></li><li data-type='method' class='private'><a href="Lexer.html#peek" class="methodName">peek</a></li><li data-type='method' class='private'><a href="Lexer.html#scan" class="methodName">scan</a></li></ul></li><li class="private"><a href="MathAtom.html" class="className">MathAtom</a><ul class='methods private'><li data-type='method' class='private'><a href="MathAtom.html#.toSpeakableText" class="methodName">toSpeakableText</a></li><li data-type='method' class='private'><a href="MathAtom.html#bind" class="methodName">bind</a></li><li data-type='method' class='private'><a href="MathAtom.html#decompose" class="methodName">decompose</a></li><li data-type='method' class='private'><a href="MathAtom.html#decomposeGenfrac" class="methodName">decomposeGenfrac</a></li><li data-type='method' class='private'><a href="MathAtom.html#decomposeLeftright" class="methodName">decomposeLeftright</a></li><li data-type='method' class='private'><a href="MathAtom.html#decomposeLine" class="methodName">decomposeLine</a></li><li data-type='method' class='private'><a href="MathAtom.html#decomposeRule" class="methodName">decomposeRule</a></li><li data-type='method' class='private'><a href="MathAtom.html#filter" class="methodName">filter</a></li><li data-type='method' class='private'><a href="MathAtom.html#makeSpan" class="methodName">makeSpan</a></li></ul></li><li><a href="MathField.html" class="className">MathField</a><ul class='methods'><li data-type='method' class='private'><a href="MathField.html#_getCaretPosition" class="methodName">_getCaretPosition</a></li><li data-type='method' class='private'><a href="MathField.html#_onKeystroke" class="methodName">_onKeystroke</a></li><li data-type='method' class='private'><a href="MathField.html#complete_" class="methodName">complete_</a></li><li data-type='method'><a href="MathField.html#el" class="methodName">el</a></li><li data-type='method' class='private'><a href="MathField.html#enterCommandMode_" class="methodName">enterCommandMode_</a></li><li data-type='method'><a href="MathField.html#insert" class="methodName">insert</a></li><li data-type='method'><a href="MathField.html#keystroke" class="methodName">keystroke</a></li><li data-type='method'><a href="MathField.html#latex" class="methodName">latex</a></li><li data-type='method'><a href="MathField.html#perform" class="methodName">perform</a></li><li data-type='method'><a href="MathField.html#perform" class="methodName">perform</a></li><li data-type='method' class='private'><a href="MathField.html#render" class="methodName">render</a></li><li data-type='method'><a href="MathField.html#revertToOriginalContent" class="methodName">revertToOriginalContent</a></li><li data-type='method'><a href="MathField.html#selectedText" class="methodName">selectedText</a></li><li data-type='method'><a href="MathField.html#selectionAtEnd" class="methodName">selectionAtEnd</a></li><li data-type='method'><a href="MathField.html#selectionAtStart" class="methodName">selectionAtStart</a></li><li data-type='method'><a href="MathField.html#selectionDepth" class="methodName">selectionDepth</a></li><li data-type='method'><a href="MathField.html#selectionIsCollapsed" class="methodName">selectionIsCollapsed</a></li><li data-type='method'><a href="MathField.html#setConfig" class="methodName">setConfig</a></li><li data-type='method'><a href="MathField.html#text" class="methodName">text</a></li><li data-type='method'><a href="MathField.html#typedText" class="methodName">typedText</a></li></ul></li><li class="private"><a href="module-mathstyle.Mathstyle.html" class="className">Mathstyle</a></li><li class="private"><a href="Parser.html" class="className">Parser</a><ul class='methods private'><li data-type='method' class='private'><a href="Parser.html#end" class="methodName">end</a></li><li data-type='method' class='private'><a href="Parser.html#hasLiteral" class="methodName">hasLiteral</a></li><li data-type='method' class='private'><a href="Parser.html#hasLiteralPattern" class="methodName">hasLiteralPattern</a></li><li data-type='method' class='private'><a href="Parser.html#hasToken" class="methodName">hasToken</a></li><li data-type='method' class='private'><a href="Parser.html#lastMathAtom" class="methodName">lastMathAtom</a></li><li data-type='method' class='private'><a href="Parser.html#parseAtom" class="methodName">parseAtom</a></li><li data-type='method' class='private'><a href="Parser.html#parseKeyword" class="methodName">parseKeyword</a></li><li data-type='method' class='private'><a href="Parser.html#parseLimits" class="methodName">parseLimits</a></li><li data-type='method' class='private'><a href="Parser.html#parseSupSub" class="methodName">parseSupSub</a></li><li data-type='method' class='private'><a href="Parser.html#parseToken" class="methodName">parseToken</a></li><li data-type='method' class='private'><a href="Parser.html#scanArg" class="methodName">scanArg</a></li><li data-type='method' class='private'><a href="Parser.html#scanColor" class="methodName">scanColor</a></li><li data-type='method' class='private'><a href="Parser.html#scanDelim" class="methodName">scanDelim</a></li><li data-type='method' class='private'><a href="Parser.html#scanDimen" class="methodName">scanDimen</a></li><li data-type='method' class='private'><a href="Parser.html#scanEnvironment" class="methodName">scanEnvironment</a></li><li data-type='method' class='private'><a href="Parser.html#scanGroup" class="methodName">scanGroup</a></li><li data-type='method' class='private'><a href="Parser.html#scanImplicitGroup" class="methodName">scanImplicitGroup</a></li><li data-type='method' class='private'><a href="Parser.html#scanLeftRight" class="methodName">scanLeftRight</a></li><li data-type='method' class='private'><a href="Parser.html#scanModeSet" class="methodName">scanModeSet</a></li><li data-type='method' class='private'><a href="Parser.html#scanModeShift" class="methodName">scanModeShift</a></li><li data-type='method' class='private'><a href="Parser.html#scanNumber" class="methodName">scanNumber</a></li><li data-type='method' class='private'><a href="Parser.html#scanString" class="methodName">scanString</a></li><li data-type='method' class='private'><a href="Parser.html#scanToken" class="methodName">scanToken</a></li></ul></li><li class="private"><a href="Span.html" class="className">Span</a></li><li class="private"><a href="Token.html" class="className">Token</a></li><li class="private"><a href="UndoManager.html" class="className">UndoManager</a><ul class='methods private'><li data-type='method' class='private'><a href="UndoManager.html#canRedo" class="methodName">canRedo</a></li><li data-type='method' class='private'><a href="UndoManager.html#canUndo" class="methodName">canUndo</a></li><li data-type='method' class='private'><a href="UndoManager.html#redo" class="methodName">redo</a></li><li data-type='method' class='private'><a href="UndoManager.html#snapshot" class="methodName">snapshot</a></li><li data-type='method' class='private'><a href="UndoManager.html#undo" class="methodName">undo</a></li></ul></li></ul><h3 class="group-title">Modules</h3><ul><li class="private"><a href="module-color.html" class="className">color</a><ul class='methods private'><li data-type='method' class='private'><a href="module-color.html#.stringToColor" class="methodName">stringToColor</a></li></ul></li><li class="private"><a href="module-definitions.html" class="className">definitions</a><ul class='methods private'><li data-type='method' class='private'><a href="module-definitions.html#.defineEnvironment" class="methodName">defineEnvironment</a></li><li data-type='method' class='private'><a href="module-definitions.html#.defineFunction" class="methodName">defineFunction</a></li><li data-type='method' class='private'><a href="module-definitions.html#.defineSymbol" class="methodName">defineSymbol</a></li><li data-type='method' class='private'><a href="module-definitions.html#.defineSymbolRange" class="methodName">defineSymbolRange</a></li><li data-type='method' class='private'><a href="module-definitions.html#.defineSymbols" class="methodName">defineSymbols</a></li><li data-type='method' class='private'><a href="module-definitions.html#.frequency" class="methodName">frequency</a></li><li data-type='method' class='private'><a href="module-definitions.html#.getInfo" class="methodName">getInfo</a></li><li data-type='method' class='private'><a href="module-definitions.html#.matchCodepoint" class="methodName">matchCodepoint</a></li><li data-type='method' class='private'><a href="module-definitions.html#.matchFunction" class="methodName">matchFunction</a></li><li data-type='method' class='private'><a href="module-definitions.html#.parseParamTemplateArgument" class="methodName">parseParamTemplateArgument</a></li><li data-type='method' class='private'><a href="module-definitions.html#.suggest" class="methodName">suggest</a></li></ul></li><li class="private"><a href="module-delimiters.html" class="className">delimiters</a><ul class='methods private'><li data-type='method' class='private'><a href="module-delimiters.html#.makeCustomSizedDelim" class="methodName">makeCustomSizedDelim</a></li><li data-type='method' class='private'><a href="module-delimiters.html#.makeInner" class="methodName">makeInner</a></li><li data-type='method' class='private'><a href="module-delimiters.html#.makeLargeDelim" class="methodName">makeLargeDelim</a></li><li data-type='method' class='private'><a href="module-delimiters.html#.makeLeftRightDelim" class="methodName">makeLeftRightDelim</a></li><li data-type='method' class='private'><a href="module-delimiters.html#.makeNullFence" class="methodName">makeNullFence</a></li><li data-type='method' class='private'><a href="module-delimiters.html#.makeSizedDelim" class="methodName">makeSizedDelim</a></li><li data-type='method' class='private'><a href="module-delimiters.html#.makeSmallDelim" class="methodName">makeSmallDelim</a></li><li data-type='method' class='private'><a href="module-delimiters.html#.makeStackedDelim" class="methodName">makeStackedDelim</a></li><li data-type='method' class='private'><a href="module-delimiters.html#.traverseSequence" class="methodName">traverseSequence</a></li></ul></li><li class="private"><a href="module-editor_editableMathlist.html" class="className">editor/editableMathlist</a><ul class='methods private'><li data-type='method' class='private'><a href="module-editor_editableMathlist.html#~atomContains" class="methodName">atomContains</a></li></ul></li><li class="private"><a href="module-editor_keyboard.html" class="className">editor/keyboard</a><ul class='methods private'><li data-type='method' class='private'><a href="module-editor_keyboard.html#.delegateKeyboardEvents" class="methodName">delegateKeyboardEvents</a></li><li data-type='method' class='private'><a href="module-editor_keyboard.html#.keyboardEventToString" class="methodName">keyboardEventToString</a></li></ul></li><li class="private"><a href="module-editor_mathfield.html" class="className">editor/mathfield</a><ul class='methods private'><li data-type='method' class='private'><a href="module-editor_mathfield.html#nearestElementFromPoint" class="methodName">nearestElementFromPoint</a></li></ul></li><li class="private"><a href="module-editor_mathpath.html" class="className">editor/mathpath</a><ul class='methods private'><li data-type='method' class='private'><a href="module-editor_mathpath.html#.pathCommonAncestor" class="methodName">pathCommonAncestor</a></li><li data-type='method' class='private'><a href="module-editor_mathpath.html#.pathDistance" class="methodName">pathDistance</a></li><li data-type='method' class='private'><a href="module-editor_mathpath.html#.pathFromString" class="methodName">pathFromString</a></li><li data-type='method' class='private'><a href="module-editor_mathpath.html#.pathToString" class="methodName">pathToString</a></li></ul></li><li class="private"><a href="module-editor_shortcuts.html" class="className">editor/shortcuts</a><ul class='methods private'><li data-type='method' class='private'><a href="module-editor_shortcuts.html#.match" class="methodName">match</a></li><li data-type='method' class='private'><a href="module-editor_shortcuts.html#.matchKeystroke" class="methodName">matchKeystroke</a></li><li data-type='method' class='private'><a href="module-editor_shortcuts.html#.platform" class="methodName">platform</a></li><li data-type='method' class='private'><a href="module-editor_shortcuts.html#.stringify" class="methodName">stringify</a></li></ul></li><li class="private"><a href="module-fontMetrics.html" class="className">fontMetrics</a><ul class='methods private'><li data-type='method' class='private'><a href="module-fontMetrics.html#.getCharacterMetrics" class="methodName">getCharacterMetrics</a></li></ul></li><li class="private"><a href="module-lexer.html" class="className">lexer</a><ul class='methods private'><li data-type='method' class='private'><a href="module-lexer.html#.tokenize" class="methodName">tokenize</a></li></ul></li><li class="private"><a href="module-mathAtom.html" class="className">mathAtom</a><ul class='methods private'><li data-type='method' class='private'><a href="module-mathAtom.html#.decompose" class="methodName">decompose</a></li><li data-type='method' class='private'><a href="module-mathAtom.html#.getFontName" class="methodName">getFontName</a></li><li data-type='method' class='private'><a href="module-mathAtom.html#.makeColGap" class="methodName">makeColGap</a></li><li data-type='method' class='private'><a href="module-mathAtom.html#.makeColOfRepeatingElements" class="methodName">makeColOfRepeatingElements</a></li><li data-type='method' class='private'><a href="module-mathAtom.html#.makeStack" class="methodName">makeStack</a></li></ul></li><li><a href="module-mathlive.html" class="className">mathlive</a><ul class='methods'><li data-type='method'><a href="module-mathlive.html#latexToAST" class="methodName">latexToAST</a></li><li data-type='method'><a href="module-mathlive.html#latexToMarkup" class="methodName">latexToMarkup</a></li><li data-type='method'><a href="module-mathlive.html#latexToMathML" class="methodName">latexToMathML</a></li><li data-type='method'><a href="module-mathlive.html#latexToSpeakableText" class="methodName">latexToSpeakableText</a></li><li data-type='method'><a href="module-mathlive.html#makeMathField" class="methodName">makeMathField</a></li><li data-type='method'><a href="module-mathlive.html#pauseReadAloud" class="methodName">pauseReadAloud</a></li><li data-type='method'><a href="module-mathlive.html#playReadAloud" class="methodName">playReadAloud</a></li><li data-type='method'><a href="module-mathlive.html#readAloud" class="methodName">readAloud</a></li><li data-type='method'><a href="module-mathlive.html#readAloudStatus" class="methodName">readAloudStatus</a></li><li data-type='method'><a href="module-mathlive.html#renderMathInDocument" class="methodName">renderMathInDocument</a></li><li data-type='method'><a href="module-mathlive.html#renderMathInElement" class="methodName">renderMathInElement</a></li><li data-type='method'><a href="module-mathlive.html#resumeReadAloud" class="methodName">resumeReadAloud</a></li><li data-type='method'><a href="module-mathlive.html#revertToOriginalContent" class="methodName">revertToOriginalContent</a></li><li data-type='method'><a href="module-mathlive.html#revertToOriginalContent" class="methodName">revertToOriginalContent</a></li></ul></li><li class="private"><a href="module-mathstyle.html" class="className">mathstyle</a><ul class='methods private'><li data-type='method' class='private'><a href="module-mathstyle.html#.toMathstyle" class="methodName">toMathstyle</a></li></ul></li><li class="private"><a href="module-span.html" class="className">span</a><ul class='methods private'><li data-type='method' class='private'><a href="module-span.html#.coalesce" class="methodName">coalesce</a></li><li data-type='method' class='private'><a href="module-span.html#.makeFontSizer" class="methodName">makeFontSizer</a></li><li data-type='method' class='private'><a href="module-span.html#.makeHlist" class="methodName">makeHlist</a></li><li data-type='method' class='private'><a href="module-span.html#.makeSpan" class="methodName">makeSpan</a></li><li data-type='method' class='private'><a href="module-span.html#.makeSpanOfType" class="methodName">makeSpanOfType</a></li><li data-type='method' class='private'><a href="module-span.html#.makeSymbol" class="methodName">makeSymbol</a></li><li data-type='method' class='private'><a href="module-span.html#.makeVlist" class="methodName">makeVlist</a></li><li data-type='method' class='private'><a href="module-span.html#.toString" class="methodName">toString</a></li></ul></li></ul><label class="checkbox"><input id="toggle-private" type="checkbox" onclick=" if (!document.getElementById('toggle-private').checked) { document.documentElement.classList.add('no-private') writeCookie('symbol-access', 'no-private') } else { document.documentElement.classList.remove('no-private') writeCookie('symbol-access', 'private') } ">Include Private Symbols</input></label></div> <footer role="contentInfo"> Made with <a href="https://github.com/jsdoc3/jsdoc">JSDoc 3.5.5</a>. </footer> </nav> </section> <main id="main"> <h1 class="page-title">core/lexer.js</h1> <section> <article> <pre class="prettyprint source linenums"> <code>/*global require:false*/ /*global define:false*/ /** * @module lexer * @private */ /** * ## Reference * TeX source code: * {@link http://tug.org/texlive/devsrc/Build/source/texk/web2c/tex.web|Tex.web} * * For a list of standard TeX macros, see: * {@link ftp://tug.ctan.org/pub/tex-archive/systems/knuth/dist/lib/plain.tex|plain.tex} */ define(['mathlive/core/grapheme-splitter'], function(GraphemeSplitter) { /** * * A token can be of type: * - `literal`: the value is the character this token represents. This can be * a combination of Unicode codepoints, for example for emojis. * - `^` and `_`: superscript and subscript commands. * - command: a command such as \sin * - `{` and `}`: begin and end group (use for arguments of commands and for grouping) * - `#`: parameter * * - `esc`: start of a special command. Followed by commandliteral tokens. * - `backslash`: start of a special command. Followed by commandliteral tokens. * - `commandliteral`: a-zA-Z for special commands (esc sequence, etc...) * - `placeholder`: a placeholder value meant to be replaced by some actual value * - `space`: one or more space characters (including tab, etc...) * * See: [TeX:289](http://tug.org/texlive/devsrc/Build/source/texk/web2c/tex.web) * @property {string} value * @property {string} type * @class Token * @global * @private */ function Token(type, value) { this.type = type; this.value = value; console.assert(!(type === 'literal' && value === '}')); } /** * @param {string} s * @class Lexer * @global * @private */ function Lexer(s) { this.s = GraphemeSplitter.splitGraphemes(s); this.pos = 0; } /** * @return {boolean} True if we reached the end of the stream * @method Lexer#end * @private */ Lexer.prototype.end = function() { return this.pos >= this.s.length; } /** * Return the next char and advance * @return {string} * @method Lexer#get */ Lexer.prototype.get = function() { return this.pos < this.s.length ? this.s[this.pos++] : null; } /** * Return the next char, but do not advance * @return {string} * @method Lexer#peek * @private */ Lexer.prototype.peek = function() { return this.pos < this.s.length ? this.s[this.pos] : null; } /** * Return the next substring matching regEx and advance. * @param {RegEx} regEx * @return {string | null} * @method Lexer#scan * @private */ Lexer.prototype.scan = function(regEx) { let result; // this.s can either be a string, if it's made up only of ASCII chars // or an array of graphemes, if it's more complicated. if (typeof this.s === 'string') { result = regEx.exec(this.s.slice(this.pos)); } else { result = regEx.exec(this.s.slice(this.pos).join('')); } if (result) { this.pos += result[0].length; return result[0]; } return null; } /** * Return true if next char is white space. Does not advance. * Note that browsers are inconsistent in their definitions of the * `\s` metacharacter, so use an explicit string match instead. * * - Chrome: `[ \t\n\v\f\r\u00A0]` * - Firefox: `[ \t\n\v\f\r\u00A0\u2028\u2029]` * - IE: `[ \t\n\v\f\r]` * * See [Stackoverflow](http://stackoverflow.com/questions/6073637/) * @method Lexer#isWhiteSpace * @private */ Lexer.prototype.isWhiteSpace = function() { return ' \f\n\r\t\v\u00A0\u2028\u2029'.indexOf(this.s[this.pos]) !== -1; /* - \t \u0009: tab (CHARACTER TABULATION) - \n \u000A: linefeed (LINE FEED) - \v \u000B: vertical tab (LINE TABULATION) - \f \u000C: form feed (FORM FEED) - \r \u000D: carriage return - \u00A0: NON-BREAKING SPACE - \u2028: LINE SEPARATOR - \u2029: PARAGRAPH SEPARATOR Could be considered: - \u1680 OGHAM SPACE MARK - \u2000-\u200a spacing - \u202f NARROW NO-BREAK SPACE - \u205F MEDIDUM MATHEMATICAL SPACE - \u3000 IDEOGRAPHIC SPACE - \uFEFF ZERO WITH NON-BREAKING SPACE */ } /*** * Advance until non-white-space char. * Returns number of chars skipped. * @method Lexer#skipWhiteSpace * @private */ Lexer.prototype.skipWhiteSpace = function() { const savedPos = this.pos; while (!this.end() && this.isWhiteSpace()) { this.get(); } return this.pos - savedPos; } /** * Return a single token, or null, created from the lexer. * * @returns {Token} * @method Lexer#makeToken * @private */ Lexer.prototype.makeToken = function() { // If we've reached the end, exit if (this.end()) return null; // Skip white space if (this.skipWhiteSpace() > 0) return new Token('space'); let result = null; // #region iTutor modification // Is it a variable? let variable = this.scan(/^\\?##.*?##/); if (variable) { // remove wrapped ##..## variable = variable.replace(/^\\?##/, '').replace(/##/, ''); return new Token('variable', variable); } // It is an expression? let expression = this.scan(/^\\?<<.*?>>/) if (expression) { // remove wrapped <<..>> expression = expression.replace(/^\\?<</, '').replace(/>>/, ''); return new Token('expression', expression); } // #endRegion // Is it a command? if (this.peek() === '\\') { this.get(); // Skip the initial \ if (!this.end()) { // A command is either a string of letters and asterisks... let command = this.scan(/^[a-zA-Z*]+/); if (!command) { // ... or a single non-letter character command = this.get(); } // There are a few special commands that are handled here... if (command === 'bgroup') { // Begin group, synonym for opening brace result = new Token('{'); } else if (command === 'egroup') { // End group, synonym for closing brace result = new Token('}'); } else { result = new Token('command', command); } } // Is it a group start/end? } else if (this.peek() === '{' || this.peek() === '}') { result = new Token(this.get()); } else if (this.peek() === '#') { // This could be either a param token, or a literal # (used for // colorspecs, for example). A param token is a '#' followed by // - a digit 0-9 followed by a non-alpha, non-digit // - or '?'. // Otherwise, it's a literal '#'. this.get(); if (!this.end()) { let isParam = false; let next = this.peek(); if (/[0-9?]/.test(next)) { // Could be a param isParam = true; // Need to look ahead to the following char if (this.pos + 1 < this.s.length) { const after = this.s[this.pos + 1]; isParam = /[^0-9A-Za-z]/.test(after); } } if (isParam) { result = new Token('#'); next = this.get(); if (next >= '0' && next <= '9') { result.value = parseInt(next); } else { result.value = '?'; } } else { result = new Token('literal', '#'); } } // result = new Token(this.get()); // if (!this.end()) { // const next = this.get(); // if (next >= '0' && next <= '9') { // result.value = parseInt(next); // } else { // result.value = next; // } // } } else if (this.peek() === '^') { result = new Token(this.get()); } else if (this.peek() === '_') { result = new Token(this.get()); } else if (this.peek() === '~') { // Spacing this.get(); result = new Token('command', 'space'); // Is it ESCAPE } else if (this.peek() === '\u001b') { // ESCAPE character result = new Token('esc', this.get()); // Is it a mode switch? } else if (this.peek() === '$') { this.get(); if (this.peek() === '$') { // $$ this.get(); result = new Token('$$'); } else { // $ result = new Token('$'); } } else { result = new Token('literal', this.get()); } return result; } /** * Create Tokens from a stream of LaTeX * * @param {string} s - A string o LaTeX. It can include comments (with the `%` * marker) and multiple lines. * @return {Token[]} * @memberof module:lexer * @private */ function tokenize(s) { const result = []; const lines = s.toString().split(/\r?\n/); let stream = ''; for (const line of lines) { // Remove everything after a % (comment marker) // (but \% should be preserved...) // @todo there's probably a better way of doing this using s.split(regex) let previousChar = ''; for (let i = 0; i < line.length; i++) { const c = line.charAt(i); if (c === '%' && previousChar !== '\\') { break; } stream += c; previousChar = c; } } const lex = new Lexer(stream); while (!lex.end()) { const token = lex.makeToken(); if (token) result.push(token); } return result; } return { tokenize } }) </code></pre> </article> </section> </main> <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script> <script src="scripts/namis.search.jquery.js"></script> <script> function readCookie(a) { var b = document.cookie.match('(^|;)\\s*' + a + '\\s*=\\s*([^;]+)'); return b ? b.pop() : ''; } function writeCookie(a, v) { var date = new Date(); date.setTime(date.getTime()+(30*24*60*60*1000)); document.cookie = a + "=" + v + "; expires=" + date.toUTCString() + "; path=/"; } var cookie = readCookie('symbol-access'); if (cookie === 'no-private') { document.documentElement.classList.add('no-private'); } document.getElementById('toggle-private').checked = !document.documentElement.classList.contains('no-private'); prettyPrint(); (function($){ $('#nav').search(['className', 'methodName']); })(jQuery); </script> <script src="scripts/linenumber.js"></script> </body> </html>