imba/src/compiler/lexer.mjs

Intuitive and powerful language for building webapps that fly

import * as T from './token.mjs';
import { INVERSES as INVERSES } from './constants.mjs';
import { Compilation } from './compilation.mjs';
import * as ERR from './errors.mjs';
import * as helpers from './helpers.mjs';
function len$(a){
	return a && (a.len instanceof Function ? a.len() : a.length) || 0;
};
function iter$(a){ return a ? (a.toArray ? a.toArray() : a) : []; };
function map$(list){
	var map = Object.create(null);
	for (let i = 0, items = iter$(list), len = items.length; i < len; i++) {
		map[items[i]] = 1;
	};
	return map;
};
// helper for subclassing
function subclass$(obj,sup) {
	for (var k in sup) {
		if (sup.hasOwnProperty(k)) obj[k] = sup[k];
	};
	// obj.__super__ = sup;
	obj.prototype = Object.create(sup.prototype);
	obj.__super__ = obj.prototype.__super__ = sup.prototype;
	obj.prototype.initialize = obj.prototype.constructor = obj;
};

// imba$inlineHelpers=1
// imba$v2=0

var Token = T.Token;

var K = 0;

// Constants
// ---------

// Keywords that Imba shares in common with JavaScript.
var JS_KEYWORDS = [
	'true','false','null','this',
	'delete','typeof','in','instanceof',
	'throw','break','continue','debugger',
	'if','else','switch','for','while','do','try','catch','finally',
	'class','extends','super','return','yield'
];

var TSC_CARET_BEFORE = {
	',': 1,
	'\n': 1,
	')': 1,
	']': 1,
	'}': 1,
	'>': 1,
	' ': 1
};

var TYPE_GENERICS_AFTER = /\w|\]|\)$/;

// new can be used as a keyword in imba, since object initing is done through
// MyObject.new. new is a very useful varname.

// We want to treat return like any regular call for now
// Must be careful to throw the exceptions in AST, since the parser
// wont

// Imba-only keywords. var should move to JS_Keywords
// some words (like tokid) should be context-specific
var IMBA_KEYWORDS = [
	'undefined','then','unless','until','loop','of','by',
	'when','def','tag','do','elif','begin','var','let','const','await','import','module','export','static','extend','yield'
];

var IMBA_CONTEXTUAL_KEYWORDS = ['extend','local','global','prop'];

var CONTEXTUAL_KEYWORDS = {
	LET: {'global': 1,'declare': 1,static: 1},
	CONST: {'global': 1,'declare': 1,static: 1},
	VAR: {'global': 1,'declare': 1,static: 1},
	CLASS: {'global': 1,'declare': 1,static: 1,extend: 1,strict: 1,abstract: 1,mixin: 1},
	MIXIN: {'global': 1,'declare': 1,extend: 1,strict: 1,abstract: 1},
	INTERFACE: {'global': 1,'declare': 1,extend: 1,strict: 1,mixin: 1},
	TAG: {'global': 1,'declare': 1,'local': 1,extend: 1,strict: 1,abstract: 1},
	DEF: {'global': 1,'declare': 1,protected: 1},
	PROP: {'static': 1},
	ATTR: {'static': 1},
	CSS: {'global': 1,export: 1}
};

var MEMBER_KEYWORDS = {
	DEF: 1,
	PROP: 1,
	ATTR: 1
};

// FixedArray for performance
// var ALL_KEYWORDS = JS_KEYWORDS.concat(IMBA_KEYWORDS)
var ALL_KEYWORDS = [
	'true','false','null','this','self',
	'delete','typeof','in','instanceof',
	'throw','break','continue','debugger',
	'if','else','switch','for','while','do','try','catch','finally',
	'class','extends','super','return',
	'undefined','then','unless','until','loop','of','by',
	'when','def','tag','do','elif','begin','var','let','const','await','import',
	'and','or','is','isnt','not','yes','no','isa','case','nil','module','export','static','extend',
	'yield'
];
var ALL_KEYWORDS_MAP = map$(ALL_KEYWORDS);
var KEYWORD_CANDIDATE_MAP = map$(ALL_KEYWORDS.concat(['mixin','guard','alter','watch','css','interface','attr','prop','get','set','constructor','declare']));
var SPECIAL_IDENTIFIER_MAP = map$(['default','type','from','as','new','arguments']);

// The list of keywords that are reserved by JavaScript, but not used, or are
// used by Imba internally. We throw an error when these are encountered,
// to avoid having a JavaScript error at runtime.  # 'var', 'let', - not inside here
var RESERVED = ['case','default','function','void','with','const','enum','native'];
var STRICT_RESERVED = ['case','function','void','const'];

// The superset of both JavaScript keywords and reserved words, none of which may
// be used as identifiers or properties.
var JS_FORBIDDEN = JS_KEYWORDS.concat(RESERVED);

var METHOD_IDENTIFIER = /^((([\x23]?[\$A-Za-z_\x7f-\uffff][$\-\w\x7f-\uffff]*)([\=]?))|(<=>|\|(?![\|=])))/;
// removed ~=|~| |&(?![&=])

// Token matching regexes.
// added hyphens to identifiers now - to test
var IDENTIFIER = /^((\$|##|#|@@|@|\%)[\$\wA-Za-z_\-\x7f-\uffff][$\w\x7f-\uffff]*(\-[$\w\x7f-\uffff]+)*[\?]?|[$A-Za-z_][$\w\x7f-\uffff]*(\-[$\w\x7f-\uffff]+)*[\?]?)([^\n\S]*:)?/;

var IMPORTS = /^import\s+(\{?[^\"\'\}]+\}?)(?=\s+from\s+)/;

var OBJECT_KEY = /^((\$?)[$A-Za-z_\x7f-\uffff\-][$\w\x7f-\uffff\-]*)([^\n\S\s]*:(?![\*\=:$A-Za-z\_\x7f-\uffff]))/;

var TAG = /^(\<)(?=[A-Za-z\#\.\%\$\[\{\@\>\(])/;

var TAG_TYPE = /^(\w[\w\d]*:)?(\w[\w\d]*)(-[\w\d]+)*/;
var TAG_ID = /^#((\w[\w\d]*)(-[\w\d]+)*)/;
var TAG_PART = /^[\:\.\#]?([A-Za-z\_][\w\-]*)(\:[A-Za-z\_][\w\-]*)?/;
var TAG_ATTR = /^([\.\:]?[\w\_]+([\-\:\.][\w]+)*)(\s)*\=(?!\>)/;

var SELECTOR = /^([%\$]{1,2})([\(])/;
var SELECTOR_PART = /^(\#|\.|:|::)?([\w]+(\-[\w]+)*)/;
var SELECTOR_COMBINATOR = /^ (\+|\>|\~)*\s*(?=[\w\.\#\:\{\*\[])/;

var SELECTOR_PSEUDO_CLASS = /^(::?)([\w]+(\-[\w]+)*)/;
var SELECTOR_ATTR_OP = /^(\$=|\~=|\^=|\*=|\|=|=|\!=)/;
var SELECTOR_ATTR = /^\[([\w\_\-]+)(\$=|\~=|\^=|\*=|\|=|=|\!=)/;

var SYMBOL = /^\:((([\*\@$\w\x7f-\uffff]+)+([\-\\\:][\w\x7f-\uffff]+)*)|==|\<=\>)/;

var STYLE_HEX = /^\#[\w\-]+/;

var STYLE_NUMERIC = /^(\-?\d*\.?\d+)([A-Za-z]+|\%)?(?![\d\w])/;

var STYLE_IDENTIFIER = /^[\w\-\$]*\w[\w\-\$]*/;

var STYLE_URL = /^url\(([^\)]*)\)/;

var STYLE_PROPERTY = /^(\^?[\w\-\$\@\.\!\#\^]+)(?=\:([^\:]|$)|\s*\=)/;

var STYLE_MODIFIERS = /^(\@?[\w\-\$]*\w[\w\-\$]*)([\.\@][\w\-\$]*\w[\w\-\$]*)*(\@[\w\-\$]*\w[\w\-\$]*)*(?=\:)/;

var isStyleWordCode = function(code) {
	return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122) || code == 95;
};

var isStylePropertyCode = function(code) {
	return isStyleWordCode(code) || code == 45 || code == 36 || code == 64 || code == 46 || code == 33 || code == 35 || code == 94;
};

var isStyleSpaceCode = function(code) {
	return (code >= 9 && code <= 13) || code == 32 || code == 160 || code == 5760 || code == 65279 || code == 8232 || code == 8233 || code == 8239 || code == 8287 || code == 12288 || (code >= 8192 && code <= 8202);
};

var stylePropertyLengthAt = function(str,start) {
	if(start === undefined) start = 0;
	var i = start;
	var len = str.length;
	while (i < len && isStylePropertyCode(str.charCodeAt(i))) {
		i++;
	};
	if (i == start) { return 0 };
	if (str.charCodeAt(i) == 58 && str.charCodeAt(i + 1) != 58) {
		return i - start;
	};
	var j = i;
	while (j < len && isStyleSpaceCode(str.charCodeAt(j))) {
		j++;
	};
	return str.charCodeAt(j) == 61 ? i - start : 0;
};

var canStartStyleSelector = function(str,chr) {
	var code = chr.charCodeAt(0);
	if (isStyleWordCode(code) || code == 94 || code == 37 || code == 42 || code == 38 || code == 36 || code == 62 || code == 47 || code == 46 || code == 91 || code == 64 || code == 33) {
		return true;
	};
	if (code == 35) {
		let next = str.charCodeAt(1);
		return isStyleWordCode(next) || next == 45;
	};
	return code == 58 && str.charCodeAt(1) == 58;
};

var NUMBER = /^0x[\da-f_]+|^0b[01_]+|^0o[\d_]+|^[\-]?(?:\d[_\d]*)\.?\d[_\d]*(?:e[+-]?\d+)?|^[\-]?\d*\.?\d+(?:e[+-]?\d+)?/i;

var HEREDOC = /^("""|''')([\s\S]*?)(?:\n[^\n\S]*)?\1/;

var OPERATOR = /^(?:[-=]=>|!&(?=[\s\n])|[&|~^]?=\?|[&|~^]=|\?\?=|===|---|->|=>|\/>|!==|\*\*=?|[-+*\/%<>&|^!?=]=|=<|>>>=?|([-+:])\1|([&|<>])\2=?|\?\.|\?\?|\.{2,3}|\*(?=[a-zA-Z\_]))/;

// FIXME splat should only be allowed when the previous thing is spaced or inside call?

var WHITESPACE = /^[^\n\S]+/;

var COMMENT = /^###([^#][\s\S]*?)(?:###[^\n\S]*|(?:###)?$)/;
var JS_COMMENT = /^\/\*([\s\S]*?)\*\//;
// var INLINE_COMMENT = /^(\s*)((#[ \t\!]||\/\/)(.*)|#[ \t]?(?=\n|$))+/
var INLINE_COMMENT = /^(\s*)((#[ \t\!]|\/\/(?!\/))(.*)|#[ \t]?(?=\n|$))+/;

var CODE = /^[-=]=>/;

var MULTI_DENT = /^(?:\n[^\n\S]*)+/;

var SIMPLESTR = /^'[^\\']*(?:\\.[^\\']*)*'/;

var JSTOKEN = /^`[^\\`]*(?:\\.[^\\`]*)*`/;

// Regex-matching-regexes.
var REGEX = /^(\/(?![\s=])[^[\/\n\\]*(?:(?:\\[\s\S]|\[[^\]\n\\]*(?:\\[\s\S][^\]\n\\]*)*])[^[\/\n\\]*)*\/)([a-z]{0,8})(?!\w)/;

var HEREGEX = /^\/{3}([\s\S]+?)\/{3}([a-z]{0,8})(?!\w)/;

var HEREGEX_OMIT = /\s+(?:#.*)?/g;

// Token cleaning regexes.
var MULTILINER = /\n/g;

var HEREDOC_INDENT = /\n+([^\n\S]*)/g;

var HEREDOC_ILLEGAL = /\*\//;

// expensive?
var LINE_CONTINUER = /^\s*(?:,|\??\.(?![.\d])|(?:&&|\|\||and|or)[\n\s])/;

var TRAILING_SPACES = /\s+$/;

var ENV_FLAG = /^\$\w+\$/;

var ARGVAR = /^\$\d$/;

// Compound assignment tokens.
var COMPOUND_ASSIGN = [
	'-=','+=','/=','*=','%=','||=','&&=','?=','??=',
	'<<=','>>=','>>>=','&=','^=','|=','~=','=<','**=',
	'=?','~=?','|=?','&=?','^=?'
];
var COMPOUND_ASSIGN_MAP = map$(COMPOUND_ASSIGN);

// Unary tokens.
var UNARY = ['!','~','NEW','TYPEOF','DELETE'];
var UNARY_MAP = map$(UNARY);

// Logical tokens.
var LOGIC = ['&&','||','??','and','or'];
var LOGIC_MAP = map$(LOGIC);

// Bit-shifting tokens.
var SHIFT = ['<<','>>','>>>'];
var SHIFT_MAP = map$(SHIFT);

// Comparison tokens.
var COMPARE = ['===','!==','==','!=','<','>','<=','>=','===','!==','&','|','^','!&'];
var COMPARE_MAP = map$(COMPARE);

// Mathematical tokens.
var MATH = ['*','/','%','∪','∩','√'];
var MATH_MAP = map$(MATH);

// Relational tokens that are negatable with `not` prefix.
var RELATION = ['IN','OF','INSTANCEOF','ISA'];
var RELATION_MAP = map$(RELATION);

// Boolean tokens.
var BOOL = ['TRUE','FALSE','NULL','UNDEFINED'];

// Our list is shorter, due to sans-parentheses method calls.
var NOT_REGEX = ['NUMBER','REGEX','BOOL','TRUE','FALSE','++','--',']'];
var NOT_REGEX_MAP = map$(NOT_REGEX);

// If the previous token is not spaced, there are more preceding tokens that
// force a division parse:
var NOT_SPACED_REGEX = ['NUMBER','REGEX','BOOL','TRUE','FALSE','++','--',']',')','}','THIS','SELF','IDENTIFIER','STRING'];
var NOT_SPACED_REGEX_MAP = map$(NOT_SPACED_REGEX);

// Tokens which could legitimately be invoked or indexed. An opening
// parentheses or bracket following these tokens will be recorded as the start
// of a function invocation or indexing operation.
// really?!

var UNFINISHED = ['\\','.','UNARY','MATH','EXP','+','-','SHIFT','RELATION','COMPARE','THROW','EXTENDS'];
var UNFINISHED_MAP = map$(UNFINISHED);

// } should not be callable anymore!!! '}', '::',
var CALLABLE = ['IDENTIFIER','SYMBOLID','STRING','REGEX',')',']','INDEX_END','THIS','SUPER','TAG_END','IVAR','SELF','NEW','ARGVAR','SYMBOL','RETURN','INDEX_END','CALL_END','DECORATOR','@','GENERICS'];
var CALLABLE_MAP = map$(CALLABLE);

// optimize for FixedArray
var INDEXABLE = [
	'IDENTIFIER','SYMBOLID','STRING','REGEX',')',']','THIS','SUPER','TAG_END','IVAR','SELF','NEW','ARGVAR','SYMBOL','RETURN','BANG',
	'NUMBER','BOOL','TAG_SELECTOR','ARGUMENTS','}','TAG_TYPE','TAG_REF','INDEX_END','CALL_END','DO_VALUE'
];
var INDEXABLE_MAP = map$(INDEXABLE);

var NOT_KEY_AFTER = ['.','?','?.','UNARY','??','+','-','*'];

var GLOBAL_IDENTIFIERS = ['global','exports'];

// Tokens that, when immediately preceding a `WHEN`, indicate that the `WHEN`
// occurs at the start of a line. We disambiguate these from trailing whens to
// avoid an ambiguity in the grammar.
var LINE_BREAK = ['INDENT','OUTDENT','TERMINATOR'];
var LINE_BREAK_MAP = map$(LINE_BREAK);

var DECLARE_START_MAP = map$(['INDENT','TERMINATOR','DECORATOR']);
var TAG_DECLARATION_START_MAP = map$(['INDENT','OUTDENT','TERMINATOR','EXPORT','DEFAULT','DECLARE','GLOBAL','LOCAL','EXTEND','ABSTRACT','STRICT','DECORATOR']);
var TAG_DECLARATION_PREFIX_MAP = map$(['global','declare','local','extend','abstract','strict']);
var PROPERTY_ACCESS_PREV_MAP = map$(['IDENTIFIER',')','}',']','NUMBER']);
var SYMBOL_STRING_PREV_MAP = map$(['(','[','=']);
var SPLAT_PREV_VALUE_MAP = map$([',','(','[','{','|','\n','\t']);
var AMPER_REF_TIGHT_TOKENS = map$(['COMPARE','.','(','[']);

function LexerError(message,file,line){
	this.message = message;
	this.file = file;
	this.line = line;
	return this;
};
subclass$(LexerError,SyntaxError);



var last = function(array,back) {
	if(back === undefined) back = 0;
	return array[array.length - back - 1];
};

var countLineBreaks = function(str) {
	var count = 0;
	for (let i = 0, len = str.length; i < len; i++) {
		if (str.charCodeAt(i) == 10) count++;
	};
	return count;
};

var repeatString = function(str,times) {
	var res = '';
	while (times > 0){
		if (times % 2 == 1) {
			res += str;
		};
		str += str;
		times >>= 1;
	};
	return res;
};

var tT = T.typ;
var tV = T.val;
var tTs = T.setTyp;
var tVs = T.setVal;

// The Lexer class reads a stream of Imba and divvies it up into tokidged
// tokens. Some potential ambiguity in the grammar has been avoided by
// pushing some extra smarts into the Lexer.

// Based on the original lexer.coffee from CoffeeScript
function Lexer(){
	this.reset();
	this;
};


Lexer.prototype.reset = function (){
	this._code = null;
	this._chunk = null; // The remainder of the source code.
	this._opts = null;
	this._state = {};
	
	this._indent = 0; // The current indentation level.
	this._indebt = 0; // The over-indentation at the current level.
	this._outdebt = 0; // The under-outdentation at the current level.
	
	this._indents = []; // The stack of all current indentation levels.
	this._ends = []; // The stack for pairing up tokens.
	this._contexts = []; // suplements @ends
	this._scopes = [];
	this._nextScope = null; // the scope to add on the next indent
	this._context = null;
	// should rather make it like a statemachine that moves from CLASS_DEF to CLASS_BODY etc
	// Things should compile differently when you are in a CLASS_BODY than when in a DEF_BODY++
	
	this._indentStyle = '\t';
	this._inTag = false;
	this._inStyle = 0;
	
	this._tokens = []; // Stream of parsed tokens in the form `['TYPE', value, line]`.
	this._seenFor = false;
	this._loc = 0;
	this._locOffset = 0;
	
	this._end = null;
	this._char = null;
	this._bridge = null;
	
	this._last = null;
	this._lastTyp = '';
	this._lastVal = null;
	this._script = null;
	return this;
};

Lexer.prototype.jisonBridge = function (jison){
	return this._bridge = {
		lex: T.lex,
		setInput: function(tokens) {
			this.tokens = tokens;
			return this.pos = 0;
		},
		
		upcomingInput: function() { return ""; }
	};
};

Lexer.prototype.tokenize = function (code,o,script){
	
	var m;
	if(script === undefined) script = null;
	if (code.length == 0) {
		return [];
	};
	
	if (!o.inline) {
		if (WHITESPACE.test(code)) {
			code = ("\n" + code);
			if (code.match(/^\s*$/g)) { return [] };
		};
		
		if (code.indexOf('\r') >= 0) {
			code = code.replace(/\r/g,'');
		};
		let lastCode = code.charCodeAt(code.length - 1);
		if (lastCode == 9 || lastCode == 32) {
			code = code.replace(/[\t ]+$/g,'');
		};
	};
	
	this._last = null;
	this._lastTyp = null;
	this._lastVal = null;
	
	this._script = script;
	this._code = code;
	this._opts = o;
	this._locOffset = o.loc || 0;
	this._platform = o.platform || o.target;
	this._indentStyle = '\t';
	
	// if the very first line is indented, take this as a gutter
	if (m = code.match(/^([\ \t]*)[^\n\s\t]/)) {
		this._state.gutter = m[1];
	};
	
	if (o.gutter !== undefined) {
		this._state.gutter = o.gutter;
	};
	
	if (this._script && !o.inline) {
		this._script.tokens = this._tokens;
	};
	
	this.parse(code);
	if (!o.inline) this.closeIndentation();
	
	
	
	if (this._ends.length) {
		this.unclosedError("end of file");
	};
	
	if (this._platform == 'tsc') {
		for (let i = 0, items = iter$(this._tokens), len = items.length, token; i < len; i++) {
			token = items[i];
			if (token._type == 'SYMBOLID') {
				token._type = 'IDENTIFIER';
				// token.@value = token.@value.replace(/#/g,'_$SYM$_')
			};
		};
	};
	
	return this._tokens;
};

Lexer.prototype.parse = function (code){
	var i = 0;
	var pi = 0;
	
	this._loc = this._locOffset + i;
	
	while (this._chunk = code.slice(i)){
		let ctx = this._context;
		if (ctx && ctx.pop) {
			if (ctx.pop.test(this._chunk)) {
				this.popEnd();
			};
		};
		
		// we should let the current context decide which methods to call
		pi = (ctx && ctx.lexer && ctx.lexer.call(this)) || (this._end == 'TAG' && this.tagDefContextToken()) || (this._inTag && this.tagContextToken()) || (this._inStyle2 && this.lexStyleBody()) || this.basicContext();
		i += pi;
		this._loc = this._locOffset + i;
	};
	
	return;
};

Lexer.prototype.basicContext = function (){
	var chr = this._chunk.charAt(0);

	if (this._end == '%') {
		return this.selectorToken() || this.symbolToken() || this.identifierToken() || this.whitespaceToken() || this.lineToken() || this.commentToken() || this.heredocToken() || this.tagToken() || this.stringToken() || this.numberToken() || this.regexToken() || this.literalToken() || 0;
	};

	switch (chr) {
		case ' ':
		case '\t': {
			return this.whitespaceToken() || this.commentToken() || this.literalToken() || 0;
		}
		case '\n': {
			return this.whitespaceToken() || this.lineToken() || this.commentToken() || this.literalToken() || 0;
		}
		case '$':
		case '%': {
			return this.selectorToken() || this.identifierToken() || this.literalToken() || 0;
		}
		case '#': {
			return this.identifierToken() || this.commentToken() || this.literalToken() || 0;
		}
		case '@': {
			return this.identifierToken() || this.literalToken() || 0;
		}
		case ':': {
			return this.symbolToken() || this.literalToken() || 0;
		}
		case '"':
		case "'": {
			return this.heredocToken() || this.stringToken() || this.literalToken() || 0;
		}
		case '`': {
			return this.stringToken() || this.literalToken() || 0;
		}
		case '/': {
			return this.commentToken() || this.regexToken() || this.literalToken() || 0;
		}
		case '<': {
			return this.tagToken() || this.literalToken() || 0;
		}
		case '.':
		case '-': {
			return this.numberToken() || this.literalToken() || 0;
		}
	};

	var code = chr.charCodeAt(0);
	if ((code > 127 || code == 11 || code == 12 || code == 13) && WHITESPACE.test(this._chunk)) {
		return this.whitespaceToken() || this.commentToken() || this.literalToken() || 0;
	};

	if ((code >= 65 && code <= 90) || (code >= 97 && code <= 122) || chr == '_') {
		return this.identifierToken() || this.literalToken() || 0;
	};

	if (code >= 48 && code <= 57) {
		return this.numberToken() || this.literalToken() || 0;
	};

	return this.literalToken() || 0;
};

Lexer.prototype.prevChars = function (n){
	if(n === undefined) n = 1;
	return (n == 1) ? this._code[this._loc - 1] : this._code.slice(this._loc - n,this._loc);
};

Lexer.prototype.moveCaret = function (i){
	return this._loc += i;
};

Lexer.prototype.context = function (){
	return this._ends[this._ends.length - 1];
};

Lexer.prototype.inContext = function (key){
	var o = this._contexts[this._contexts.length - 1];
	return o && o[key];
};

Lexer.prototype.pushEnd = function (val,ctx){
	let prev = this._context;
	this._ends.push(val);
	this._contexts.push(this._context = (ctx || {}));
	this._end = val;
	this.refreshScope();
	
	if (ctx && (ctx.closeType == 'STYLE_END' || ctx.style)) {
		ctx.lexer = this.lexStyleBody;
		ctx.style = true;
		this._inStyle++;
	};
	
	if (prev && prev.style && val != '}') {
		ctx.lexer = this.lexStyleBody;
		ctx.style = true;
	};
	
	if (ctx && ctx.id) {
		ctx.start = new Token(ctx.id + '_START',val,this._last.region()[1],0);
		this._tokens.push(ctx.start);
	};
	return this;
};

Lexer.prototype.popEnd = function (val){
	var popped = this._ends.pop();
	this._end = this._ends[this._ends.length - 1];
	
	// automatically adding a closer if this is defined
	var ctx = this._context;
	if (ctx && ctx.start) {
		ctx.end = new Token(ctx.closeType || ctx.id + '_END',popped,this._last.region()[1],0);
		ctx.end._start = ctx.start;
		ctx.start._end = ctx.end;
		this._tokens.push(ctx.end);
	};
	
	if (ctx && (ctx.closeType == 'STYLE_END' || ctx.style)) {
		this._inStyle--;
	};
	
	this._contexts.pop();
	this._context = this._contexts[this._contexts.length - 1];
	
	this.refreshScope();
	return [popped,ctx];
};

Lexer.prototype.refreshScope = function (){
	var ctx0 = this._ends[this._ends.length - 1];
	var ctx1 = this._ends[this._ends.length - 2];
	return this._inTag = ctx0 == 'TAG_END' || (ctx1 == 'TAG_END' && ctx0 == 'OUTDENT');
};

Lexer.prototype.queueScope = function (val){
	this._scopes[this._indents.length] = val;
	return this;
};

Lexer.prototype.popScope = function (val){
	this._scopes.pop();
	return this;
};

Lexer.prototype.getScope = function (){
	return this._scopes[this._indents.length - 1];
};

Lexer.prototype.scope = function (sym,opts){
	var len = this._ends.push(this._end = sym);
	this._contexts.push(opts || null);
	return sym;
};

Lexer.prototype.closeSelector = function (){
	if (this._end == '%') {
		this.token('SELECTOR_END','%',0);
		return this.pair('%');
	};
};

Lexer.prototype.openDef = function (){
	return this.pushEnd('DEF');
};

Lexer.prototype.closeDef = function (){
	if (this.context() == 'DEF') {
		var prev = last(this._tokens);
		
		if (tT(prev) == 'TERMINATOR') {
			let n = this._tokens.pop();
			this.token('DEF_BODY','DEF_BODY',0);
			this._tokens.push(n);
		} else {
			this.token('DEF_BODY','DEF_BODY',0);
		};
		
		this.pair('DEF');
	};
	return;
};

Lexer.prototype.tagContextToken = function (){
	let chr = this._chunk[0];
	let chr2 = this._chunk[1];
	
	// let m = TAG_PART.exec(@chunk)
	let m = /^([A-Za-z\_\-\$\%\#][\w\-\$]*(\:[A-Za-z\_\-\$]+)*)/.exec(this._chunk); // (\:[A-Za-z\_][\w\-]*)
	
	if (m) { // and false
		let tok = m[1];
		let typ = 'TAG_LITERAL';
		let len = m[0].length;
		
		if (tok == 'self' && this._lastVal == '<') {
			typ = 'SELF';
		};
		
		if (chr == '$' && (this._lastTyp == 'TAG_TYPE' || this._lastTyp == 'TAG_START')) {
			typ = 'TAG_REF';
		};
		
		if (chr == '%') {
			typ = 'CSS_MIXIN';
		};
		
		if (chr == '#') {
			typ = 'TAG_SYMBOL_ID';
			if (tok.length == 1) {
				return 0;
			};
		};
		
		this.token(typ,tok,len);
		return len;
	};
	
	if (chr == '/' && chr2 == '>') {
		this.token("TAG_END",'/>',2);
		this.pair('TAG_END');
		return 2;
	};
	
	if (chr == '%' || chr == ':' || chr == '.' || chr == '@') {
		this.token(("T" + chr),chr,1);
		
		if (chr == '.' && (!chr2 || TSC_CARET_BEFORE[chr2]) && this._platform == 'tsc') {
			this.token('TAG_LITERAL','$CARET$',0,1);
		};
		return 1;
	} else if (chr == ' ' || chr == '\n' || chr == '\t') {
		// add whitespace inside tag
		let m = /^[\n\s\t]+/.exec(this._chunk);
		this.token('TAG_WS',m[0],m[0].length);
		return m[0].length;
	} else if (chr == '=' && this._chunk[1] != '>') {
		this.token('=','=',1);
		this.pushEnd('TAG_ATTR',{id: 'VALUE',pop: /^([\s\n\>]|\/\>)/});
		return 1;
	};
	return 0;
};

Lexer.prototype.tagDefContextToken = function (){
	// console.log "tagContextToken"
	var match;
	if (match = TAG_TYPE.exec(this._chunk)) {
		this.token('TAG_TYPE',match[0],match[0].length);
		return match[0].length;
	};
	
	if (match = TAG_ID.exec(this._chunk)) {
		var input = match[0];
		this.token('TAG_ID',input,input.length);
		return input.length;
	};
	
	if (this._chunk[0] == '\n') {
		this.pair('TAG');
	};
	
	return 0;
};

Lexer.prototype.findTypeAnnotation = function (str,autoend){
	if(autoend === undefined) autoend = false;
	var stack = [];
	var i = 0;
	var replaces = [];
	var ending = /[\=\n\ \t\.\,\:\+]/;
	// could it not happen here?
	while (i < str.length){
		var chr = str.charAt(i);
		let end = stack[0];
		let instr = end == '"' || end == "'";
		
		if (chr && chr == end) {
			stack.shift();
			if (autoend && stack.length == 0) {
				i++;break;
			};
		} else if (!end && (chr == ')' || chr == ']' || chr == '}' || chr == '>')) {
			break;
		} else if (chr == '(') {
			stack.unshift(')');
		} else if (chr == '[') {
			stack.unshift(']');
		} else if (chr == '{') {
			stack.unshift('}');
		} else if (chr == '<') {
			stack.unshift('>');
		} else if (chr == '"') {
			stack.unshift('"');
		} else if (chr == "'") {
			stack.unshift("'");
		} else if (!end && ending.test(chr)) {
			break;
		};
		i++;
	};
	
	if (i == 0) { return null };
	return str.slice(0,i);
};

Lexer.prototype.findBalancedSelector = function (str){
	var stack = [];
	var i = 0;
	var replaces = [];
	// could it not happen here?
	while (i < (str.length - 1)){
		var letter = str.charAt(i);
		
		let end = stack[0];
		let instr = end == '"' || end == "'";
		
		if (letter && letter == end) {
			stack.shift();
		} else if (!instr && (letter == ')' || letter == ']' || letter == '}')) {
			console.log('out of balance!!');
			break;
		} else if (letter == '/') {
			replaces.unshift([i,1,':']);
		} else if (letter == '(' && !instr) {
			stack.unshift(')');
		} else if (letter == '[' && !instr) {
			stack.unshift(']');
		} else if (letter == '"') {
			stack.unshift('"');
		} else if (letter == "'") {
			stack.unshift("'");
		};
		
		if (!end && (letter == '=' || letter == '\n' || letter == '{')) {
			break;
		};
		
		if (!end && letter == ' ') {
			if (stylePropertyLengthAt(str,i + 1)) {
				break;
			};
			
			let after = str.slice(i + 1);
			if (INLINE_COMMENT.exec(after)) {
				break;
			};
		};
		
		// console.log 'try',letter,i,end,str.substr(i,5)
		i++;
	};
	
	if (i == 0) { return null };
	
	let sel = str.slice(0,i);
	if (replaces.length) {
		sel = sel.split('');
		for (let j = 0, len = replaces.length; j < len; j++) {
			sel.splice.apply(sel,replaces[j]);
		};
		sel = sel.join('');
	};
	
	return sel;
};

Lexer.prototype.lexStyleRule = function (offset,force){
	// when we meet = enter into style context?
	if(offset === undefined) offset = 0;
	if(force === undefined) force = false;
	let chunk = offset ? this._chunk.slice(offset) : this._chunk;
	let sel = this.findBalancedSelector(chunk);
	
	if (sel || force) {
		let len = sel ? sel.length : 0;
		this.token('CSS_SEL',sel || '',len,offset);
		let seltoken = this._last;
		let next = chunk[len];
		
		if (next == '=') {
			len++;
		};
		
		// if @context
		//	@context:lexer = null
		
		this._indents.push(1);
		this._outdebt = this._indebt = 0;
		this.token('INDENT',"1",0,1);
		this.pushEnd('OUTDENT',{lexer: this.lexStyleBody,opener: seltoken,style: true});
		this._indent++;
		return len;
	};
	
	return 0;
};

Lexer.prototype.lexStyleBody = function (){
	if (this._end == '%') { return 0 };
	// return 0
	let chr = this._chunk[0];
	var m;
	
	let styleprop = stylePropertyLengthAt(this._chunk);
	let ltyp = this._lastTyp;
	
	if (!styleprop && (ltyp == 'TERMINATOR' || ltyp == 'INDENT') && canStartStyleSelector(this._chunk,chr)) {
		let sel = this.findBalancedSelector(this._chunk);
		if (sel) { return this.lexStyleRule(0) };
	};
	
	if (styleprop) {
		// what is the last one?
		this.token('CSSPROP',this._chunk.slice(0,styleprop),styleprop);
		return styleprop;
	};
	
	if (chr[0] == '#' && (m = STYLE_HEX.exec(this._chunk))) {
		let next = this._chunk[m[0].length];
		let typ = (next == '(') ? 'COLORMIX' : 'COLOR';
		this.token(typ,m[0],m[0].length);
		return m[0].length;
	};
	
	if (chr == '/' && !this._last.spaced) {
		// console.log '!!!'
		this.token('/',chr,1);
		return 1;
	};
	
	if (m = STYLE_NUMERIC.exec(this._chunk)) {
		let len = m[0].length;
		let typ = 'NUMBER';
		
		if (m[2] == '%') {
			typ = 'PERCENTAGE';
			// if @chunk[len]
		} else if (m[2]) {
			typ = 'DIMENSION';
		};
		
		if (this._lastTyp == 'COMPARE' && !this._last.spaced) {
			true;
		};
		this.token(typ,m[0],len);
		return len;
	} else if (m = STYLE_URL.exec(this._chunk)) {
		// console.log 'matching style url',m
		let len = m[0].length;
		this.token('CSSURL',m[0],len);
		return m[0].length;
	} else if (m = STYLE_IDENTIFIER.exec(this._chunk)) {
		let id = 'CSSIDENTIFIER';
		let val = m[0];
		let len = val.length;
		if (val[0] == '-' && val[1] == '-') {
			id = 'CSSVAR';
		} else if (this._last && !this._last.spaced && (ltyp == '}' || ltyp == ')')) {
			id = 'CSSUNIT';
		};
		
		if (this._chunk[len] == '(') {
			id = 'CSSFUNCTION';
		};
		
		this.token(id,val,len);
		
		return len;
	} else if (this._last && !this._last.spaced && (ltyp == '}' || ltyp == ')') && chr == '%') {
		this.token('CSSUNIT',chr,1);
		return 1;
	};
	
	return 0;
};

Lexer.prototype.importsToken = function (){
	var match;
	if (match = IMPORTS.exec(this._chunk)) {
		this.token('IMPORTS',match[1],match[1].length,7);
		return match[0].length;
	};
	return 0;
};

Lexer.prototype.tagToken = function (){
	var match, ary;
	if (!(match = TAG.exec(this._chunk))) { return 0 };
	var ary = iter$(match);var input = ary[0],type = ary[1],identifier = ary[2];
	
	
	if (type == '<') {
		if (TYPE_GENERICS_AFTER.test(this.prevChars(1) || '')) {
			return 0;
		};
		// if @last and !@last:spaced and @lastTyp != 'TERMINATOR' and @lastTyp != 'INDENT'
		//	return 0
		
		this.token('TAG_START','<',1);
		this.pushEnd(INVERSES.TAG_START,{i: this._tokens.length - 1});
		
		if (match = TAG_TYPE.exec(this._chunk.substr(1,40))) {
			let next = this._chunk[match[0].length + 1];
			
			if (match[0] != 'self' && (next != '{' && next != '-')) {
				this.token('TAG_TYPE',match[0],match[0].length,1);
				return input.length + match[0].length;
			};
		} else if (this._chunk[1] == '>') {
			this.token('TAG_TYPE','fragment',0,0);
		};
		
		if (identifier) {
			if (identifier.substr(0,1) == '{') {
				return type.length;
			} else {
				this.token('TAG_NAME',input.substr(1),0);
			};
		};
	};
	
	return input.length;
};

Lexer.prototype.selectorToken = function (){
	var ary;
	var match;
	
	// special handling if we are in this context
	if (this._end == '%') {
		var chr = this._chunk[0];
		var ctx = this._context;
		
		var i = 0;
		var part = '';
		var ending = false;
		
		while (chr = this._chunk[i++]){
			if (chr == ')' && ctx.parens == 0) {
				ending = true;
				break;
			} else if (chr == '(') {
				ctx.parens++;
				part += '(';
			} else if (chr == ')') {
				ctx.parens--;
				part += ')';
			} else if (chr == '{') {
				break;
			} else {
				part += chr;
			};
		};
		
		if (part) {
			this.token('SELECTOR_PART',part,i - 1);
		};
		if (ending) {
			this.token('SELECTOR_END',')',1,i - 1);
			this.pair('%');
			return i;
		};
		return i - 1;
	};
	
	if (!(match = SELECTOR.exec(this._chunk))) { return 0 };
	
	var ary = iter$(match);var input = ary[0],id = ary[1],kind = ary[2];
	
	// this is a closed selector
	if (kind == '(') {
		// token '(','('
		this.token('SELECTOR_START',id,id.length + 1);
		this.pushEnd('%',{parens: 0});
		return id.length + 1;
	} else if (id == '%') {
		// we are already scoped in on a selector
		if (this.context() == '%') { return 1 };
		this.token('SELECTOR_START',id,id.length);
		// this is a separate - scope. Full selector should rather be $, and keep the single selector as %
		
		this.pushEnd('%',{open: true});
		// @ends.push '%'
		// make sure a terminator breaks out
		return id.length;
	} else {
		return 0;
	};
};

Lexer.prototype.inTag = function (){
	var len = this._ends.length;
	if (len > 0) {
		var ctx0 = this._ends[len - 1];
		var ctx1 = (len > 1) ? this._ends[len - 2] : ctx0;
		return ctx0 == 'TAG_END' || (ctx1 == 'TAG_END' && ctx0 == 'OUTDENT');
	};
	return false;
};

Lexer.prototype.isKeyword = function (id,next){
	var m;
	if(next === undefined) next = '';
	if (id == 'tag') {
		return this.isTagDeclarationKeyword();
	};

	if (id == 'mixin' && (!next || next == ' ')) {
		if (MEMBER_KEYWORDS[this._lastTyp]) { return false };
		return true;
	};
	
	if (this._lastTyp == 'ATTR' || this._lastTyp == 'PROP' || this._lastTyp == 'DEF') {
		return false;
	};
	
	// hack to allow imba.when to be exported with tree-shaking
	if (id == 'when' && this._lastTyp == 'CONST') {
		return false;
	};
	
	if (id == 'get' || id == 'set') {
		if (m = this._chunk.match(/^[gs]et ([\$\w\-]+|\[)/)) { // ( (do)|\n(\t+))
			let ctx = this._contexts[this._contexts.length - 1] || {};
			let before = ctx.opener && this._tokens[this._tokens.indexOf(ctx.opener) - 1];
			
			if (this._lastTyp == 'TERMINATOR' || this._lastTyp == 'INDENT') {
				if (before && (before._type == '=' || before._type == '{')) {
					return true;
				};
			};
		};
	};
	
	if ((id == 'guard' || id == 'alter' || id == 'watch') && (this.getScope() == 'PROP')) {
		// TODO Remove
		return true;
	};
	
	if (id == 'css') {
		// experimental css inside tag trees - making css keyword everywhere
		return true;
		
		if ((this._lastTyp == 'TERMINATOR' || !this._lastTyp)) {
			return true;
		};
		
		if ((this._lastVal == 'global' || this._lastVal == 'local' || this._lastVal == 'export' || this._lastVal == 'default')) {
			return true;
		};
		
		if ((this._lastTyp == '=')) {
			return true;
		};
	};
	
	if (id == 'interface') {
		if ((this._lastVal == 'global' || this._lastVal == 'export' || this._lastVal == 'default' || this._lastVal == 'declare')) {
			return true;
		};
	};
	
	if ((id == 'attr' || id == 'prop' || id == 'get' || id == 'set' || id == 'css' || id == 'constructor' || id == 'declare')) {
		
		var scop = this.getScope();
		var incls = scop == 'CLASS' || scop == 'TAG' || scop == 'EXTEND';
		
		// if id == 'css' and !@context and (@lastTyp in ['TERMINATOR'] or !@lastTyp)
		// 	return true
		
		// if id == 'css' and (@lastVal in ['global','local','export'])
		// 	return true
		
		if (id == 'declare') {
			return incls && DECLARE_START_MAP[this._lastTyp] == 1;
		};
		
		if (id == 'constructor') {
			return incls && DECLARE_START_MAP[this._lastTyp] == 1;
		};
		
		if (incls) { return true };
	};
	
	return ALL_KEYWORDS_MAP[id] == 1;
};

Lexer.prototype.isTagDeclarationKeyword = function (){
	var ltyp = this._lastTyp;
	var lval = this._lastVal;
	var atDeclarationStart = !ltyp || TAG_DECLARATION_START_MAP[ltyp] == 1 || (ltyp == 'IDENTIFIER' && TAG_DECLARATION_PREFIX_MAP[lval] == 1);
	return atDeclarationStart && /^tag[^\n\S]+(\w[\w\d]*:)?(\w[\w\d]*)(-[\w\d]+)*/.test(this._chunk);
};

// Matches identifying literals: variables, keywords, method names, etc.
// Check to ensure that JavaScript reserved words aren't being used as
// identifiers. Because Imba reserves a handful of keywords that are
// allowed in JavaScript, we're careful not to tokid them as keywords when
// referenced as property names here, so you can still do `jQuery.is()` even
// though `is` means `===` otherwise.
Lexer.prototype.identifierToken = function (){
	var ary;
	var match;
	var typ;
	
	if (!(match = IDENTIFIER.exec(this._chunk))) {
		return 0;
	};
	
	var ary = iter$(match);var input = ary[0],id = ary[1],typ = ary[2],m3 = ary[3],m4 = ary[4],colon = ary[5];
	var idlen = id.length;
	
	// What is the logic here?
	if (id === 'own' && this.lastTokenType() == 'FOR') {
		this.token('OWN',id,id.length);
		return id.length;
	};
	
	var prev = last(this._tokens);
	var lastTyp = this._lastTyp;
	
	if (lastTyp == '#') {
		this.token('IDENTIFIER',id,idlen);
		return idlen;
	};
	
	var forcedIdentifier = colon || lastTyp == '.' || lastTyp == '?.';
	
	if (colon && lastTyp == '?') { forcedIdentifier = false }; // for ternary
	
	if (!typ && !colon && (lastTyp == '.' || lastTyp == '?.')) {
		this.token('IDENTIFIER',id,idlen);
		return idlen;
	};

	if (id == 'css' && (/css\s\:\:/).exec(this._chunk)) {
		input = id + ' ';
		colon = null;
		forcedIdentifier = false;
	};
	
	var isKeyword = false;
	
	// little reason to check for this right here? but I guess it is only a simple check
	if (typ == '$' && ARGVAR.test(id)) {
		// console.log "TYP $"
		// if id == '$0'
		// 	typ = 'ARGUMENTS'
		// else
		typ = 'ARGVAR';
		id = id.substr(1);
	} else if (typ == '$' && ENV_FLAG.test(id)) {
		typ = 'ENV_FLAG';
		id = id.toUpperCase(); // .slice(1, -1)
	} else if (typ == '@') {
		if (lastTyp == '.') {
			typ = 'IDENTIFIER';
		} else {
			typ = 'DECORATOR';
		};
	} else if (typ == '#') {
		typ = 'SYMBOLID';
	} else if (typ == '##') {
		typ = 'SYMBOLID';
	} else if (typ == '%') {
		// use for something else
		let ltyp = this._lastTyp;
		if (ltyp == 'TERMINATOR' || ltyp == 'INDENT' || ltyp == 'EXPORT') {
			this.token('CSS',id,0);
			this.queueScope('CSS');
			return this.lexStyleRule(0,true);
		};
		typ = 'CSS_MIXIN';
	} else if (typ == '$' && !colon) {
		typ = 'IDENTIFIER';
	} else if (id == 'elif' && !forcedIdentifier) {
		this.token('ELSE','elif',id.length);
		this.token('IF','if');
		return id.length;
	} else {
		typ = 'IDENTIFIER';
	};
	
	if (typ == 'IDENTIFIER' && !colon && !KEYWORD_CANDIDATE_MAP[id] && !SPECIAL_IDENTIFIER_MAP[id] && lastTyp != 'CATCH' && !(lastTyp == 'IDENTIFIER' && this._lastVal == 'protected') && !((lastTyp == 'NUMBER' || lastTyp == ')') && prev && !prev.spaced) && this._end != 'IMPORT' && this._end != 'EXPORT' && lastTyp != 'IMPORT') {
		this.token('IDENTIFIER',id,idlen);
		return idlen;
	};

	var ctx0 = this._end;

	// this catches all
	if (!forcedIdentifier && KEYWORD_CANDIDATE_MAP[id] && (isKeyword = this.isKeyword(id,this._chunk[id.length]))) {
		// (id in JS_KEYWORDS or id in IMBA_KEYWORDS)
		
		if (typeof isKeyword == 'string') {
			typ = isKeyword;
		} else {
			typ = id.toUpperCase();
		};
		
		if (typ == 'MODULE') {
			if (!(/^module [a-zA-Z]/).test(this._chunk) || ctx0 == 'TAG_ATTR') {
				typ = 'IDENTIFIER';
			};
		};
		
		// clumsy - but testing performance
		if (typ == 'YES') {
			typ = 'TRUE';
		} else if (typ == 'NO') {
			typ = 'FALSE';
		} else if (typ == 'NIL') {
			typ = 'NULL';
		} else if ((typ == 'MIXIN' || typ == 'INTERFACE')) {
			typ = 'CLASS';
		} else if (typ == 'VAR' || typ == 'CONST' || typ == 'LET') {
			let ltyp = this._lastTyp;
			
			// extremely flakey - comma separated declaration blocks are inherently
			// ambiguous in the current syntax
			// if ltyp != 'TERMINATOR' and ltyp != 'INDENT' and ltyp != 'EXPORT' and ltyp
			//	typ = "INLINE_{typ}"
			
			// if @lastVal == 'export'
			//	tTs(prev,'EXPORT_VAR')
		} else if (typ == 'IF' || typ == 'ELSE' || typ == 'TRUE' || typ == 'FALSE' || typ == 'NULL') {
			true;
		} else if (typ == 'TAG') {
			this.pushEnd('TAG');
		} else if ((typ == 'DEF' || typ == 'GET' || typ == 'SET')) {
			typ = 'DEF';
			this.openDef();
		} else if ((typ == 'CONSTRUCTOR')) {
			this.token('DEF','',0);
			typ = 'IDENTIFIER';
			this.openDef();
		} else if (typ == 'DO') {
			if (this.context() == 'DEF') this.closeDef();
		} else if (typ === 'WHEN' && LINE_BREAK_MAP[this.lastTokenType()] == 1) {
			typ = 'LEADING_WHEN';
		} else if (typ === 'FOR') {
			this._seenFor = true;
		} else if (typ === 'UNLESS') {
			typ = 'IF'; // WARN
		} else if (UNARY_MAP[typ] == 1) {
			typ = 'UNARY';
		} else if (RELATION_MAP[typ] == 1) {
			if (typ != 'INSTANCEOF' && typ != 'ISA' && this._seenFor) {
				typ = 'FOR' + typ; // ?
				this._seenFor = false;
			} else {
				typ = 'RELATION';
				
				if (prev._type == 'UNARY') {
					prev._type = 'NOT';
				};
			};
		};
	};
	
	// do we really want to check this here
	if (!forcedIdentifier) {
		// should already have dealt with this
		
		if (this._lastVal == 'export' && id == 'default') {
			// console.log 'id is default!!!'
			tTs(prev,'EXPORT');
			typ = 'DEFAULT';
		};
		
		// these really should not go here?!?
		switch (id) {
			case '!': 
			case 'not': {
				typ = 'UNARY';break;
			}
			case '==': 
			case '!=': 
			case '===': 
			case '!==': 
			case 'is': 
			case 'isnt': {
				typ = 'COMPARE';break;
			}
			case '&&': 
			case '||': 
			case 'and': 
			case 'or': 
			case '??': {
				typ = 'LOGIC';break;
			}
			case 'super': 
			case 'break': 
			case 'continue': 
			case 'debugger': 
			case 'arguments': {
				typ = id.toUpperCase();break;
			}
		};
	};
	
	// prev = last @tokens
	var len = input.length;
	
	// if typ == 'CONST' or typ == 'LET'
	//	if @lastVal == 'global' or @lastVal == 'declare'
	//		tTs(prev,@lastVal.toUpperCase)
	
	
	// should be strict about the order, check this manually instead
	if (typ == 'CLASS' || typ == 'DEF' || typ == 'TAG' || typ == "PROP" || typ == 'CSS') {
		this.queueScope(typ);
	};
	
	if (isKeyword && CONTEXTUAL_KEYWORDS[typ]) {
		var i = this._tokens.length;
		var alts = CONTEXTUAL_KEYWORDS[typ];
		
		while (i){
			prev = this._tokens[--i];
			var ctrl = "" + tV(prev);
			if (alts[ctrl]) {
				tTs(prev,ctrl.toUpperCase());
			} else {
				break;
			};
		};
	} else if (typ == 'IF') {
		this.queueScope(typ);
	} else if (typ == 'EXTEND' && !this._chunk.match(/^extend (class|tag|interface|mixin)(\s|\n|$)/)) {
		this.queueScope(typ);
	} else if (typ == 'IMPORT') {
		// console.log 'import last type',lastTyp,@chunk[idlen]
		let next = this._chunk[idlen];
		if (lastTyp == 'AWAIT' || next == '(' || next == '.') {
			typ = 'IDENTIFIER';
		} else {
			this.pushEnd('IMPORT');
			this.token(typ,id,idlen);
			// return importsToken or len
			return len;
		};
	} else if (id == 'type' && lastTyp == 'IMPORT') {
		this.token('TYPEIMPORT',id,idlen);
		// return importsToken or len
		return len;
	} else if (typ == 'EXPORT') {
		this.pushEnd('EXPORT');
		this.token(typ,id,idlen);
		return len;
	} else if (id == 'from' && ctx0 == 'IMPORT') {
		typ = 'FROM';
		this.pair('IMPORT');
	} else if (id == 'from' && ctx0 == 'EXPORT') {
		typ = 'FROM';
		this.pair('EXPORT');
	} else if (id == 'as' && (ctx0 == 'IMPORT' || this._lastTyp == 'IDENTIFIER' || ctx0 == 'EXPORT')) {
		typ = 'AS';
	};
	
	// if id == 'new'
	//	console.log 'new keyword', @chunk.slice(0,5),@chunk.match(/^new\s+[\w\$\(]/)
	
	if (id == 'new' && (this._lastTyp != '.' && this._chunk.match(/^new\s+[\w\$\(\<\#]/)) && this._lastTyp != 'DEF') {
		typ = 'NEW';
	};
	
	if (typ == 'IDENTIFIER') {
		// see if previous was catch -- belongs in rewriter?
		if (lastTyp == 'CATCH') {
			typ = 'CATCH_VAR';
		};
		
		if (this._lastVal == 'protected' && lastTyp == 'IDENTIFIER') {
			tTs(prev,'PROTECTED');
		};
	};
	
	if ((lastTyp == 'NUMBER' || lastTyp == ')') && !prev.spaced && (typ == 'IDENTIFIER' || id == '%')) {
		typ = 'UNIT';
	};
	
	if (colon) {
		this.token(typ,id,idlen);
		var colonOffset = colon.indexOf(':');
		
		this.moveCaret(idlen + colonOffset);
		
		this.token(':',':',1);
		this.moveCaret(-(idlen + colonOffset));
	} else {
		this.token(typ,id,idlen);
	};
	
	if (typ == 'CSS') {
		return len + this.lexStyleRule(len,true);
	};
	
	
	
	return len;
};

// Matches numbers, including decimals, hex, and exponential notation.
// Be careful not to interfere with ranges-in-progress.
Lexer.prototype.numberToken = function (){
	var binaryLiteral;
	var match,number,lexedLength;
	
	if (!(match = NUMBER.exec(this._chunk))) { return 0 };
	
	number = match[0];
	lexedLength = number.length;
	
	if (binaryLiteral = /0b([01_]+)/.exec(number)) {
		
		number = "" + parseInt(binaryLiteral[1].replace(/_/g,''),2);
	};
	
	var prev = last(this._tokens);
	
	if (match[0][0] == '.' && prev && !prev.spaced && PROPERTY_ACCESS_PREV_MAP[tT(prev)] == 1) {
		// console.log "got here"
		this.token(".",".");
		number = number.substr(1);
	};
	
	this.token('NUMBER',number,lexedLength);
	return lexedLength;
};

Lexer.prototype.symbolToken = function (){
	var match,symbol,prev;
	if (!(match = SYMBOL.exec(this._chunk))) { return 0 };
	symbol = match[0];
	prev = last(this._tokens);
	
	if (!prev || prev.spaced || SYMBOL_STRING_PREV_MAP[this._prevVal] == 1) {
		let sym = helpers.dashToCamelCase(symbol.slice(1));
		this.token('STRING','"' + sym + '"',match[0].length);
		return match[0].length;
	};
	return 0;
};

Lexer.prototype.escapeStr = function (str,heredoc,q){
	str = str.replace(MULTILINER,(heredoc ? '\\n' : ''));
	if (q) {
		var r = RegExp(("\\\\[" + q + "]"),"g");
		str = str.replace(r,q);
		str = str.replace(RegExp(("" + q),"g"),'\\$&');
	};
	return str;
	
	// str = str.replace(MULTILINER, '\\n')
	// str = str.replace(/\t/g, '\\t')
};
// Matches strings, including multi-line strings. Ensures that quotation marks
// are balanced within the string's contents, and within nested interpolations.
Lexer.prototype.stringToken = function (){
	var match,string;
	
	switch (this._chunk.charAt(0)) {
		case "'": {
			if (!(match = SIMPLESTR.exec(this._chunk))) { return 0 };
			string = match[0];
			this.token('STRING',this.escapeStr(string),string.length);
			// token 'STRING', (string = match[0]).replace(MULTILINER, '\\\n'), string:length
			break;
		}
		case '"': {
			if (!(string = this.balancedString(this._chunk,'"'))) { return 0 };
			// what about tripe quoted strings?
			
			if (string.indexOf('{') >= 0) {
				var len = string.length;
				// if this has no interpolation?
				// we are now messing with locations - beware
				this.token('STRING_START',string.charAt(0),1);
				this.interpolateString(string.slice(1,-1));
				this.token('STRING_END',string.charAt(len - 1),1,string.length - 1);
			} else {
				len = string.length;
				// string = string.replace(MULTILINER, '\\\n')
				this.token('STRING',this.escapeStr(string),len);
			};
			break;
		}
		case '`': {
			if (!(string = this.balancedString(this._chunk,'`'))) { return 0 };
			
			if (string.indexOf('{') >= 0) {
				len = string.length;
				// if this has no interpolation?
				// we are now messing with locations - beware
				this.token('STRING_START',string.charAt(0),1);
				this.interpolateString(string.slice(1,-1),{heredoc: true});
				this.token('STRING_END',string.charAt(len - 1),1,string.length - 1);
			} else {
				len = string.length;
				// string = string.replace(MULTILINER, '\\\n')
				this.token('STRING',this.escapeStr(string,true),len);
			};
			break;
		}
		default:
		
			return 0;
	
	};
	
	this.moveHead(string);
	return string.length;
};

// Matches heredocs, adjusting indentation to the correct level, as heredocs
// preserve whitespace, but ignore indentation to the left.
Lexer.prototype.heredocToken = function (){
	var match,heredoc,quote,doc;
	
	if (!(match = HEREDOC.exec(this._chunk))) { return 0 };
	
	heredoc = match[0];
	quote = heredoc.charAt(0);
	var opts = {quote: quote,indent: null,offset: 0};
	doc = this.sanitizeHeredoc(match[2],opts);
	// doc = match[2]
	// console.log "found heredoc {match[0]:length} {doc:length}"
	
	if (quote == '"' && doc.indexOf('{') >= 0) {
		var open = match[1];
		// console.log doc.substr(0,3),match[1]
		// console.log 'heredoc here',open:length,open
		
		this.token('STRING_START',open,open.length);
		this.interpolateString(doc,{heredoc: true,offset: (open.length + opts.offset),quote: quote,indent: opts.realIndent});
		this.token('STRING_END',open,open.length,heredoc.length - open.length);
	} else {
		this.token('STRING',this.makeString(doc,quote,true),0);
	};
	
	this.moveHead(heredoc);
	return heredoc.length;
};

Lexer.prototype.parseMagicalOptions = function (str){
	var self = this;
	if (str.indexOf('imba$') >= 0) {
		str.replace(/imba\$(\w+)\=(\S*)\b/g,function(m,name,val) {
			if ((/^\d+$/).test(val)) {
				val = parseInt(val);
			};
			return self._opts[name] = val;
		});
	};
	return self;
};

// Matches and consumes comments.
Lexer.prototype.commentToken = function (){
	var match,length,comment,indent,prev;
	
	var typ = 'HERECOMMENT';
	
	if (match = JS_COMMENT.exec(this._chunk)) {
		this.token('HERECOMMENT',match[1],match[1].length);
		this.token('TERMINATOR','\n');
		return match[0].length;
	};
	
	if (match = INLINE_COMMENT.exec(this._chunk)) { // .match(INLINE_COMMENT)
		// console.log "match inline comment"
		length = match[0].length;
		indent = match[1];
		comment = match[2];
		let commentBody = (match[4] || '');
		if (comment[0] == '#') {
			commentBody = ' ' + commentBody;
		};
		
		prev = last(this._tokens);
		var pt = prev && tT(prev);
		var note = '//' + commentBody; // comment.substr(1)
		
		this.parseMagicalOptions(note);
		
		if (this._last && this._last.spaced) {
			note = ' ' + note;
			// console.log "the previous node was SPACED"
		};
		// console.log "comment {note} - indent({indent}) - {length} {comment:length}"
		if (note.match(/^\/\/ \@(type|param)/)) {
			note = '/**' + commentBody + '*/';
		} else if (note.match(/^\/\/ \<(reference)/)) {
			note = '///' + commentBody;
		};
		
		if ((pt && pt != 'INDENT' && pt != 'TERMINATOR') || !pt) {
			// console.log "skip comment"
			// token 'INLINECOMMENT', comment.substr(2)
			// console.log "adding as terminator"
			this.token('TERMINATOR',note,length); // + '\n'
		} else {
			if (pt == 'TERMINATOR') {
				tVs(prev,tV(prev) + note);
				// prev[1] += note
			} else if (pt == 'INDENT') {
				this.addLinebreaks(1,note);
			} else {
				// console.log "comment here"
				// should we ever get here?
				this.token(typ,comment.substr(2),length); // are we sure?
			};
		};
		
		return length; // disable now while compiling
	};
	
	// should use exec?
	if (!(match = COMMENT.exec(this._chunk))) { return 0 };
	
	comment = match[0];
	var here = match[1];
	
	if (here) {
		this.token('HERECOMMENT',this.sanitizeHeredoc(here,{herecomment: true,indent: Array(this._indent + 1).join(' ')}),comment.length);
		this.token('TERMINATOR','\n');
	} else {
		this.token('HERECOMMENT',comment,comment.length);
		this.token('TERMINATOR','\n'); // auto? really?
	};
	
	this.moveHead(comment);
	return comment.length;
};

// Matches regular expression literals. Lexing regular expressions is difficult
// to distinguish from division, so we borrow some basic heuristics from
// JavaScript and Ruby.
Lexer.prototype.regexToken = function (){
	var ary;
	var match,length,prev;
	
	if (this._chunk.charAt(0) != '/') { return 0 };
	
	if (match = HEREGEX.exec(this._chunk)) {
		length = this.heregexToken(match);
		this.moveHead(match[0]);
		return length;
	};
	
	prev = last(this._tokens);
	// FIX
	if (prev && ((prev.spaced ? 
		NOT_REGEX_MAP
	 : 
		NOT_SPACED_REGEX_MAP
	)[tT(prev)] == 1)) { return 0 };
	if (!(match = REGEX.exec(this._chunk))) { return 0 };
	var ary = iter$(match);var m = ary[0],regex = ary[1],flags = ary[2];
	
	this.token('REGEX',("" + regex + flags),m.length);
	return m.length;
};

// Matches multiline extended regular expressions.
// The escaping should rather happen in AST - possibly as an additional flag?
Lexer.prototype.heregexToken = function (match){
	var ary;
	var ary = iter$(match);var heregex = ary[0],body = ary[1],flags = ary[2];
	this.token('REGEX',heregex,heregex.length);
	return heregex.length;
};

// Matches newlines, indents, and outdents, and determines which is which.
// If we can detect that the current line is contin