coffee-lex
Version:
Stupid lexer for CoffeeScript.
1,470 lines (1,458 loc) • 51.5 kB
JavaScript
// src/SourceLocation.ts
var SourceLocation = class {
constructor(type, index) {
this.type = type;
this.index = index;
}
};
// src/utils/assert.ts
function assert(condition, message = `expected '${condition}' to be truthy`) {
if (!condition) {
throw new Error(message);
}
}
function assertNever(value, message = `unexpected value: ${value}`) {
throw new Error(message);
}
// src/SourceToken.ts
var SourceToken = class {
constructor(type, start, end) {
this.type = type;
this.start = start;
this.end = end;
assert(start <= end, `Token start may not be after end. Got ${type}, ${start}, ${end}`);
}
};
// src/SourceTokenListIndex.ts
var SourceTokenListIndex = class {
constructor(sourceTokenList, index) {
this._sourceTokenList = sourceTokenList;
this._index = index;
}
advance(offset) {
const newIndex = this._index + offset;
if (newIndex < 0 || this._sourceTokenList.length < newIndex) {
return null;
}
return this._sourceTokenList["_getIndex"](newIndex);
}
next() {
return this.advance(1);
}
previous() {
return this.advance(-1);
}
isBefore(other) {
return this.compare(other) > 0;
}
isAfter(other) {
return this.compare(other) < 0;
}
compare(other) {
return this.distance(other);
}
distance(other) {
assert(other._sourceTokenList === this._sourceTokenList, "cannot compare indexes from different lists");
return other._index - this._index;
}
};
// src/SourceType.ts
var SourceType = /* @__PURE__ */ ((SourceType2) => {
SourceType2["AT"] = "AT";
SourceType2["BOOL"] = "BOOL";
SourceType2["BREAK"] = "BREAK";
SourceType2["CATCH"] = "CATCH";
SourceType2["CALL_END"] = "CALL_END";
SourceType2["CALL_START"] = "CALL_START";
SourceType2["CLASS"] = "CLASS";
SourceType2["COLON"] = "COLON";
SourceType2["COMMA"] = "COMMA";
SourceType2["COMMENT"] = "COMMENT";
SourceType2["CONTINUATION"] = "CONTINUATION";
SourceType2["CONTINUE"] = "CONTINUE";
SourceType2["CSX_BODY"] = "CSX_BODY";
SourceType2["CSX_CLOSE_TAG_START"] = "CSX_CLOSE_TAG_START";
SourceType2["CSX_CLOSE_TAG_END"] = "CSX_CLOSE_TAG_END";
SourceType2["CSX_OPEN_TAG_START"] = "CSX_OPEN_TAG_START";
SourceType2["CSX_OPEN_TAG_END"] = "CSX_OPEN_TAG_END";
SourceType2["CSX_SELF_CLOSING_TAG_END"] = "CSX_SELF_CLOSING_TAG_END";
SourceType2["DECREMENT"] = "DECREMENT";
SourceType2["DEFAULT"] = "DEFAULT";
SourceType2["DELETE"] = "DELETE";
SourceType2["DO"] = "DO";
SourceType2["DOT"] = "DOT";
SourceType2["DSTRING_START"] = "DSTRING_START";
SourceType2["DSTRING_END"] = "DSTRING_END";
SourceType2["ELSE"] = "ELSE";
SourceType2["EXPORT"] = "EXPORT";
SourceType2["EOF"] = "EOF";
SourceType2["EXISTENCE"] = "EXISTENCE";
SourceType2["EXTENDS"] = "EXTENDS";
SourceType2["FINALLY"] = "FINALLY";
SourceType2["FOR"] = "FOR";
SourceType2["FUNCTION"] = "FUNCTION";
SourceType2["HERECOMMENT"] = "HERECOMMENT";
SourceType2["HEREJS"] = "HEREJS";
SourceType2["HEREGEXP_COMMENT"] = "HEREGEXP_COMMENT";
SourceType2["HEREGEXP_START"] = "HEREGEXP_START";
SourceType2["HEREGEXP_END"] = "HEREGEXP_END";
SourceType2["IF"] = "IF";
SourceType2["IMPORT"] = "IMPORT";
SourceType2["INCREMENT"] = "INCREMENT";
SourceType2["INTERPOLATION_START"] = "INTERPOLATION_START";
SourceType2["INTERPOLATION_END"] = "INTERPOLATION_END";
SourceType2["JS"] = "JS";
SourceType2["LBRACE"] = "LBRACE";
SourceType2["LBRACKET"] = "LBRACKET";
SourceType2["LOOP"] = "LOOP";
SourceType2["LPAREN"] = "LPAREN";
SourceType2["NEW"] = "NEW";
SourceType2["NEWLINE"] = "NEWLINE";
SourceType2["NORMAL"] = "NORMAL";
SourceType2["NULL"] = "NULL";
SourceType2["NUMBER"] = "NUMBER";
SourceType2["OPERATOR"] = "OPERATOR";
SourceType2["OWN"] = "OWN";
SourceType2["PROTO"] = "PROTO";
SourceType2["RANGE"] = "RANGE";
SourceType2["REGEXP"] = "REGEXP";
SourceType2["RBRACE"] = "RBRACE";
SourceType2["RBRACKET"] = "RBRACKET";
SourceType2["RELATION"] = "RELATION";
SourceType2["RETURN"] = "RETURN";
SourceType2["RPAREN"] = "RPAREN";
SourceType2["SEMICOLON"] = "SEMICOLON";
SourceType2["SPACE"] = "SPACE";
SourceType2["SUPER"] = "SUPER";
SourceType2["SWITCH"] = "SWITCH";
SourceType2["SSTRING_START"] = "SSTRING_START";
SourceType2["SSTRING_END"] = "SSTRING_END";
SourceType2["STRING_CONTENT"] = "STRING_CONTENT";
SourceType2["STRING_LINE_SEPARATOR"] = "STRING_LINE_SEPARATOR";
SourceType2["STRING_PADDING"] = "STRING_PADDING";
SourceType2["TDSTRING_START"] = "TDSTRING_START";
SourceType2["TDSTRING_END"] = "TDSTRING_END";
SourceType2["THEN"] = "THEN";
SourceType2["THIS"] = "THIS";
SourceType2["THROW"] = "THROW";
SourceType2["TRY"] = "TRY";
SourceType2["TSSTRING_START"] = "TSSTRING_START";
SourceType2["TSSTRING_END"] = "TSSTRING_END";
SourceType2["UNDEFINED"] = "UNDEFINED";
SourceType2["UNKNOWN"] = "UNKNOWN";
SourceType2["WHEN"] = "WHEN";
SourceType2["WHILE"] = "WHILE";
SourceType2["IDENTIFIER"] = "IDENTIFIER";
SourceType2["YIELD"] = "YIELD";
SourceType2["YIELDFROM"] = "YIELDFROM";
return SourceType2;
})(SourceType || {});
// src/SourceTokenList.ts
var SourceTokenList = class {
constructor(tokens) {
this._validateTokens(tokens);
this._tokens = tokens;
this._indexCache = new Array(tokens.length);
this.length = tokens.length;
this.startIndex = this._getIndex(0);
this.endIndex = this._getIndex(tokens.length);
this._indexBySourceIndex = [];
this._indexByStartSourceIndex = [];
this._indexByEndSourceIndex = [];
for (let tokenIndex = tokens.length - 1; tokenIndex >= 0; tokenIndex--) {
const token = tokens[tokenIndex];
for (let sourceIndex = token.start; sourceIndex < token.end; sourceIndex++) {
this._indexBySourceIndex[sourceIndex] = this._getIndex(tokenIndex);
}
this._indexByStartSourceIndex[token.start] = this._getIndex(tokenIndex);
this._indexByEndSourceIndex[token.end] = this._getIndex(tokenIndex);
}
}
forEach(iterator) {
this._tokens.forEach((token, i) => iterator(token, this._getIndex(i), this));
}
map(mapper) {
const result = [];
this.forEach((token, index, list) => {
result.push(mapper(token, index, list));
});
return result;
}
filter(predicate) {
const result = [];
this.forEach((token, index, list) => {
if (predicate(token, index, list)) {
result.push(token);
}
});
return new SourceTokenList(result);
}
slice(start, end) {
assert(start["_sourceTokenList"] === this && end["_sourceTokenList"] === this, "cannot slice a list using indexes from another list");
return new SourceTokenList(this._tokens.slice(start["_index"], end["_index"]));
}
tokenAtIndex(index) {
this._validateIndex(index);
return this._tokens[index["_index"]] || null;
}
rangeOfInterpolatedStringTokensContainingTokenIndex(index) {
let bestRange = null;
for (const [startType, endType] of [
["DSTRING_START" /* DSTRING_START */, "DSTRING_END" /* DSTRING_END */],
["TDSTRING_START" /* TDSTRING_START */, "TDSTRING_END" /* TDSTRING_END */],
["HEREGEXP_START" /* HEREGEXP_START */, "HEREGEXP_END" /* HEREGEXP_END */]
]) {
const range = this.rangeOfMatchingTokensContainingTokenIndex(startType, endType, index);
if (bestRange === null || bestRange === void 0 || range !== null && range !== void 0 && range[0].distance(range[1]) < bestRange[0].distance(bestRange[1])) {
bestRange = range;
}
}
return bestRange;
}
rangeOfMatchingTokensContainingTokenIndex(startType, endType, index) {
this._validateIndex(index);
const token = this.tokenAtIndex(index);
if (!token) {
return null;
}
switch (token.type) {
case startType: {
let level = 0;
const start = index;
const endIndex = this.indexOfTokenMatchingPredicate((token2) => {
if (token2.type === startType) {
level += 1;
} else if (token2.type === endType) {
level -= 1;
if (level === 0) {
return true;
}
}
return false;
}, start);
if (!endIndex) {
return null;
} else {
const rangeEnd = endIndex.next();
if (!rangeEnd) {
return null;
}
return [start, rangeEnd];
}
}
case endType: {
let level = 0;
const endIndex = index;
const startIndex = this.lastIndexOfTokenMatchingPredicate((token2) => {
if (token2.type === startType) {
level -= 1;
if (level === 0) {
return true;
}
} else if (token2.type === endType) {
level += 1;
}
return false;
}, endIndex);
if (!startIndex) {
return null;
} else {
const rangeEnd = endIndex.next();
if (!rangeEnd) {
return null;
} else {
return [startIndex, rangeEnd];
}
}
}
default: {
let level = 0;
const startIndex = this.lastIndexOfTokenMatchingPredicate((token2) => {
if (token2.type === startType) {
if (level === 0) {
return true;
}
level -= 1;
} else if (token2.type === endType) {
level += 1;
}
return false;
}, index);
if (!startIndex) {
return null;
} else {
return this.rangeOfMatchingTokensContainingTokenIndex(startType, endType, startIndex);
}
}
}
}
indexOfTokenContainingSourceIndex(index) {
this._validateSourceIndex(index);
return this._indexBySourceIndex[index] || null;
}
indexOfTokenNearSourceIndex(index) {
this._validateSourceIndex(index);
for (let searchIndex = index; searchIndex >= 0; searchIndex--) {
const tokenIndex = this._indexBySourceIndex[searchIndex];
if (tokenIndex) {
return tokenIndex;
}
}
return this.startIndex;
}
indexOfTokenStartingAtSourceIndex(index) {
this._validateSourceIndex(index);
return this._indexByStartSourceIndex[index] || null;
}
indexOfTokenEndingAtSourceIndex(index) {
this._validateSourceIndex(index);
return this._indexByEndSourceIndex[index] || null;
}
indexOfTokenMatchingPredicate(predicate, start = null, end = null) {
if (!start) {
start = this.startIndex;
}
if (!end) {
end = this.endIndex;
}
this._validateIndex(start);
this._validateIndex(end);
for (let i = start; i && i !== end; i = i.next()) {
const token = this.tokenAtIndex(i);
if (!token) {
break;
} else if (predicate(token)) {
return i;
}
}
return null;
}
lastIndexOfTokenMatchingPredicate(predicate, start = null, end = null) {
if (!start) {
start = this.endIndex.previous();
if (!start) {
return null;
}
}
this._validateIndex(start);
if (end) {
this._validateIndex(end);
}
let i = start;
do {
const token = this.tokenAtIndex(i);
if (!token) {
break;
} else if (predicate(token)) {
return i;
} else {
i = i.previous();
}
} while (i && i !== end);
return null;
}
[Symbol.iterator]() {
let index = this.startIndex;
const { endIndex } = this;
return {
next() {
if (index === endIndex) {
return { done: true, value: void 0 };
} else {
const result = { done: false, value: this.tokenAtIndex(index) };
const nextIndex = index.next();
assert(nextIndex, `unexpected null index before the end index`);
index = nextIndex;
return result;
}
}
};
}
_validateTokens(tokens) {
for (let i = 0; i < tokens.length - 1; i++) {
assert(tokens[i].end <= tokens[i + 1].start, `Tokens not in order. Expected ${JSON.stringify(tokens[i])} before ${JSON.stringify(tokens[i + 1])}`);
}
}
_validateIndex(index) {
assert(index, `unexpected 'null' index, perhaps you forgot to check the result of 'indexOfTokenContainingSourceIndex'?`);
assert(typeof index !== "number", `to get a token at index ${index}, use list.tokenAtIndex(list.startIndex.advance(${index}))`);
assert(index["_sourceTokenList"] === this, "cannot get token in one list using an index from another");
}
_validateSourceIndex(index) {
assert(typeof index === "number", `expected source index to be a number, got: ${index}`);
}
_getIndex(index) {
let cached = this._indexCache[index];
if (!cached) {
cached = new SourceTokenListIndex(this, index);
this._indexCache[index] = cached;
}
return cached;
}
toArray() {
return this._tokens.slice();
}
};
// src/utils/BufferedStream.ts
var BufferedStream = class {
constructor(stream2) {
this.pending = [];
this._getNextLocation = stream2;
}
shift() {
return this.pending.shift() || this._getNextLocation();
}
hasNext(...types) {
const locationsToPutBack = [];
const result = types.every((type) => {
const next = this.shift();
locationsToPutBack.push(next);
return next.type === type;
});
this.unshift(...locationsToPutBack);
return result;
}
peek() {
const result = this.shift();
this.unshift(result);
return result;
}
unshift(...tokens) {
this.pending.unshift(...tokens);
}
};
// src/utils/PaddingTracker.ts
var PaddingTracker = class {
constructor(source, stream2, endType) {
this.fragments = [];
this._originalLocations = [];
let interpolationLevel = 0;
let location;
do {
location = stream2.shift();
this._originalLocations.push(location);
if (interpolationLevel === 0 && location.type === "STRING_CONTENT" /* STRING_CONTENT */) {
const start = location.index;
const end = stream2.peek().index;
const content = source.slice(start, end);
const index = this.fragments.length;
this.fragments.push(new TrackedFragment(content, start, end, index));
} else if (location.type === "INTERPOLATION_START" /* INTERPOLATION_START */) {
interpolationLevel += 1;
} else if (location.type === "INTERPOLATION_END" /* INTERPOLATION_END */) {
interpolationLevel -= 1;
}
} while (interpolationLevel > 0 || location.type !== endType);
}
computeSourceLocations() {
const resultLocations = [];
let rangeIndex = 0;
for (const location of this._originalLocations) {
const currentRange = this.fragments[rangeIndex];
if (location.type === "STRING_CONTENT" /* STRING_CONTENT */ && currentRange && location.index === currentRange.start) {
resultLocations.push(...currentRange.computeSourceLocations());
rangeIndex++;
} else {
resultLocations.push(location);
}
}
assert(rangeIndex === this.fragments.length, "Expected ranges to correspond to original locations.");
return resultLocations;
}
};
var TrackedFragment = class {
constructor(content, start, end, index) {
this.content = content;
this.start = start;
this.end = end;
this.index = index;
this._paddingRanges = [];
this._lineSeparators = [];
}
markPadding(startIndex, endIndex) {
this._paddingRanges.push({ start: startIndex, end: endIndex });
}
markLineSeparator(index) {
this._lineSeparators.push(index);
}
computeSourceLocations() {
if (this.start === this.end) {
return [new SourceLocation("STRING_CONTENT" /* STRING_CONTENT */, this.start)];
}
const eventsByIndex = [];
for (let i = 0; i < this.end - this.start + 1; i++) {
eventsByIndex.push([]);
}
for (const range of this._paddingRanges) {
eventsByIndex[range.start].push("START_PADDING");
eventsByIndex[range.end].push("END_PADDING");
}
for (const separatorIndex of this._lineSeparators) {
eventsByIndex[separatorIndex].push("START_LINE_SEPARATOR");
eventsByIndex[separatorIndex + 1].push("END_LINE_SEPARATOR");
}
const resultLocations = [];
let lastSourceType = null;
let paddingDepth = 0;
let lineSeparatorDepth = 0;
for (let sourceIndex = this.start; sourceIndex < this.end; sourceIndex++) {
for (const event of eventsByIndex[sourceIndex - this.start]) {
if (event === "START_PADDING") {
paddingDepth += 1;
} else if (event === "END_PADDING") {
paddingDepth -= 1;
} else if (event === "START_LINE_SEPARATOR") {
lineSeparatorDepth += 1;
} else if (event === "END_LINE_SEPARATOR") {
lineSeparatorDepth -= 1;
}
}
assert(paddingDepth >= 0 && lineSeparatorDepth >= 0 && (paddingDepth <= 0 || lineSeparatorDepth <= 0), `Illegal padding state: paddingDepth: ${paddingDepth}, lineSeparatorDepth: ${lineSeparatorDepth}`);
let sourceType;
if (paddingDepth > 0) {
sourceType = "STRING_PADDING" /* STRING_PADDING */;
} else if (lineSeparatorDepth > 0) {
sourceType = "STRING_LINE_SEPARATOR" /* STRING_LINE_SEPARATOR */;
} else {
sourceType = "STRING_CONTENT" /* STRING_CONTENT */;
}
if (sourceType !== lastSourceType) {
resultLocations.push(new SourceLocation(sourceType, sourceIndex));
lastSourceType = sourceType;
}
}
return resultLocations;
}
};
// src/utils/calculateHeregexpPadding.ts
function calculateHeregexpPadding(source, stream2) {
if (!stream2.hasNext("HEREGEXP_START" /* HEREGEXP_START */)) {
return [];
}
const paddingTracker = new PaddingTracker(source, stream2, "HEREGEXP_END" /* HEREGEXP_END */);
for (const fragment of paddingTracker.fragments) {
const content = fragment.content;
let pos = 0;
while (pos < content.length) {
if (/\s/.test(content[pos])) {
if (isWhitespaceEscaped(content, pos)) {
fragment.markPadding(pos - 1, pos);
} else {
fragment.markPadding(pos, pos + 1);
}
pos++;
} else if (content[pos] === "#" && (pos === 0 || /\s/.test(content[pos - 1]))) {
const commentStart = pos;
while (pos < content.length && content[pos] !== "\n") {
pos++;
}
fragment.markPadding(commentStart, pos);
} else {
pos++;
}
}
}
return paddingTracker.computeSourceLocations();
}
function isWhitespaceEscaped(content, whitespacePos) {
let prevPos = whitespacePos - 1;
while (prevPos >= 0 && content[prevPos] === "\\") {
prevPos--;
}
return (whitespacePos - prevPos) % 2 === 0;
}
// src/utils/isNewlineEscaped.ts
function isNewlineEscaped(content, newlinePos) {
let numSeenBackslashes = 0;
let prevPos = newlinePos - 1;
while (prevPos >= 0) {
const char = content[prevPos];
if (numSeenBackslashes === 0 && (char === " " || char === " ")) {
prevPos--;
} else if (char === "\\") {
numSeenBackslashes++;
prevPos--;
} else {
break;
}
}
return numSeenBackslashes % 2 === 1;
}
// src/utils/calculateNormalStringPadding.ts
function calculateNormalStringPadding(source, stream2) {
let paddingTracker;
if (stream2.hasNext("SSTRING_START" /* SSTRING_START */)) {
paddingTracker = new PaddingTracker(source, stream2, "SSTRING_END" /* SSTRING_END */);
} else if (stream2.hasNext("DSTRING_START" /* DSTRING_START */)) {
paddingTracker = new PaddingTracker(source, stream2, "DSTRING_END" /* DSTRING_END */);
} else {
return [];
}
for (let fragmentIndex = 0; fragmentIndex < paddingTracker.fragments.length; fragmentIndex++) {
const fragment = paddingTracker.fragments[fragmentIndex];
const content = fragment.content;
let lastNonWhitespace = -1;
let pos = 0;
while (pos < content.length) {
if (content[pos] === "\n") {
const startIndex = lastNonWhitespace + 1;
fragment.markPadding(lastNonWhitespace + 1, pos);
const newlinePos = pos;
pos++;
while (pos < content.length && " \n".includes(content[pos]) || content.slice(pos, pos + 2) === "\\\n") {
pos++;
}
const endIndex = pos;
if (isNewlineEscaped(content, newlinePos)) {
const backslashPos = content.lastIndexOf("\\", newlinePos);
fragment.markPadding(backslashPos, endIndex);
} else if (fragmentIndex === 0 && startIndex === 0 || fragmentIndex === paddingTracker.fragments.length - 1 && endIndex === content.length) {
fragment.markPadding(startIndex, endIndex);
} else {
fragment.markPadding(startIndex, newlinePos);
fragment.markLineSeparator(newlinePos);
fragment.markPadding(newlinePos + 1, endIndex);
}
lastNonWhitespace = pos;
} else {
if (content[pos] !== " " && content[pos] !== " ") {
lastNonWhitespace = pos;
}
pos++;
}
}
}
return paddingTracker.computeSourceLocations();
}
// src/utils/calculateTripleQuotedStringPadding.ts
function calculateTripleQuotedStringPadding(source, stream2) {
let paddingTracker;
if (stream2.hasNext("TSSTRING_START" /* TSSTRING_START */)) {
paddingTracker = new PaddingTracker(source, stream2, "TSSTRING_END" /* TSSTRING_END */);
} else if (stream2.hasNext("TDSTRING_START" /* TDSTRING_START */)) {
paddingTracker = new PaddingTracker(source, stream2, "TDSTRING_END" /* TDSTRING_END */);
} else {
return [];
}
const firstFragment = paddingTracker.fragments[0];
const firstContent = firstFragment.content;
const lastFragment = paddingTracker.fragments[paddingTracker.fragments.length - 1];
const lastContent = lastFragment.content;
const sharedIndent = getIndentForFragments(paddingTracker.fragments);
const firstContentLines = splitUnescapedNewlines(firstContent);
if (firstContentLines.length > 1 && isWhitespace(firstContentLines[0])) {
firstFragment.markPadding(0, firstContentLines[0].length + 1);
}
if (!shouldSkipRemovingLastLine(paddingTracker)) {
const lastLines = splitUnescapedNewlines(lastContent);
if (lastLines.length > 1) {
const lastLine = lastLines[lastLines.length - 1];
if (isWhitespace(lastLine)) {
lastFragment.markPadding(lastFragment.content.length - lastLine.length - 1, lastFragment.content.length);
}
}
}
for (const fragment of paddingTracker.fragments) {
for (let i = 0; i < fragment.content.length; i++) {
if (fragment.content[i] === "\n" && isNewlineEscaped(fragment.content, i)) {
const backslashPos = fragment.content.lastIndexOf("\\", i);
let paddingEnd = i;
while (paddingEnd < fragment.content.length && " \n".includes(fragment.content[paddingEnd])) {
paddingEnd++;
}
fragment.markPadding(backslashPos, paddingEnd);
}
const isStartOfLine = i > 0 && fragment.content[i - 1] === "\n";
if (isStartOfLine) {
const paddingStart = i;
const paddingEnd = i + sharedIndent.length;
if (fragment.content.slice(paddingStart, paddingEnd) === sharedIndent) {
fragment.markPadding(paddingStart, paddingEnd);
}
}
}
}
return paddingTracker.computeSourceLocations();
}
function getIndentForFragments(fragments) {
let hasSeenLine = false;
let smallestIndent = null;
for (const fragment of fragments) {
const lines = fragment.content.split("\n");
for (let i = 1; i < lines.length; i++) {
const line = lines[i];
const indent = getLineIndent(line);
if (!hasSeenLine && indent.length === 0 && line.length > 0) {
return "";
}
hasSeenLine = true;
const isFullLine = i < lines.length - 1 || fragment.index === fragments.length - 1;
if (indent.length === 0 || isFullLine && indent === line) {
continue;
}
if (smallestIndent === null || indent.length < smallestIndent.length) {
smallestIndent = indent;
}
}
}
if (smallestIndent === null) {
return "";
}
return smallestIndent;
}
function shouldSkipRemovingLastLine(paddingTracker) {
if (paddingTracker.fragments.length !== 1) {
return false;
}
const lines = paddingTracker.fragments[0].content.split("\n");
return lines.length === 2 && isWhitespace(lines[0]) && isWhitespace(lines[1]);
}
function getLineIndent(line) {
const match = /[^\n\S]*/.exec(line);
if (match) {
return match[0];
} else {
return "";
}
}
function splitUnescapedNewlines(str) {
const lines = [""];
let numBackslashes = 0;
let isEatingWhitespace = false;
for (const chr of str) {
if (chr === "\n" && numBackslashes % 2 === 0 && !isEatingWhitespace) {
lines.push("");
} else {
if (chr === "\n" && numBackslashes % 2 === 1) {
isEatingWhitespace = true;
}
if (chr === "\\") {
numBackslashes++;
} else {
numBackslashes = 0;
}
if (!" \n".includes(chr)) {
isEatingWhitespace = false;
}
lines[lines.length - 1] += chr;
}
}
return lines;
}
function isWhitespace(line) {
for (let i = 0; i < line.length; i++) {
if (!" \n".includes(line[i]) && !(line[i] === "\\" && line[i + 1] === "\n")) {
return false;
}
}
return true;
}
// src/lex.ts
var DEFAULT_OPTIONS = {
useCS2: false
};
var CoffeeLexError = class extends SyntaxError {
constructor(message, index, context) {
super(message);
this.index = index;
this.context = context;
}
};
function lex(source, options = DEFAULT_OPTIONS) {
let location;
let previousLocation;
const tokens = [];
const pending = new BufferedStream(stream(source, 0, options));
do {
pending.unshift(...calculateNormalStringPadding(source, pending));
pending.unshift(...calculateTripleQuotedStringPadding(source, pending));
pending.unshift(...calculateHeregexpPadding(source, pending));
pending.unshift(...combinedLocationsForNegatedOperators(pending, source));
location = pending.shift();
if (previousLocation && previousLocation.type !== "SPACE" /* SPACE */) {
tokens.push(new SourceToken(previousLocation.type, previousLocation.index, location.index));
}
previousLocation = location;
} while (location.type !== "EOF" /* EOF */);
return new SourceTokenList(tokens);
}
function combinedLocationsForNegatedOperators(stream2, source) {
if (!stream2.hasNext("OPERATOR" /* OPERATOR */)) {
return [];
}
const locationsToRestore = [];
function shift() {
const location = stream2.shift();
locationsToRestore.push(location);
return location;
}
const not = shift();
const space = shift();
const text = source.slice(not.index, space.index);
let operator;
if (text === "not") {
if (space.type === "SPACE" /* SPACE */) {
operator = shift();
} else {
return locationsToRestore;
}
} else if (text === "!") {
if (space.type === "SPACE" /* SPACE */) {
operator = shift();
} else {
operator = space;
}
} else {
return locationsToRestore;
}
const next = stream2.peek();
const op = source.slice(operator.index, next.index);
switch (op) {
case "in":
case "of":
return [new SourceLocation("RELATION" /* RELATION */, not.index)];
case "instanceof":
return [new SourceLocation("OPERATOR" /* OPERATOR */, not.index)];
}
return locationsToRestore;
}
var REGEXP_FLAGS = ["i", "g", "m", "u", "y"];
var STRING = [
"SSTRING_END" /* SSTRING_END */,
"DSTRING_END" /* DSTRING_END */,
"TSSTRING_END" /* TSSTRING_END */,
"TDSTRING_END" /* TDSTRING_END */
];
var CALLABLE = [
"IDENTIFIER" /* IDENTIFIER */,
"CALL_END" /* CALL_END */,
"RPAREN" /* RPAREN */,
"RBRACKET" /* RBRACKET */,
"EXISTENCE" /* EXISTENCE */,
"AT" /* AT */,
"THIS" /* THIS */,
"SUPER" /* SUPER */
];
var INDEXABLE = CALLABLE.concat([
"NUMBER" /* NUMBER */,
...STRING,
"REGEXP" /* REGEXP */,
"HEREGEXP_END" /* HEREGEXP_END */,
"BOOL" /* BOOL */,
"NULL" /* NULL */,
"UNDEFINED" /* UNDEFINED */,
"RBRACE" /* RBRACE */,
"PROTO" /* PROTO */
]);
var NOT_REGEXP = INDEXABLE.concat([
"INCREMENT" /* INCREMENT */,
"DECREMENT" /* DECREMENT */
]);
var IDENTIFIER_PATTERN = /^(?!\d)((?:(?!\s)[$\w\x7f-\uffff])+)/;
var CSX_IDENTIFIER_PATTERN = /^(?!\d)((?:(?!\s)[.\-$\w\x7f-\uffff])+)/;
var NUMBER_PATTERN = /^0b[01]+|^0o[0-7]+|^0x[\da-f]+|^\d*\.?\d+(?:e[+-]?\d+)?/i;
var SPACE_PATTERN = /^[^\n\r\S]+/;
var REGEXP_PATTERN = /^\/(?!\/)((?:[^[/\n\\]|\\[^\n]|\[(?:\\[^\n]|[^\]\n\\])*\])*)(\/)?/;
var YIELDFROM_PATTERN = /^yield[^\n\r\S]+from/;
var OPERATORS = [
"===",
"==",
"!==",
"!=",
"=",
"+=",
"-=",
"/=",
"*=",
"%=",
"%%=",
"||=",
"&&=",
"^=",
"or=",
"and=",
"?=",
"|=",
"&=",
"~=",
"<<=",
">>>=",
">>=",
"++",
"--",
"+",
"-",
"//",
"/",
"*",
"%",
"%%",
"||",
"&&",
"^",
"!",
"?",
"|",
"&",
"~",
"<<",
">>>",
">>",
"<=",
"<",
">=",
">",
"::"
];
function stream(source, index = 0, options = DEFAULT_OPTIONS) {
let location = new SourceLocation("NORMAL" /* NORMAL */, index);
const contextStack = [];
let start = index;
const locations = [];
function currentContext() {
return contextStack[contextStack.length - 1] || null;
}
function currentContextType() {
const context = currentContext();
return context ? context.type : null;
}
function lexError(message) {
throw new CoffeeLexError(`${message} at ${index}`, index, currentContext() ?? void 0);
}
return function step() {
const lastLocation = location;
let shouldStepAgain = false;
do {
start = index;
if (index >= source.length) {
setType("EOF" /* EOF */);
}
switch (location.type) {
case "NORMAL" /* NORMAL */:
case "SPACE" /* SPACE */:
case "IDENTIFIER" /* IDENTIFIER */:
case "DOT" /* DOT */:
case "NUMBER" /* NUMBER */:
case "OPERATOR" /* OPERATOR */:
case "INCREMENT" /* INCREMENT */:
case "DECREMENT" /* DECREMENT */:
case "COMMA" /* COMMA */:
case "LPAREN" /* LPAREN */:
case "RPAREN" /* RPAREN */:
case "CALL_START" /* CALL_START */:
case "CALL_END" /* CALL_END */:
case "NEW" /* NEW */:
case "LBRACE" /* LBRACE */:
case "RBRACE" /* RBRACE */:
case "LBRACKET" /* LBRACKET */:
case "RBRACKET" /* RBRACKET */:
case "NEWLINE" /* NEWLINE */:
case "COLON" /* COLON */:
case "FUNCTION" /* FUNCTION */:
case "THIS" /* THIS */:
case "AT" /* AT */:
case "SEMICOLON" /* SEMICOLON */:
case "IF" /* IF */:
case "ELSE" /* ELSE */:
case "THEN" /* THEN */:
case "FOR" /* FOR */:
case "OWN" /* OWN */:
case "WHILE" /* WHILE */:
case "BOOL" /* BOOL */:
case "NULL" /* NULL */:
case "UNDEFINED" /* UNDEFINED */:
case "REGEXP" /* REGEXP */:
case "SSTRING_END" /* SSTRING_END */:
case "DSTRING_END" /* DSTRING_END */:
case "TSSTRING_END" /* TSSTRING_END */:
case "TDSTRING_END" /* TDSTRING_END */:
case "INTERPOLATION_START" /* INTERPOLATION_START */:
case "SUPER" /* SUPER */:
case "TRY" /* TRY */:
case "CATCH" /* CATCH */:
case "FINALLY" /* FINALLY */:
case "SWITCH" /* SWITCH */:
case "WHEN" /* WHEN */:
case "BREAK" /* BREAK */:
case "CONTINUE" /* CONTINUE */:
case "EXISTENCE" /* EXISTENCE */:
case "CLASS" /* CLASS */:
case "PROTO" /* PROTO */:
case "RANGE" /* RANGE */:
case "DELETE" /* DELETE */:
case "RETURN" /* RETURN */:
case "RELATION" /* RELATION */:
case "LOOP" /* LOOP */:
case "DO" /* DO */:
case "YIELD" /* YIELD */:
case "YIELDFROM" /* YIELDFROM */:
case "THROW" /* THROW */:
case "EXTENDS" /* EXTENDS */:
case "IMPORT" /* IMPORT */:
case "EXPORT" /* EXPORT */:
case "DEFAULT" /* DEFAULT */:
case "CSX_OPEN_TAG_START" /* CSX_OPEN_TAG_START */:
case "CSX_CLOSE_TAG_START" /* CSX_CLOSE_TAG_START */:
case "CONTINUATION" /* CONTINUATION */:
if (consume(SPACE_PATTERN)) {
setType("SPACE" /* SPACE */);
} else if (consume("\n")) {
setType("NEWLINE" /* NEWLINE */);
} else if (consume("...") || consume("..")) {
setType("RANGE" /* RANGE */);
} else if (consume(NUMBER_PATTERN)) {
setType("NUMBER" /* NUMBER */);
} else if (consume(".")) {
setType("DOT" /* DOT */);
} else if (consume('"""')) {
contextStack.push({
type: "STRING" /* STRING */,
allowComments: false,
allowInterpolations: true,
endingDelimiter: '"""',
endSourceType: "TDSTRING_END" /* TDSTRING_END */
});
setType("TDSTRING_START" /* TDSTRING_START */);
} else if (consume('"')) {
contextStack.push({
type: "STRING" /* STRING */,
allowComments: false,
allowInterpolations: true,
endingDelimiter: '"',
endSourceType: "DSTRING_END" /* DSTRING_END */
});
setType("DSTRING_START" /* DSTRING_START */);
} else if (consume("'''")) {
contextStack.push({
type: "STRING" /* STRING */,
allowComments: false,
allowInterpolations: false,
endingDelimiter: "'''",
endSourceType: "TSSTRING_END" /* TSSTRING_END */
});
setType("TSSTRING_START" /* TSSTRING_START */);
} else if (consume("'")) {
contextStack.push({
type: "STRING" /* STRING */,
allowComments: false,
allowInterpolations: false,
endingDelimiter: "'",
endSourceType: "SSTRING_END" /* SSTRING_END */
});
setType("SSTRING_START" /* SSTRING_START */);
} else if (consume(/^###[^#]/)) {
setType("HERECOMMENT" /* HERECOMMENT */);
} else if (consume("#")) {
setType("COMMENT" /* COMMENT */);
} else if (consume("///")) {
contextStack.push({
type: "STRING" /* STRING */,
allowComments: true,
allowInterpolations: true,
endingDelimiter: "///",
endSourceType: "HEREGEXP_END" /* HEREGEXP_END */
});
setType("HEREGEXP_START" /* HEREGEXP_START */);
} else if (consume("(")) {
if (CALLABLE.indexOf(location.type) >= 0) {
contextStack.push({
type: "PAREN" /* PAREN */,
sourceType: "CALL_START" /* CALL_START */
});
setType("CALL_START" /* CALL_START */);
} else {
contextStack.push({
type: "PAREN" /* PAREN */,
sourceType: "LPAREN" /* LPAREN */
});
setType("LPAREN" /* LPAREN */);
}
} else if (consume(")")) {
const context = contextStack.pop();
if (!context || context.type !== "PAREN" /* PAREN */) {
throw lexError(`unexpected ')'`);
}
const { sourceType } = context;
switch (sourceType) {
case "LPAREN" /* LPAREN */:
setType("RPAREN" /* RPAREN */);
break;
case "CALL_START" /* CALL_START */:
setType("CALL_END" /* CALL_END */);
break;
default:
throw lexError(`unexpected token type for '(' matching ')': ${sourceType ? sourceType.toString() : "??"}`);
}
} else if (consume("[")) {
setType("LBRACKET" /* LBRACKET */);
} else if (consume("]")) {
setType("RBRACKET" /* RBRACKET */);
} else if (consume("{")) {
contextStack.push({ type: "BRACE" /* BRACE */ });
setType("LBRACE" /* LBRACE */);
} else if (consume("}")) {
if (currentContextType() === "INTERPOLATION" /* INTERPOLATION */) {
popInterpolation();
} else if (currentContextType() === "BRACE" /* BRACE */) {
contextStack.pop();
setType("RBRACE" /* RBRACE */);
} else {
throw lexError(`Unexpected context type: ${currentContextType()}`);
}
} else if (consumeCSXOpenTagStart()) {
contextStack.push({ type: "CSX_OPEN_TAG" /* CSX_OPEN_TAG */ });
setType("CSX_OPEN_TAG_START" /* CSX_OPEN_TAG_START */);
} else if (currentContextType() === "CSX_OPEN_TAG" /* CSX_OPEN_TAG */ && consume(">")) {
contextStack.pop();
setType("CSX_OPEN_TAG_END" /* CSX_OPEN_TAG_END */);
} else if (currentContextType() === "CSX_OPEN_TAG" /* CSX_OPEN_TAG */ && consume("/>")) {
contextStack.pop();
setType("CSX_SELF_CLOSING_TAG_END" /* CSX_SELF_CLOSING_TAG_END */);
} else if (currentContextType() === "CSX_CLOSE_TAG" /* CSX_CLOSE_TAG */ && consume(">")) {
contextStack.pop();
setType("CSX_CLOSE_TAG_END" /* CSX_CLOSE_TAG_END */);
} else if (consumeAny(["->", "=>"])) {
setType("FUNCTION" /* FUNCTION */);
} else if (consumeRegexp()) {
setType("REGEXP" /* REGEXP */);
} else if (consume("::")) {
setType("PROTO" /* PROTO */);
} else if (consume(":")) {
setType("COLON" /* COLON */);
} else if (consume(",")) {
setType("COMMA" /* COMMA */);
} else if (consume("@")) {
setType("AT" /* AT */);
} else if (consume(";")) {
setType("SEMICOLON" /* SEMICOLON */);
} else if (consume("```")) {
setType("HEREJS" /* HEREJS */);
} else if (consume("`")) {
setType("JS" /* JS */);
} else if (consumeAny(OPERATORS)) {
if (consumed() === "?") {
setType("EXISTENCE" /* EXISTENCE */);
} else if (consumed() === "++") {
setType("INCREMENT" /* INCREMENT */);
} else if (consumed() === "--") {
setType("DECREMENT" /* DECREMENT */);
} else {
setType("OPERATOR" /* OPERATOR */);
}
} else if (consume(YIELDFROM_PATTERN)) {
setType("YIELDFROM" /* YIELDFROM */);
} else if (currentContextType() === "CSX_OPEN_TAG" /* CSX_OPEN_TAG */ && consume(CSX_IDENTIFIER_PATTERN)) {
setType("IDENTIFIER" /* IDENTIFIER */);
} else if (consume(IDENTIFIER_PATTERN)) {
let prevLocationIndex = locations.length - 1;
while (prevLocationIndex > 0 && (locations[prevLocationIndex].type === "NEWLINE" /* NEWLINE */ || locations[prevLocationIndex].type === "SPACE" /* SPACE */)) {
prevLocationIndex--;
}
const prev = locations[prevLocationIndex];
const nextIsColon = match(/^\s*:/);
if (nextIsColon || prev && (prev.type === "DOT" /* DOT */ || prev.type === "PROTO" /* PROTO */ || prev.type === "AT" /* AT */ && prevLocationIndex === locations.length - 1)) {
setType("IDENTIFIER" /* IDENTIFIER */);
} else {
switch (consumed()) {
case "if":
case "unless":
setType("IF" /* IF */);
break;
case "else":
setType("ELSE" /* ELSE */);
break;
case "return":
setType("RETURN" /* RETURN */);
break;
case "for":
setType("FOR" /* FOR */);
break;
case "own":
setType("OWN" /* OWN */);
break;
case "while":
case "until":
setType("WHILE" /* WHILE */);
break;
case "loop":
setType("LOOP" /* LOOP */);
break;
case "then":
setType("THEN" /* THEN */);
break;
case "switch":
setType("SWITCH" /* SWITCH */);
break;
case "when":
setType("WHEN" /* WHEN */);
break;
case "null":
setType("NULL" /* NULL */);
break;
case "undefined":
setType("UNDEFINED" /* UNDEFINED */);
break;
case "this":
setType("THIS" /* THIS */);
break;
case "new":
setType("NEW" /* NEW */);
break;
case "super":
setType("SUPER" /* SUPER */);
break;
case "true":
case "false":
case "yes":
case "no":
case "on":
case "off":
setType("BOOL" /* BOOL */);
break;
case "and":
case "or":
case "not":
case "is":
case "isnt":
case "instanceof":
setType("OPERATOR" /* OPERATOR */);
break;
case "class":
setType("CLASS" /* CLASS */);
break;
case "break":
setType("BREAK" /* BREAK */);
break;
case "continue":
setType("CONTINUE" /* CONTINUE */);
break;
case "try":
setType("TRY" /* TRY */);
break;
case "catch":
setType("CATCH" /* CATCH */);
break;
case "finally":
setType("FINALLY" /* FINALLY */);
break;
case "delete":
setType("DELETE" /* DELETE */);
break;
case "in":
case "of":
setType("RELATION" /* RELATION */);
break;
case "do":
setType("DO" /* DO */);
break;
case "yield":
setType("YIELD" /* YIELD */);
break;
case "throw":
setType("THROW" /* THROW */);
break;
case "extends":
setType("EXTENDS" /* EXTENDS */);
break;
case "import":
setType("IMPORT" /* IMPORT */);
break;
case "export":
setType("EXPORT" /* EXPORT */);
break;
case "default":
setType("DEFAULT" /* DEFAULT */);
break;
default:
setType("IDENTIFIER" /* IDENTIFIER */);
}
}
} else if (consume("\\")) {
setType("CONTINUATION" /* CONTINUATION */);
} else {
setType("UNKNOWN" /* UNKNOWN */);
}
break;
case "SSTRING_START" /* SSTRING_START */:
case "DSTRING_START" /* DSTRING_START */:
case "TSSTRING_START" /* TSSTRING_START */:
case "TDSTRING_START" /* TDSTRING_START */:
case "HEREGEXP_START" /* HEREGEXP_START */:
setType("STRING_CONTENT" /* STRING_CONTENT */);
break;
case "STRING_CONTENT" /* STRING_CONTENT */: {
const context = currentContext();
if (!context || context.type !== "STRING" /* STRING */) {
throw lexError("Unexpected STRING_CONTENT without anything on the string stack.");
}
const {
allowComments,
allowInterpolations,
endingDelimiter,
endSourceType
} = context;
if (consume("\\")) {
index++;
} else if (consume(endingDelimiter)) {
contextStack.pop();
setType(endSourceType);
} else if (allowInterpolations && consume("#{")) {
pushInterpolation();
} else if (options.useCS2 && allowComments && source[index - 1].match(/\s/) && match("#") && !match("#{")) {
setType("HEREGEXP_COMMENT" /* HEREGEXP_COMMENT */);
} else {
index++;
}
break;
}
case "COMMENT" /* COMMENT */:
if (consume("\n")) {
setType("NEWLINE" /* NEWLINE */);
} else {
index++;
}
break;
case "HERECOMMENT" /* HERECOMMENT */:
if (consume("###")) {
setType("NORMAL" /* NORMAL */);
} else {
index++;
}
break;
case "HEREGEXP_COMMENT" /* HEREGEXP_COMMENT */:
if (consume("\n")) {
setType("STRING_CONTENT" /* STRING_CONTENT */);
} else {
index++;
}
break;
case "INTERPOLATION_END" /* INTERPOLATION_END */: {
const context = contextStack.pop();
if (!context || context.type !== "INTERPOLATION" /* INTERPOLATION */) {
throw lexError(`found interpolation end without any interpolation start`);
}
setType(context.interpolationType);
break;
}
case "HEREGEXP_END" /* HEREGEXP_END */:
while (consumeAny(REGEXP_FLAGS)) {
}
setType("NORMAL" /* NORMAL */);
break;
case "JS" /* JS */:
if (consume("\\")) {
index++;
} else if (consume("`")) {
setType("NORMAL" /* NORMAL */);
} else {
index++;
}
break;
case "HEREJS" /* HEREJS */:
if (consume("\\")) {
index++;
} else if (consume("```")) {
setType("NORMAL" /* NORMAL */);
} else {
index++;
}
break;
case "CSX_OPEN_TAG_END" /* CSX_OPEN_TAG_END */:
setType("CSX_BODY" /* CSX_BODY */);
contextStack.push({ type: "CSX_BODY" /* CSX_BODY */ });
break;
case "CSX_BODY" /* CSX_BODY */: {
if (consume("</")) {
contextStack.pop();
setType("CSX_CLOSE_TAG_START" /* CSX_CLOSE_TAG_START */);
contextStack.push({ type: "CSX_CLOSE_TAG" /* CSX_CLOSE_TAG */ });
} else if (consumeCSXOpenTagStart()) {
setType("CSX_OPEN_TAG_START" /* CSX_OPEN_TAG_START */);
contextStack.push({ type: "CSX_OPEN_TAG" /* CSX_OPEN_TAG */ });
} else if (consume("{")) {
pushInterpolation();
} else {
index++;
}
break;
}
case "CSX_SELF_CLOSING_TAG_END" /* CSX_SELF_CLOSING_TAG_END */:
case "CSX_CLOSE_TAG_END" /* CSX_CLOSE_TAG_END */:
if (currentContextType() === "CSX_BODY" /* CSX_BODY */) {
setType("CSX_BODY" /* CSX_BODY */);
} else {
setType("NORMAL" /* NORMAL */);
}
break;
case "EOF" /* EOF */: {
const context = currentContext();
if (context !== null) {
throw lexError(`unexpected EOF while in context ${context.type}`);
}
break;
}
case "UNKNOWN" /* UNKNOWN */:
index = source.length;
break;
case "STRING_LINE_SEPARATOR" /* STRING_LINE_SEPARATOR */:
case "STRING_PADDING" /* STRING_PADDING */:
throw lexError(`unexpected source type at offset ${location.index}: ${location.type}`);
default:
assertNever(location.type, `unexpected source type at offset ${location.index}: ${location.type}`);
}
shouldStepAgain = location.type === "NORMAL" /* NORMAL */ || location === lastLocation && location.type !== "EOF" /* EOF */;
} while (shouldStepAgain);
locations.push(location);
return location;
};
function consumeAny(strings) {
return strings.some((string) => consume(string));
}
function consume(value) {
const matchData = match(value);
if (matchData) {
index += matchData[0].length;
return true;
} else {
return false;
}
}
function consumeRegexp() {
const matchData = match(REGEXP_PATTERN);
if (!matchData) {
return false;
}
const [regex, , closed] = matchData;
let prev = locations[locations.length - 1];
if (prev) {
let spaced = false;
if (prev.type === "SPACE" /* SPACE */) {
spaced = true;
prev = locations[locations.length - 2];
}
if (spaced && CALLABLE.indexOf(prev.type) >= 0) {
if (!closed || /^\/=?\s/.test(regex)) {
return false;
}
} else if (NOT_REGEXP.indexOf(prev.type) >= 0) {
return false;
}
}
if (!closed) {
throw lexError("missing / (unclosed regex)");
}
index += regex.length;
while (consumeAny(REGEXP_FLAGS)) {
}
return true;
}
function consumeCSXOpenTagStart() {
if (!match("<")) {
return false;
}
if (source[index + 1] !== ">" && !source.slice(index + 1).match(CSX_IDENTIFIER_PATTERN)) {
return false;
}
const contextType = currentContextType();
if (contextType !== "CSX_BODY" /* CSX_BODY */ && contextType !== "CSX_OPEN_TAG" /* CSX_OPEN_TAG */ && [
"ID