3h-ast
Version:
A simple AST generator.
339 lines (335 loc) • 11.4 kB
JavaScript
const tokenizeDefaults = {
normalSymbols: new Set('()[]{}<>+-*/\\?~!@#$%^&=|,.:;'),
twoCharacterSymbols: new Set([
'&&', '||', '??', '==', '>=', '<=', '!=', '<<', '>>',
'++', '--', '**', '+=', '-=', '*=', '/=', '%=',
'&=', '^=', '|=', '|>', '->', '=>', '..',
'//', '/*', '*/',
]),
threeCharacterSymbols: new Set([
'&&=', '||=', '??=', '===', '!==', '<<=', '>>=', '**=',
'...', '>>>',
]),
fourCharacterSymbols: new Set([
'>>>=',
]),
globSymbols: new Set(['"', "'", '`']),
numberCharacters: new Set('0123456789'),
extendedNumberCharacters: new Set('.ABCDEF'),
numberSuffixes: new Set(['D', 'B', 'O', 'H']),
spaceCharacters: new Set(' \r\n\t'),
};
/** dts2md break */
/**
* Split source code into tokens
*/
const tokenize = (source, options) => {
const config = Object.assign({}, tokenizeDefaults, options);
const { normalSymbols, twoCharacterSymbols, threeCharacterSymbols, fourCharacterSymbols, globSymbols, numberCharacters, extendedNumberCharacters, numberSuffixes, spaceCharacters, } = config;
const tokens = [];
let tokenBuffer = source[0];
let globFlag = '';
let state = 0 /* ANY */;
let escapeFlag = false;
// init state
if (globSymbols.has(tokenBuffer)) {
globFlag = tokenBuffer;
}
else if (spaceCharacters.has(tokenBuffer)) {
state = 1 /* SPACE */;
}
else if (numberCharacters.has(tokenBuffer)) {
state = 2 /* NUMBER */;
}
else if (normalSymbols.has(tokenBuffer)) {
tokens.push(tokenBuffer);
tokenBuffer = '';
}
for (let i = 1; i < source.length; i++) {
const character = source[i];
if (globFlag) {
tokenBuffer += character;
if (escapeFlag) {
escapeFlag = false;
}
else {
if (character === globFlag) {
tokens.push(tokenBuffer);
tokenBuffer = '';
globFlag = '';
state = 0 /* ANY */;
}
else {
escapeFlag = (character === '\\');
}
}
continue;
}
if (globSymbols.has(character)) {
if (tokenBuffer) {
tokens.push(tokenBuffer);
}
tokenBuffer = character;
globFlag = character;
escapeFlag = false;
continue;
}
if ((state === 2 /* NUMBER */ || state === 3 /* NUMBER_SUFFIX */)
&& numberSuffixes.has(character)) {
tokenBuffer += character;
state = 3 /* NUMBER_SUFFIX */;
continue;
}
if (state === 0 /* ANY */
&& !tokenBuffer
&& normalSymbols.has(character)
&& tokens.length) {
const lastToken = tokens[tokens.length - 1];
if (lastToken.length === 1
&& twoCharacterSymbols.has(lastToken + character)) {
tokens[tokens.length - 1] += character;
continue;
}
if (lastToken.length === 2
&& threeCharacterSymbols.has(lastToken + character)) {
tokens[tokens.length - 1] += character;
continue;
}
if (lastToken.length === 3
&& fourCharacterSymbols.has(lastToken + character)) {
tokens[tokens.length - 1] += character;
continue;
}
}
let flag = 0 /* ANY */;
if (spaceCharacters.has(character)) {
flag = 1 /* SPACE */;
}
else if (numberCharacters.has(character)
|| (state === 2 /* NUMBER */
&& extendedNumberCharacters.has(character)
&& !tokenBuffer.includes('.'))) {
flag = 2 /* NUMBER */;
}
else if (normalSymbols.has(character)) {
if (tokenBuffer) {
tokens.push(tokenBuffer);
}
tokens.push(character);
tokenBuffer = '';
state = 0 /* ANY */;
continue;
}
if (state === flag) {
tokenBuffer += character;
}
else if (state === 0 /* ANY */
&& flag === 2 /* NUMBER */) {
tokenBuffer += character;
if (tokenBuffer.length > 1) { // \w+\d
continue;
}
}
else {
if (tokenBuffer) {
tokens.push(tokenBuffer);
}
tokenBuffer = character;
}
state = flag;
}
if (tokenBuffer) {
tokens.push(tokenBuffer);
}
return tokens;
};
/** dts2md break */
/**
* Default span symbols:
* '(' -> ')'
* '[' -> ']'
* '{' -> '}'
*/
const astDefaults = Object.assign({}, tokenizeDefaults, {
spanSymbols: new Map([
['(', ')'],
['[', ']'],
['{', '}'],
]),
escapeCharacters: new Map([
['t', '\t'],
['r', '\r'],
['n', '\n'],
['\n', ''],
['\\', '\\'],
['"', '"'],
["'", "'"],
['`', '`'],
]),
});
/** dts2md break */
/**
* Convert tokens to AST nodes
*/
const token2ast = (tokens, options, initOffset = 0, initLine = 1, initColumn = 1, cursorStart = 0, cursorStop = tokens.length) => {
const config = Object.assign({}, astDefaults, options);
const { normalSymbols, twoCharacterSymbols, threeCharacterSymbols, fourCharacterSymbols, globSymbols, numberCharacters, numberSuffixes, spaceCharacters, spanSymbols, escapeCharacters, } = config;
const result = new Array();
let offset = initOffset;
let line = initLine;
let column = initColumn;
for (let mainCursor = cursorStart; mainCursor < cursorStop; mainCursor++) {
const token = tokens[mainCursor];
if (spanSymbols.has(token)) {
const spanStopSymbol = spanSymbols.get(token);
let nested = 0;
let spanStopPosition = mainCursor + 1;
for (; spanStopPosition < tokens.length; spanStopPosition++) {
if (tokens[spanStopPosition] === token) {
nested++;
continue;
}
if (tokens[spanStopPosition] === spanStopSymbol) {
if (!nested) {
break;
}
nested--;
}
}
if (spanStopPosition === tokens.length) {
throw new SyntaxError(`missing ${spanStopSymbol} for ${token} at line ${line} column ${column}`);
}
if (spanStopPosition === mainCursor + 1) {
result.push({
type: 'span',
start: token,
stop: spanStopSymbol,
body: [],
offset,
line,
column,
});
// stop symbol
offset++;
}
else {
const ast = token2ast(tokens, config, offset + 1, // +1 for start symbol
line, column + 1, mainCursor + 1, spanStopPosition);
const body = ast.ast;
result.push({
type: 'span',
start: token,
stop: spanStopSymbol,
body,
offset,
line,
column,
});
offset = ast.stopOffset;
line = ast.stopLine;
column = ast.stopColumn;
}
mainCursor = spanStopPosition;
// mainCursor++ automatically goes here
}
else if (normalSymbols.has(token)
|| twoCharacterSymbols.has(token)
|| threeCharacterSymbols.has(token)
|| fourCharacterSymbols.has(token)) {
result.push({
type: 'symbol',
value: token,
offset,
line,
column,
});
}
else if (globSymbols.has(token[0])) {
let value = token[0];
let escapeFlag = false;
for (let i = 1; i < token.length - 1; i++) {
const character = token[i];
if (escapeFlag) {
if (!escapeCharacters.has(character)) {
if (character === '\r' && token[i + 1] === '\n') {
continue;
}
throw new SyntaxError(`unknown escape character: \\${character}`);
}
value += escapeCharacters.get(character);
escapeFlag = false;
}
else {
if (character === '\\') {
escapeFlag = true;
continue;
}
value += character;
}
}
if (escapeFlag
|| (token[token.length - 1] !== token[0])
|| (token.length === 1)) {
throw new SyntaxError(`missing corresponding ${spanSymbols} at line ${line} column ${column}`);
}
value += token[0];
result.push({
type: 'glob',
value,
offset,
line,
column,
});
}
else if (numberCharacters.has(token[0])) {
let suffixOffset = token.length - 1;
for (; suffixOffset >= 0; suffixOffset--) {
if (!numberSuffixes.has(token[suffixOffset])) {
break;
}
}
suffixOffset++;
result.push({
type: 'number',
value: token.slice(0, suffixOffset),
suffix: token.slice(suffixOffset),
offset,
line,
column,
});
}
else if (!spaceCharacters.has(token[0])) {
result.push({
type: 'word',
value: token,
offset,
line,
column,
});
}
offset += token.length;
for (let i = 0; i < token.length; i++) {
if (token[i] === '\n') {
line++;
}
}
const lastNextLineIndex = token.lastIndexOf('\n');
if (~lastNextLineIndex) {
column = token.length - lastNextLineIndex;
}
else {
column += token.length;
}
}
return {
ast: result,
stopOffset: offset,
stopLine: line,
stopColumn: column,
};
};
/**
* Convert source code into AST nodes
*/
const parse = (source, options) => (token2ast(tokenize(source, options), options));
export { astDefaults, parse, token2ast, tokenize, tokenizeDefaults };