key-value-file
Version:
A simple key/value file parser/writer
162 lines • 5.08 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", { value: true });
exports.Tokenizer = void 0;
const string_walker_1 = require("string-walker");
const token_1 = require("./token");
class Tokenizer extends string_walker_1.StringWalker {
constructor(data) {
super(data, true);
this._tokens = [];
this.spaceAndTab = [9, 32];
}
get tokens() {
return this._tokens;
}
tokenize() {
while (!this.isEof()) {
this.readKey();
}
return this._tokens;
}
eatWhitespace() {
if (!this.isEof()) {
const start = this.cursor;
this.eatNewline();
this.eatSpacesAndTabs();
if (start !== this.cursor) {
this.eatWhitespace();
}
}
}
eatNewline() {
while (!this.isEof() && this.isNewline()) {
this.pushCurrentToken(token_1.TokenType.Newline);
this.next();
}
}
eatSpacesAndTabs() {
while (!this.isEof() && this.isSpaceOrTab()) {
this.pushCurrentToken(token_1.TokenType.Whitespace);
this.next();
}
}
pushToken(type, value) {
this._tokens.push({ type, value });
}
pushCurrentToken(type) {
this._tokens.push({ type, value: this.currentChar() });
}
readKey() {
this.eatWhitespace();
this.readComment();
if (this.isEof()) {
return;
}
let endpos = NaN;
if (this.isQuoteChar()) {
endpos = this.findStringEnd();
}
else {
endpos = this.findNextOf([' ', '\t', '=']);
}
if (isNaN(endpos)) {
throw new Error('Syntax error');
}
const key = this.substring(this.cursor, endpos);
this.pushToken(token_1.TokenType.Key, key);
this.moveTo(endpos);
this.eatSpacesAndTabs();
if (this.currentChar() !== '=') {
throw new Error(`Expected "=" after key, got "${this.currentChar()}"`);
}
this.pushCurrentToken(token_1.TokenType.Delimiter);
this.next();
this.readValue();
}
readValue() {
this.eatSpacesAndTabs();
if (this.isEof() || this.isNewline()) {
this.pushToken(token_1.TokenType.Value, '');
if (this.isNewline()) {
this.pushToken(token_1.TokenType.Newline, '\n');
}
this.next();
}
else if (this.isCommentStart()) {
this.pushToken(token_1.TokenType.Value, '');
this.readComment();
}
else {
let endpos = this.len;
if (this.isQuoteChar()) {
endpos = this.findStringEnd();
}
else {
endpos = this.findNextOf(['#', '\n']);
}
if (isNaN(endpos)) {
endpos = this.len;
}
if (this.isCommentStart(endpos)) {
// Early exit
return this.handleTrailingComment(endpos);
}
const val = this.substring(this.cursor, endpos);
this.pushToken(token_1.TokenType.Value, val);
this.moveTo(endpos);
this.readComment();
}
}
isQuoteChar(n = 0) {
return [34, 39].includes(n ? this.at(n) : this.current());
}
isCommentStart(n = 0) {
return (n ? this.at(n) : this.current()) === 35;
}
isNewline(n = 0) {
return (n ? this.at(n) : this.current()) === 10;
}
isSpaceOrTab(n = 0) {
return this.spaceAndTab.includes(n ? this.at(n) : this.current());
}
findStringEnd() {
if (!this.isQuoteChar()) {
throw new Error(`Expected current charachter to be ' or ", got ${this.currentChar()}`);
}
const endpos = this.findNext(this.current());
if (isNaN(endpos)) {
throw new Error(`Unterminated string literal`);
}
return endpos + 1;
}
handleTrailingComment(endpos) {
const prev = this.cursor;
this.moveTo(endpos);
if (this.spaceAndTab.includes(this.behind())) {
let offset = 1;
while (this.spaceAndTab.includes(this.behind(offset))) {
offset += 1;
}
const newEndPos = endpos - offset + 1;
const value = this.substring(prev, newEndPos);
this.pushToken(token_1.TokenType.Value, value);
this.moveTo(newEndPos);
this.eatSpacesAndTabs();
this.readComment();
}
}
readComment() {
if (this.isCommentStart()) {
let endpos = this.findNext('\n');
if (isNaN(endpos)) {
endpos = this.len;
}
const comment = this.substring(this.cursor, endpos);
this.pushToken(token_1.TokenType.Comment, comment);
this.moveTo(endpos);
this.eatWhitespace();
}
}
}
exports.Tokenizer = Tokenizer;
//# sourceMappingURL=tokenizer.js.map