typesxml
Version:
Open source XML library written in TypeScript
291 lines • 9.85 kB
JavaScript
/*******************************************************************************
* Copyright (c) 2023-2026 Maxprograms.
*
* This program and the accompanying materials
* are made available under the terms of the Eclipse License 1.0
* which accompanies this distribution, and is available at
* https://www.eclipse.org/org/documents/epl-v10.html
*
* Contributors:
* Maxprograms - initial API and implementation
*******************************************************************************/
import { NeedMoreDataError } from "../NeedMoreDataError.js";
export class JsonTokenizer {
buffer = '';
offset = 0;
finished = false;
lookahead;
enqueue(chunk) {
if (chunk.length === 0) {
return;
}
this.buffer += chunk;
}
markFinished() {
this.finished = true;
}
nextToken() {
if (this.lookahead) {
const token = this.lookahead;
this.lookahead = undefined;
return token;
}
return this.readToken();
}
peekToken() {
if (!this.lookahead) {
this.lookahead = this.readToken();
}
return this.lookahead;
}
readToken() {
this.skipWhitespace();
if (this.offset >= this.buffer.length) {
if (this.finished) {
return { type: 'eof' };
}
throw new NeedMoreDataError();
}
const char = this.buffer[this.offset];
switch (char) {
case '{':
this.offset++;
this.compactBuffer();
return { type: 'braceOpen' };
case '}':
this.offset++;
this.compactBuffer();
return { type: 'braceClose' };
case '[':
this.offset++;
this.compactBuffer();
return { type: 'bracketOpen' };
case ']':
this.offset++;
this.compactBuffer();
return { type: 'bracketClose' };
case ':':
this.offset++;
this.compactBuffer();
return { type: 'colon' };
case ',':
this.offset++;
this.compactBuffer();
return { type: 'comma' };
case '"':
this.offset++;
return this.makeStringToken();
case 't':
this.readLiteral('true');
return { type: 'boolean', value: true };
case 'f':
this.readLiteral('false');
return { type: 'boolean', value: false };
case 'n':
this.readLiteral('null');
return { type: 'null' };
default:
if (char === '-' || this.isDigit(char)) {
const value = this.readNumber();
return { type: 'number', value };
}
throw new Error('Unexpected character in JSON stream: ' + char);
}
}
makeStringToken() {
const openingOffset = this.offset - 1;
let result = '';
let escaping = false;
while (true) {
if (this.offset >= this.buffer.length) {
if (this.finished) {
throw new Error('Unterminated string literal');
}
this.offset = openingOffset;
throw new NeedMoreDataError();
}
const char = this.buffer[this.offset];
this.offset++;
if (escaping) {
escaping = false;
switch (char) {
case '"':
result += '"';
break;
case '\\':
result += '\\';
break;
case '/':
result += '/';
break;
case 'b':
result += '\b';
break;
case 'f':
result += '\f';
break;
case 'n':
result += '\n';
break;
case 'r':
result += '\r';
break;
case 't':
result += '\t';
break;
case 'u':
result += this.readUnicodeEscape(openingOffset);
break;
default:
throw new Error('Invalid escape sequence in string literal');
}
continue;
}
if (char === '\\') {
escaping = true;
continue;
}
if (char === '"') {
this.compactBuffer();
return { type: 'string', value: result };
}
result += char;
}
}
readUnicodeEscape(resetOffset) {
if (this.offset + 4 > this.buffer.length) {
if (this.finished) {
throw new Error('Unterminated unicode escape sequence');
}
this.offset = resetOffset;
throw new NeedMoreDataError();
}
const hexDigits = this.buffer.substring(this.offset, this.offset + 4);
if (!/^[0-9A-Fa-f]{4}$/.test(hexDigits)) {
throw new Error('Invalid unicode escape sequence: \\u' + hexDigits);
}
this.offset += 4;
const codePoint = Number.parseInt(hexDigits, 16);
return String.fromCodePoint(codePoint);
}
readNumber() {
const startOffset = this.offset;
let index = this.offset;
const charAt = (position) => {
if (position >= this.buffer.length) {
if (this.finished) {
return undefined;
}
this.offset = startOffset;
throw new NeedMoreDataError();
}
return this.buffer[position];
};
const firstChar = charAt(index);
if (firstChar === undefined) {
throw new Error('Invalid number literal');
}
if (firstChar === '-') {
index++;
}
const leadingDigit = charAt(index);
if (leadingDigit === undefined || !this.isDigit(leadingDigit)) {
throw new Error('Invalid number literal');
}
if (leadingDigit === '0') {
index++;
}
else {
do {
index++;
const nextChar = charAt(index);
if (nextChar === undefined || !this.isDigit(nextChar)) {
break;
}
} while (true);
}
const fractionIndicator = charAt(index);
if (fractionIndicator === '.') {
index++;
const fractionDigit = charAt(index);
if (fractionDigit === undefined || !this.isDigit(fractionDigit)) {
throw new Error('Invalid fractional part in number literal');
}
do {
index++;
const nextChar = charAt(index);
if (nextChar === undefined || !this.isDigit(nextChar)) {
break;
}
} while (true);
}
const exponentIndicator = charAt(index);
if (exponentIndicator === 'e' || exponentIndicator === 'E') {
index++;
const signChar = charAt(index);
if (signChar === '+' || signChar === '-') {
index++;
}
const exponentDigit = charAt(index);
if (exponentDigit === undefined || !this.isDigit(exponentDigit)) {
throw new Error('Invalid exponent in number literal');
}
do {
index++;
const nextChar = charAt(index);
if (nextChar === undefined || !this.isDigit(nextChar)) {
break;
}
} while (true);
}
const fragment = this.buffer.substring(startOffset, index);
const value = Number(fragment);
if (!Number.isFinite(value)) {
throw new Error('Invalid number literal: ' + fragment);
}
this.offset = index;
this.compactBuffer();
return value;
}
readLiteral(expected) {
const startOffset = this.offset;
const endOffset = this.offset + expected.length;
if (endOffset > this.buffer.length) {
if (this.finished) {
throw new Error('Unexpected end of JSON stream while reading literal');
}
this.offset = startOffset;
throw new NeedMoreDataError();
}
const fragment = this.buffer.substring(this.offset, endOffset);
if (fragment !== expected) {
throw new Error('Unexpected literal: ' + fragment);
}
this.offset = endOffset;
this.compactBuffer();
}
skipWhitespace() {
while (this.offset < this.buffer.length) {
const char = this.buffer[this.offset];
if (char === ' ' || char === '\n' || char === '\r' || char === '\t') {
this.offset++;
continue;
}
break;
}
this.compactBuffer();
}
isDigit(char) {
return char >= '0' && char <= '9';
}
compactBuffer() {
if (this.offset === 0) {
return;
}
if (this.offset >= 4096) {
this.buffer = this.buffer.substring(this.offset);
this.offset = 0;
}
}
}
//# sourceMappingURL=JsonTokenizer.js.map