@21epub/epub-thirdparty
Version:
epub-thirdparty
1,025 lines (1,024 loc) • 44.4 kB
JavaScript
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as arrays from '../../../base/common/arrays.js';
import { LineTokens } from '../core/lineTokens.js';
import { Position } from '../core/position.js';
import { Range } from '../core/range.js';
import { TokenMetadata } from '../modes.js';
export function countEOL(text) {
let eolCount = 0;
let firstLineLength = 0;
let lastLineStart = 0;
let eol = 0 /* Unknown */;
for (let i = 0, len = text.length; i < len; i++) {
const chr = text.charCodeAt(i);
if (chr === 13 /* CarriageReturn */) {
if (eolCount === 0) {
firstLineLength = i;
}
eolCount++;
if (i + 1 < len && text.charCodeAt(i + 1) === 10 /* LineFeed */) {
// \r\n... case
eol |= 2 /* CRLF */;
i++; // skip \n
}
else {
// \r... case
eol |= 3 /* Invalid */;
}
lastLineStart = i + 1;
}
else if (chr === 10 /* LineFeed */) {
// \n... case
eol |= 1 /* LF */;
if (eolCount === 0) {
firstLineLength = i;
}
eolCount++;
lastLineStart = i + 1;
}
}
if (eolCount === 0) {
firstLineLength = text.length;
}
return [eolCount, firstLineLength, text.length - lastLineStart, eol];
}
function getDefaultMetadata(topLevelLanguageId) {
return ((topLevelLanguageId << 0 /* LANGUAGEID_OFFSET */)
| (0 /* Other */ << 8 /* TOKEN_TYPE_OFFSET */)
| (0 /* None */ << 11 /* FONT_STYLE_OFFSET */)
| (1 /* DefaultForeground */ << 14 /* FOREGROUND_OFFSET */)
| (2 /* DefaultBackground */ << 23 /* BACKGROUND_OFFSET */)) >>> 0;
}
const EMPTY_LINE_TOKENS = (new Uint32Array(0)).buffer;
export class MultilineTokensBuilder {
constructor() {
this.tokens = [];
}
add(lineNumber, lineTokens) {
if (this.tokens.length > 0) {
const last = this.tokens[this.tokens.length - 1];
const lastLineNumber = last.startLineNumber + last.tokens.length - 1;
if (lastLineNumber + 1 === lineNumber) {
// append
last.tokens.push(lineTokens);
return;
}
}
this.tokens.push(new MultilineTokens(lineNumber, [lineTokens]));
}
}
export class SparseEncodedTokens {
constructor(tokens) {
this._tokens = tokens;
this._tokenCount = tokens.length / 4;
}
toString(startLineNumber) {
let pieces = [];
for (let i = 0; i < this._tokenCount; i++) {
pieces.push(`(${this._getDeltaLine(i) + startLineNumber},${this._getStartCharacter(i)}-${this._getEndCharacter(i)})`);
}
return `[${pieces.join(',')}]`;
}
getMaxDeltaLine() {
const tokenCount = this._getTokenCount();
if (tokenCount === 0) {
return -1;
}
return this._getDeltaLine(tokenCount - 1);
}
getRange() {
const tokenCount = this._getTokenCount();
if (tokenCount === 0) {
return null;
}
const startChar = this._getStartCharacter(0);
const maxDeltaLine = this._getDeltaLine(tokenCount - 1);
const endChar = this._getEndCharacter(tokenCount - 1);
return new Range(0, startChar + 1, maxDeltaLine, endChar + 1);
}
_getTokenCount() {
return this._tokenCount;
}
_getDeltaLine(tokenIndex) {
return this._tokens[4 * tokenIndex];
}
_getStartCharacter(tokenIndex) {
return this._tokens[4 * tokenIndex + 1];
}
_getEndCharacter(tokenIndex) {
return this._tokens[4 * tokenIndex + 2];
}
isEmpty() {
return (this._getTokenCount() === 0);
}
getLineTokens(deltaLine) {
let low = 0;
let high = this._getTokenCount() - 1;
while (low < high) {
const mid = low + Math.floor((high - low) / 2);
const midDeltaLine = this._getDeltaLine(mid);
if (midDeltaLine < deltaLine) {
low = mid + 1;
}
else if (midDeltaLine > deltaLine) {
high = mid - 1;
}
else {
let min = mid;
while (min > low && this._getDeltaLine(min - 1) === deltaLine) {
min--;
}
let max = mid;
while (max < high && this._getDeltaLine(max + 1) === deltaLine) {
max++;
}
return new LineTokens2(this._tokens.subarray(4 * min, 4 * max + 4));
}
}
if (this._getDeltaLine(low) === deltaLine) {
return new LineTokens2(this._tokens.subarray(4 * low, 4 * low + 4));
}
return null;
}
clear() {
this._tokenCount = 0;
}
removeTokens(startDeltaLine, startChar, endDeltaLine, endChar) {
const tokens = this._tokens;
const tokenCount = this._tokenCount;
let newTokenCount = 0;
let hasDeletedTokens = false;
let firstDeltaLine = 0;
for (let i = 0; i < tokenCount; i++) {
const srcOffset = 4 * i;
const tokenDeltaLine = tokens[srcOffset];
const tokenStartCharacter = tokens[srcOffset + 1];
const tokenEndCharacter = tokens[srcOffset + 2];
const tokenMetadata = tokens[srcOffset + 3];
if ((tokenDeltaLine > startDeltaLine || (tokenDeltaLine === startDeltaLine && tokenEndCharacter >= startChar))
&& (tokenDeltaLine < endDeltaLine || (tokenDeltaLine === endDeltaLine && tokenStartCharacter <= endChar))) {
hasDeletedTokens = true;
}
else {
if (newTokenCount === 0) {
firstDeltaLine = tokenDeltaLine;
}
if (hasDeletedTokens) {
// must move the token to the left
const destOffset = 4 * newTokenCount;
tokens[destOffset] = tokenDeltaLine - firstDeltaLine;
tokens[destOffset + 1] = tokenStartCharacter;
tokens[destOffset + 2] = tokenEndCharacter;
tokens[destOffset + 3] = tokenMetadata;
}
newTokenCount++;
}
}
this._tokenCount = newTokenCount;
return firstDeltaLine;
}
split(startDeltaLine, startChar, endDeltaLine, endChar) {
const tokens = this._tokens;
const tokenCount = this._tokenCount;
let aTokens = [];
let bTokens = [];
let destTokens = aTokens;
let destOffset = 0;
let destFirstDeltaLine = 0;
for (let i = 0; i < tokenCount; i++) {
const srcOffset = 4 * i;
const tokenDeltaLine = tokens[srcOffset];
const tokenStartCharacter = tokens[srcOffset + 1];
const tokenEndCharacter = tokens[srcOffset + 2];
const tokenMetadata = tokens[srcOffset + 3];
if ((tokenDeltaLine > startDeltaLine || (tokenDeltaLine === startDeltaLine && tokenEndCharacter >= startChar))) {
if ((tokenDeltaLine < endDeltaLine || (tokenDeltaLine === endDeltaLine && tokenStartCharacter <= endChar))) {
// this token is touching the range
continue;
}
else {
// this token is after the range
if (destTokens !== bTokens) {
// this token is the first token after the range
destTokens = bTokens;
destOffset = 0;
destFirstDeltaLine = tokenDeltaLine;
}
}
}
destTokens[destOffset++] = tokenDeltaLine - destFirstDeltaLine;
destTokens[destOffset++] = tokenStartCharacter;
destTokens[destOffset++] = tokenEndCharacter;
destTokens[destOffset++] = tokenMetadata;
}
return [new SparseEncodedTokens(new Uint32Array(aTokens)), new SparseEncodedTokens(new Uint32Array(bTokens)), destFirstDeltaLine];
}
acceptDeleteRange(horizontalShiftForFirstLineTokens, startDeltaLine, startCharacter, endDeltaLine, endCharacter) {
// This is a bit complex, here are the cases I used to think about this:
//
// 1. The token starts before the deletion range
// 1a. The token is completely before the deletion range
// -----------
// xxxxxxxxxxx
// 1b. The token starts before, the deletion range ends after the token
// -----------
// xxxxxxxxxxx
// 1c. The token starts before, the deletion range ends precisely with the token
// ---------------
// xxxxxxxx
// 1d. The token starts before, the deletion range is inside the token
// ---------------
// xxxxx
//
// 2. The token starts at the same position with the deletion range
// 2a. The token starts at the same position, and ends inside the deletion range
// -------
// xxxxxxxxxxx
// 2b. The token starts at the same position, and ends at the same position as the deletion range
// ----------
// xxxxxxxxxx
// 2c. The token starts at the same position, and ends after the deletion range
// -------------
// xxxxxxx
//
// 3. The token starts inside the deletion range
// 3a. The token is inside the deletion range
// -------
// xxxxxxxxxxxxx
// 3b. The token starts inside the deletion range, and ends at the same position as the deletion range
// ----------
// xxxxxxxxxxxxx
// 3c. The token starts inside the deletion range, and ends after the deletion range
// ------------
// xxxxxxxxxxx
//
// 4. The token starts after the deletion range
// -----------
// xxxxxxxx
//
const tokens = this._tokens;
const tokenCount = this._tokenCount;
const deletedLineCount = (endDeltaLine - startDeltaLine);
let newTokenCount = 0;
let hasDeletedTokens = false;
for (let i = 0; i < tokenCount; i++) {
const srcOffset = 4 * i;
let tokenDeltaLine = tokens[srcOffset];
let tokenStartCharacter = tokens[srcOffset + 1];
let tokenEndCharacter = tokens[srcOffset + 2];
const tokenMetadata = tokens[srcOffset + 3];
if (tokenDeltaLine < startDeltaLine || (tokenDeltaLine === startDeltaLine && tokenEndCharacter <= startCharacter)) {
// 1a. The token is completely before the deletion range
// => nothing to do
newTokenCount++;
continue;
}
else if (tokenDeltaLine === startDeltaLine && tokenStartCharacter < startCharacter) {
// 1b, 1c, 1d
// => the token survives, but it needs to shrink
if (tokenDeltaLine === endDeltaLine && tokenEndCharacter > endCharacter) {
// 1d. The token starts before, the deletion range is inside the token
// => the token shrinks by the deletion character count
tokenEndCharacter -= (endCharacter - startCharacter);
}
else {
// 1b. The token starts before, the deletion range ends after the token
// 1c. The token starts before, the deletion range ends precisely with the token
// => the token shrinks its ending to the deletion start
tokenEndCharacter = startCharacter;
}
}
else if (tokenDeltaLine === startDeltaLine && tokenStartCharacter === startCharacter) {
// 2a, 2b, 2c
if (tokenDeltaLine === endDeltaLine && tokenEndCharacter > endCharacter) {
// 2c. The token starts at the same position, and ends after the deletion range
// => the token shrinks by the deletion character count
tokenEndCharacter -= (endCharacter - startCharacter);
}
else {
// 2a. The token starts at the same position, and ends inside the deletion range
// 2b. The token starts at the same position, and ends at the same position as the deletion range
// => the token is deleted
hasDeletedTokens = true;
continue;
}
}
else if (tokenDeltaLine < endDeltaLine || (tokenDeltaLine === endDeltaLine && tokenStartCharacter < endCharacter)) {
// 3a, 3b, 3c
if (tokenDeltaLine === endDeltaLine && tokenEndCharacter > endCharacter) {
// 3c. The token starts inside the deletion range, and ends after the deletion range
// => the token moves left and shrinks
if (tokenDeltaLine === startDeltaLine) {
// the deletion started on the same line as the token
// => the token moves left and shrinks
tokenStartCharacter = startCharacter;
tokenEndCharacter = tokenStartCharacter + (tokenEndCharacter - endCharacter);
}
else {
// the deletion started on a line above the token
// => the token moves to the beginning of the line
tokenStartCharacter = 0;
tokenEndCharacter = tokenStartCharacter + (tokenEndCharacter - endCharacter);
}
}
else {
// 3a. The token is inside the deletion range
// 3b. The token starts inside the deletion range, and ends at the same position as the deletion range
// => the token is deleted
hasDeletedTokens = true;
continue;
}
}
else if (tokenDeltaLine > endDeltaLine) {
// 4. (partial) The token starts after the deletion range, on a line below...
if (deletedLineCount === 0 && !hasDeletedTokens) {
// early stop, there is no need to walk all the tokens and do nothing...
newTokenCount = tokenCount;
break;
}
tokenDeltaLine -= deletedLineCount;
}
else if (tokenDeltaLine === endDeltaLine && tokenStartCharacter >= endCharacter) {
// 4. (continued) The token starts after the deletion range, on the last line where a deletion occurs
if (horizontalShiftForFirstLineTokens && tokenDeltaLine === 0) {
tokenStartCharacter += horizontalShiftForFirstLineTokens;
tokenEndCharacter += horizontalShiftForFirstLineTokens;
}
tokenDeltaLine -= deletedLineCount;
tokenStartCharacter -= (endCharacter - startCharacter);
tokenEndCharacter -= (endCharacter - startCharacter);
}
else {
throw new Error(`Not possible!`);
}
const destOffset = 4 * newTokenCount;
tokens[destOffset] = tokenDeltaLine;
tokens[destOffset + 1] = tokenStartCharacter;
tokens[destOffset + 2] = tokenEndCharacter;
tokens[destOffset + 3] = tokenMetadata;
newTokenCount++;
}
this._tokenCount = newTokenCount;
}
acceptInsertText(deltaLine, character, eolCount, firstLineLength, lastLineLength, firstCharCode) {
// Here are the cases I used to think about this:
//
// 1. The token is completely before the insertion point
// ----------- |
// 2. The token ends precisely at the insertion point
// -----------|
// 3. The token contains the insertion point
// -----|------
// 4. The token starts precisely at the insertion point
// |-----------
// 5. The token is completely after the insertion point
// | -----------
//
const isInsertingPreciselyOneWordCharacter = (eolCount === 0
&& firstLineLength === 1
&& ((firstCharCode >= 48 /* Digit0 */ && firstCharCode <= 57 /* Digit9 */)
|| (firstCharCode >= 65 /* A */ && firstCharCode <= 90 /* Z */)
|| (firstCharCode >= 97 /* a */ && firstCharCode <= 122 /* z */)));
const tokens = this._tokens;
const tokenCount = this._tokenCount;
for (let i = 0; i < tokenCount; i++) {
const offset = 4 * i;
let tokenDeltaLine = tokens[offset];
let tokenStartCharacter = tokens[offset + 1];
let tokenEndCharacter = tokens[offset + 2];
if (tokenDeltaLine < deltaLine || (tokenDeltaLine === deltaLine && tokenEndCharacter < character)) {
// 1. The token is completely before the insertion point
// => nothing to do
continue;
}
else if (tokenDeltaLine === deltaLine && tokenEndCharacter === character) {
// 2. The token ends precisely at the insertion point
// => expand the end character only if inserting precisely one character that is a word character
if (isInsertingPreciselyOneWordCharacter) {
tokenEndCharacter += 1;
}
else {
continue;
}
}
else if (tokenDeltaLine === deltaLine && tokenStartCharacter < character && character < tokenEndCharacter) {
// 3. The token contains the insertion point
if (eolCount === 0) {
// => just expand the end character
tokenEndCharacter += firstLineLength;
}
else {
// => cut off the token
tokenEndCharacter = character;
}
}
else {
// 4. or 5.
if (tokenDeltaLine === deltaLine && tokenStartCharacter === character) {
// 4. The token starts precisely at the insertion point
// => grow the token (by keeping its start constant) only if inserting precisely one character that is a word character
// => otherwise behave as in case 5.
if (isInsertingPreciselyOneWordCharacter) {
continue;
}
}
// => the token must move and keep its size constant
if (tokenDeltaLine === deltaLine) {
tokenDeltaLine += eolCount;
// this token is on the line where the insertion is taking place
if (eolCount === 0) {
tokenStartCharacter += firstLineLength;
tokenEndCharacter += firstLineLength;
}
else {
const tokenLength = tokenEndCharacter - tokenStartCharacter;
tokenStartCharacter = lastLineLength + (tokenStartCharacter - character);
tokenEndCharacter = tokenStartCharacter + tokenLength;
}
}
else {
tokenDeltaLine += eolCount;
}
}
tokens[offset] = tokenDeltaLine;
tokens[offset + 1] = tokenStartCharacter;
tokens[offset + 2] = tokenEndCharacter;
}
}
}
export class LineTokens2 {
constructor(tokens) {
this._tokens = tokens;
}
getCount() {
return this._tokens.length / 4;
}
getStartCharacter(tokenIndex) {
return this._tokens[4 * tokenIndex + 1];
}
getEndCharacter(tokenIndex) {
return this._tokens[4 * tokenIndex + 2];
}
getMetadata(tokenIndex) {
return this._tokens[4 * tokenIndex + 3];
}
}
export class MultilineTokens2 {
constructor(startLineNumber, tokens) {
this.startLineNumber = startLineNumber;
this.tokens = tokens;
this.endLineNumber = this.startLineNumber + this.tokens.getMaxDeltaLine();
}
toString() {
return this.tokens.toString(this.startLineNumber);
}
_updateEndLineNumber() {
this.endLineNumber = this.startLineNumber + this.tokens.getMaxDeltaLine();
}
isEmpty() {
return this.tokens.isEmpty();
}
getLineTokens(lineNumber) {
if (this.startLineNumber <= lineNumber && lineNumber <= this.endLineNumber) {
return this.tokens.getLineTokens(lineNumber - this.startLineNumber);
}
return null;
}
getRange() {
const deltaRange = this.tokens.getRange();
if (!deltaRange) {
return deltaRange;
}
return new Range(this.startLineNumber + deltaRange.startLineNumber, deltaRange.startColumn, this.startLineNumber + deltaRange.endLineNumber, deltaRange.endColumn);
}
removeTokens(range) {
const startLineIndex = range.startLineNumber - this.startLineNumber;
const endLineIndex = range.endLineNumber - this.startLineNumber;
this.startLineNumber += this.tokens.removeTokens(startLineIndex, range.startColumn - 1, endLineIndex, range.endColumn - 1);
this._updateEndLineNumber();
}
split(range) {
// split tokens to two:
// a) all the tokens before `range`
// b) all the tokens after `range`
const startLineIndex = range.startLineNumber - this.startLineNumber;
const endLineIndex = range.endLineNumber - this.startLineNumber;
const [a, b, bDeltaLine] = this.tokens.split(startLineIndex, range.startColumn - 1, endLineIndex, range.endColumn - 1);
return [new MultilineTokens2(this.startLineNumber, a), new MultilineTokens2(this.startLineNumber + bDeltaLine, b)];
}
applyEdit(range, text) {
const [eolCount, firstLineLength, lastLineLength] = countEOL(text);
this.acceptEdit(range, eolCount, firstLineLength, lastLineLength, text.length > 0 ? text.charCodeAt(0) : 0 /* Null */);
}
acceptEdit(range, eolCount, firstLineLength, lastLineLength, firstCharCode) {
this._acceptDeleteRange(range);
this._acceptInsertText(new Position(range.startLineNumber, range.startColumn), eolCount, firstLineLength, lastLineLength, firstCharCode);
this._updateEndLineNumber();
}
_acceptDeleteRange(range) {
if (range.startLineNumber === range.endLineNumber && range.startColumn === range.endColumn) {
// Nothing to delete
return;
}
const firstLineIndex = range.startLineNumber - this.startLineNumber;
const lastLineIndex = range.endLineNumber - this.startLineNumber;
if (lastLineIndex < 0) {
// this deletion occurs entirely before this block, so we only need to adjust line numbers
const deletedLinesCount = lastLineIndex - firstLineIndex;
this.startLineNumber -= deletedLinesCount;
return;
}
const tokenMaxDeltaLine = this.tokens.getMaxDeltaLine();
if (firstLineIndex >= tokenMaxDeltaLine + 1) {
// this deletion occurs entirely after this block, so there is nothing to do
return;
}
if (firstLineIndex < 0 && lastLineIndex >= tokenMaxDeltaLine + 1) {
// this deletion completely encompasses this block
this.startLineNumber = 0;
this.tokens.clear();
return;
}
if (firstLineIndex < 0) {
const deletedBefore = -firstLineIndex;
this.startLineNumber -= deletedBefore;
this.tokens.acceptDeleteRange(range.startColumn - 1, 0, 0, lastLineIndex, range.endColumn - 1);
}
else {
this.tokens.acceptDeleteRange(0, firstLineIndex, range.startColumn - 1, lastLineIndex, range.endColumn - 1);
}
}
_acceptInsertText(position, eolCount, firstLineLength, lastLineLength, firstCharCode) {
if (eolCount === 0 && firstLineLength === 0) {
// Nothing to insert
return;
}
const lineIndex = position.lineNumber - this.startLineNumber;
if (lineIndex < 0) {
// this insertion occurs before this block, so we only need to adjust line numbers
this.startLineNumber += eolCount;
return;
}
const tokenMaxDeltaLine = this.tokens.getMaxDeltaLine();
if (lineIndex >= tokenMaxDeltaLine + 1) {
// this insertion occurs after this block, so there is nothing to do
return;
}
this.tokens.acceptInsertText(lineIndex, position.column - 1, eolCount, firstLineLength, lastLineLength, firstCharCode);
}
}
export class MultilineTokens {
constructor(startLineNumber, tokens) {
this.startLineNumber = startLineNumber;
this.tokens = tokens;
}
}
function toUint32Array(arr) {
if (arr instanceof Uint32Array) {
return arr;
}
else {
return new Uint32Array(arr);
}
}
export class TokensStore2 {
constructor(languageIdCodec) {
this._pieces = [];
this._isComplete = false;
this._languageIdCodec = languageIdCodec;
}
flush() {
this._pieces = [];
this._isComplete = false;
}
isEmpty() {
return (this._pieces.length === 0);
}
set(pieces, isComplete) {
this._pieces = pieces || [];
this._isComplete = isComplete;
}
setPartial(_range, pieces) {
// console.log(`setPartial ${_range} ${pieces.map(p => p.toString()).join(', ')}`);
let range = _range;
if (pieces.length > 0) {
const _firstRange = pieces[0].getRange();
const _lastRange = pieces[pieces.length - 1].getRange();
if (!_firstRange || !_lastRange) {
return _range;
}
range = _range.plusRange(_firstRange).plusRange(_lastRange);
}
let insertPosition = null;
for (let i = 0, len = this._pieces.length; i < len; i++) {
const piece = this._pieces[i];
if (piece.endLineNumber < range.startLineNumber) {
// this piece is before the range
continue;
}
if (piece.startLineNumber > range.endLineNumber) {
// this piece is after the range, so mark the spot before this piece
// as a good insertion position and stop looping
insertPosition = insertPosition || { index: i };
break;
}
// this piece might intersect with the range
piece.removeTokens(range);
if (piece.isEmpty()) {
// remove the piece if it became empty
this._pieces.splice(i, 1);
i--;
len--;
continue;
}
if (piece.endLineNumber < range.startLineNumber) {
// after removal, this piece is before the range
continue;
}
if (piece.startLineNumber > range.endLineNumber) {
// after removal, this piece is after the range
insertPosition = insertPosition || { index: i };
continue;
}
// after removal, this piece contains the range
const [a, b] = piece.split(range);
if (a.isEmpty()) {
// this piece is actually after the range
insertPosition = insertPosition || { index: i };
continue;
}
if (b.isEmpty()) {
// this piece is actually before the range
continue;
}
this._pieces.splice(i, 1, a, b);
i++;
len++;
insertPosition = insertPosition || { index: i };
}
insertPosition = insertPosition || { index: this._pieces.length };
if (pieces.length > 0) {
this._pieces = arrays.arrayInsert(this._pieces, insertPosition.index, pieces);
}
// console.log(`I HAVE ${this._pieces.length} pieces`);
// console.log(`${this._pieces.map(p => p.toString()).join('\n')}`);
return range;
}
isComplete() {
return this._isComplete;
}
addSemanticTokens(lineNumber, aTokens) {
const pieces = this._pieces;
if (pieces.length === 0) {
return aTokens;
}
const pieceIndex = TokensStore2._findFirstPieceWithLine(pieces, lineNumber);
const bTokens = pieces[pieceIndex].getLineTokens(lineNumber);
if (!bTokens) {
return aTokens;
}
const aLen = aTokens.getCount();
const bLen = bTokens.getCount();
let aIndex = 0;
let result = [], resultLen = 0;
let lastEndOffset = 0;
const emitToken = (endOffset, metadata) => {
if (endOffset === lastEndOffset) {
return;
}
lastEndOffset = endOffset;
result[resultLen++] = endOffset;
result[resultLen++] = metadata;
};
for (let bIndex = 0; bIndex < bLen; bIndex++) {
const bStartCharacter = bTokens.getStartCharacter(bIndex);
const bEndCharacter = bTokens.getEndCharacter(bIndex);
const bMetadata = bTokens.getMetadata(bIndex);
const bMask = (((bMetadata & 1 /* SEMANTIC_USE_ITALIC */) ? 2048 /* ITALIC_MASK */ : 0)
| ((bMetadata & 2 /* SEMANTIC_USE_BOLD */) ? 4096 /* BOLD_MASK */ : 0)
| ((bMetadata & 4 /* SEMANTIC_USE_UNDERLINE */) ? 8192 /* UNDERLINE_MASK */ : 0)
| ((bMetadata & 8 /* SEMANTIC_USE_FOREGROUND */) ? 8372224 /* FOREGROUND_MASK */ : 0)
| ((bMetadata & 16 /* SEMANTIC_USE_BACKGROUND */) ? 4286578688 /* BACKGROUND_MASK */ : 0)) >>> 0;
const aMask = (~bMask) >>> 0;
// push any token from `a` that is before `b`
while (aIndex < aLen && aTokens.getEndOffset(aIndex) <= bStartCharacter) {
emitToken(aTokens.getEndOffset(aIndex), aTokens.getMetadata(aIndex));
aIndex++;
}
// push the token from `a` if it intersects the token from `b`
if (aIndex < aLen && aTokens.getStartOffset(aIndex) < bStartCharacter) {
emitToken(bStartCharacter, aTokens.getMetadata(aIndex));
}
// skip any tokens from `a` that are contained inside `b`
while (aIndex < aLen && aTokens.getEndOffset(aIndex) < bEndCharacter) {
emitToken(aTokens.getEndOffset(aIndex), (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask));
aIndex++;
}
if (aIndex < aLen) {
emitToken(bEndCharacter, (aTokens.getMetadata(aIndex) & aMask) | (bMetadata & bMask));
if (aTokens.getEndOffset(aIndex) === bEndCharacter) {
// `a` ends exactly at the same spot as `b`!
aIndex++;
}
}
else {
const aMergeIndex = Math.min(Math.max(0, aIndex - 1), aLen - 1);
// push the token from `b`
emitToken(bEndCharacter, (aTokens.getMetadata(aMergeIndex) & aMask) | (bMetadata & bMask));
}
}
// push the remaining tokens from `a`
while (aIndex < aLen) {
emitToken(aTokens.getEndOffset(aIndex), aTokens.getMetadata(aIndex));
aIndex++;
}
return new LineTokens(new Uint32Array(result), aTokens.getLineContent(), this._languageIdCodec);
}
static _findFirstPieceWithLine(pieces, lineNumber) {
let low = 0;
let high = pieces.length - 1;
while (low < high) {
let mid = low + Math.floor((high - low) / 2);
if (pieces[mid].endLineNumber < lineNumber) {
low = mid + 1;
}
else if (pieces[mid].startLineNumber > lineNumber) {
high = mid - 1;
}
else {
while (mid > low && pieces[mid - 1].startLineNumber <= lineNumber && lineNumber <= pieces[mid - 1].endLineNumber) {
mid--;
}
return mid;
}
}
return low;
}
//#region Editing
acceptEdit(range, eolCount, firstLineLength, lastLineLength, firstCharCode) {
for (const piece of this._pieces) {
piece.acceptEdit(range, eolCount, firstLineLength, lastLineLength, firstCharCode);
}
}
}
export class TokensStore {
constructor(languageIdCodec) {
this._lineTokens = [];
this._len = 0;
this._languageIdCodec = languageIdCodec;
}
flush() {
this._lineTokens = [];
this._len = 0;
}
getTokens(topLevelLanguageId, lineIndex, lineText) {
let rawLineTokens = null;
if (lineIndex < this._len) {
rawLineTokens = this._lineTokens[lineIndex];
}
if (rawLineTokens !== null && rawLineTokens !== EMPTY_LINE_TOKENS) {
return new LineTokens(toUint32Array(rawLineTokens), lineText, this._languageIdCodec);
}
const lineTokens = new Uint32Array(2);
lineTokens[0] = lineText.length;
lineTokens[1] = getDefaultMetadata(this._languageIdCodec.encodeLanguageId(topLevelLanguageId));
return new LineTokens(lineTokens, lineText, this._languageIdCodec);
}
static _massageTokens(topLevelLanguageId, lineTextLength, _tokens) {
const tokens = _tokens ? toUint32Array(_tokens) : null;
if (lineTextLength === 0) {
let hasDifferentLanguageId = false;
if (tokens && tokens.length > 1) {
hasDifferentLanguageId = (TokenMetadata.getLanguageId(tokens[1]) !== topLevelLanguageId);
}
if (!hasDifferentLanguageId) {
return EMPTY_LINE_TOKENS;
}
}
if (!tokens || tokens.length === 0) {
const tokens = new Uint32Array(2);
tokens[0] = lineTextLength;
tokens[1] = getDefaultMetadata(topLevelLanguageId);
return tokens.buffer;
}
// Ensure the last token covers the end of the text
tokens[tokens.length - 2] = lineTextLength;
if (tokens.byteOffset === 0 && tokens.byteLength === tokens.buffer.byteLength) {
// Store directly the ArrayBuffer pointer to save an object
return tokens.buffer;
}
return tokens;
}
_ensureLine(lineIndex) {
while (lineIndex >= this._len) {
this._lineTokens[this._len] = null;
this._len++;
}
}
_deleteLines(start, deleteCount) {
if (deleteCount === 0) {
return;
}
if (start + deleteCount > this._len) {
deleteCount = this._len - start;
}
this._lineTokens.splice(start, deleteCount);
this._len -= deleteCount;
}
_insertLines(insertIndex, insertCount) {
if (insertCount === 0) {
return;
}
let lineTokens = [];
for (let i = 0; i < insertCount; i++) {
lineTokens[i] = null;
}
this._lineTokens = arrays.arrayInsert(this._lineTokens, insertIndex, lineTokens);
this._len += insertCount;
}
setTokens(topLevelLanguageId, lineIndex, lineTextLength, _tokens, checkEquality) {
const tokens = TokensStore._massageTokens(this._languageIdCodec.encodeLanguageId(topLevelLanguageId), lineTextLength, _tokens);
this._ensureLine(lineIndex);
const oldTokens = this._lineTokens[lineIndex];
this._lineTokens[lineIndex] = tokens;
if (checkEquality) {
return !TokensStore._equals(oldTokens, tokens);
}
return false;
}
static _equals(_a, _b) {
if (!_a || !_b) {
return !_a && !_b;
}
const a = toUint32Array(_a);
const b = toUint32Array(_b);
if (a.length !== b.length) {
return false;
}
for (let i = 0, len = a.length; i < len; i++) {
if (a[i] !== b[i]) {
return false;
}
}
return true;
}
//#region Editing
acceptEdit(range, eolCount, firstLineLength) {
this._acceptDeleteRange(range);
this._acceptInsertText(new Position(range.startLineNumber, range.startColumn), eolCount, firstLineLength);
}
_acceptDeleteRange(range) {
const firstLineIndex = range.startLineNumber - 1;
if (firstLineIndex >= this._len) {
return;
}
if (range.startLineNumber === range.endLineNumber) {
if (range.startColumn === range.endColumn) {
// Nothing to delete
return;
}
this._lineTokens[firstLineIndex] = TokensStore._delete(this._lineTokens[firstLineIndex], range.startColumn - 1, range.endColumn - 1);
return;
}
this._lineTokens[firstLineIndex] = TokensStore._deleteEnding(this._lineTokens[firstLineIndex], range.startColumn - 1);
const lastLineIndex = range.endLineNumber - 1;
let lastLineTokens = null;
if (lastLineIndex < this._len) {
lastLineTokens = TokensStore._deleteBeginning(this._lineTokens[lastLineIndex], range.endColumn - 1);
}
// Take remaining text on last line and append it to remaining text on first line
this._lineTokens[firstLineIndex] = TokensStore._append(this._lineTokens[firstLineIndex], lastLineTokens);
// Delete middle lines
this._deleteLines(range.startLineNumber, range.endLineNumber - range.startLineNumber);
}
_acceptInsertText(position, eolCount, firstLineLength) {
if (eolCount === 0 && firstLineLength === 0) {
// Nothing to insert
return;
}
const lineIndex = position.lineNumber - 1;
if (lineIndex >= this._len) {
return;
}
if (eolCount === 0) {
// Inserting text on one line
this._lineTokens[lineIndex] = TokensStore._insert(this._lineTokens[lineIndex], position.column - 1, firstLineLength);
return;
}
this._lineTokens[lineIndex] = TokensStore._deleteEnding(this._lineTokens[lineIndex], position.column - 1);
this._lineTokens[lineIndex] = TokensStore._insert(this._lineTokens[lineIndex], position.column - 1, firstLineLength);
this._insertLines(position.lineNumber, eolCount);
}
static _deleteBeginning(lineTokens, toChIndex) {
if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS) {
return lineTokens;
}
return TokensStore._delete(lineTokens, 0, toChIndex);
}
static _deleteEnding(lineTokens, fromChIndex) {
if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS) {
return lineTokens;
}
const tokens = toUint32Array(lineTokens);
const lineTextLength = tokens[tokens.length - 2];
return TokensStore._delete(lineTokens, fromChIndex, lineTextLength);
}
static _delete(lineTokens, fromChIndex, toChIndex) {
if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS || fromChIndex === toChIndex) {
return lineTokens;
}
const tokens = toUint32Array(lineTokens);
const tokensCount = (tokens.length >>> 1);
// special case: deleting everything
if (fromChIndex === 0 && tokens[tokens.length - 2] === toChIndex) {
return EMPTY_LINE_TOKENS;
}
const fromTokenIndex = LineTokens.findIndexInTokensArray(tokens, fromChIndex);
const fromTokenStartOffset = (fromTokenIndex > 0 ? tokens[(fromTokenIndex - 1) << 1] : 0);
const fromTokenEndOffset = tokens[fromTokenIndex << 1];
if (toChIndex < fromTokenEndOffset) {
// the delete range is inside a single token
const delta = (toChIndex - fromChIndex);
for (let i = fromTokenIndex; i < tokensCount; i++) {
tokens[i << 1] -= delta;
}
return lineTokens;
}
let dest;
let lastEnd;
if (fromTokenStartOffset !== fromChIndex) {
tokens[fromTokenIndex << 1] = fromChIndex;
dest = ((fromTokenIndex + 1) << 1);
lastEnd = fromChIndex;
}
else {
dest = (fromTokenIndex << 1);
lastEnd = fromTokenStartOffset;
}
const delta = (toChIndex - fromChIndex);
for (let tokenIndex = fromTokenIndex + 1; tokenIndex < tokensCount; tokenIndex++) {
const tokenEndOffset = tokens[tokenIndex << 1] - delta;
if (tokenEndOffset > lastEnd) {
tokens[dest++] = tokenEndOffset;
tokens[dest++] = tokens[(tokenIndex << 1) + 1];
lastEnd = tokenEndOffset;
}
}
if (dest === tokens.length) {
// nothing to trim
return lineTokens;
}
let tmp = new Uint32Array(dest);
tmp.set(tokens.subarray(0, dest), 0);
return tmp.buffer;
}
static _append(lineTokens, _otherTokens) {
if (_otherTokens === EMPTY_LINE_TOKENS) {
return lineTokens;
}
if (lineTokens === EMPTY_LINE_TOKENS) {
return _otherTokens;
}
if (lineTokens === null) {
return lineTokens;
}
if (_otherTokens === null) {
// cannot determine combined line length...
return null;
}
const myTokens = toUint32Array(lineTokens);
const otherTokens = toUint32Array(_otherTokens);
const otherTokensCount = (otherTokens.length >>> 1);
let result = new Uint32Array(myTokens.length + otherTokens.length);
result.set(myTokens, 0);
let dest = myTokens.length;
const delta = myTokens[myTokens.length - 2];
for (let i = 0; i < otherTokensCount; i++) {
result[dest++] = otherTokens[(i << 1)] + delta;
result[dest++] = otherTokens[(i << 1) + 1];
}
return result.buffer;
}
static _insert(lineTokens, chIndex, textLength) {
if (lineTokens === null || lineTokens === EMPTY_LINE_TOKENS) {
// nothing to do
return lineTokens;
}
const tokens = toUint32Array(lineTokens);
const tokensCount = (tokens.length >>> 1);
let fromTokenIndex = LineTokens.findIndexInTokensArray(tokens, chIndex);
if (fromTokenIndex > 0) {
const fromTokenStartOffset = tokens[(fromTokenIndex - 1) << 1];
if (fromTokenStartOffset === chIndex) {
fromTokenIndex--;
}
}
for (let tokenIndex = fromTokenIndex; tokenIndex < tokensCount; tokenIndex++) {
tokens[tokenIndex << 1] += textLength;
}
return lineTokens;
}
}