@dvdagames/pgn-tokenizer/dist/types/tokenizer.d.ts

TypeScript version of PGN Tokenizer, a Byte Pair Encoding (BPE) tokenizer for Chess Portable Game Notiation (PGN).

export interface PGNTokenizerInterface {
    vocabSize: number;
    encode: (pgn: string) => number[];
    decode: (tokens: number[]) => string;
}
export default class PGNTokenizer implements PGNTokenizerInterface {
    private readonly config;
    private readonly mergedVocabulary;
    vocabSize: number;
    constructor();
    private createMergedVocabulary;
    private lookupToken;
    private mergePair;
    encode(pgn: string): number[];
    decode(tokens: number[]): string;
}