@predictive-text-studio/lexical-model-compiler/dist/lexical-model-compiler/build-trie/index.d.ts

Keyman Developer lexical model compiler

/// <reference types="node" />
import { LineNoAndText, WordList } from "../wordlist";
export declare const NEWLINE_SEPARATOR: RegExp;
/**
 * Returns a data structure that can be loaded by the TrieModel.
 *
 * It implements a **weighted** trie, whose indices (paths down the trie) are
 * generated by a search key, and not concrete wordforms themselves.
 *
 * @param wordlist a complete mapping of words to frequencies
 */
export declare function compileTrieFromWordlist(wordlist: WordList, searchTermToKey: (wf: string) => string): string;
/**
 * Parses a word list from a string. The string should have multiple lines
 * with LF or CRLF line terminators.
 *
 * @param wordlist word list to merge entries into (may have existing entries)
 * @param filename filename of the word list
 */
export declare function parseWordListFromContents(wordlist: WordList, contents: string): void;
/**
 * Yields pairs of [lineno, line], given an Array of lines.
 */
export declare function enumerateLines(lines: string[]): Generator<LineNoAndText>;
/**
 * Converts wordforms into an indexable form. It does this by
 * normalizing the letter case of characters INDIVIDUALLY (to disregard
 * context-sensitive case transformations), normalizing to NFKD form,
 * and removing common diacritical marks.
 *
 * This is a very speculative implementation, that might work with
 * your language. We don't guarantee that this will be perfect for your
 * language, but it's a start.
 *
 * This uses String.prototype.normalize() to convert normalize into NFKD.
 * NFKD neutralizes some funky distinctions, e.g., ꬲ, ｅ, e should all be the
 * same character; plus, it's an easy way to separate a Latin character from
 * its diacritics; Even then, orthographies regularly use code points
 * that, under NFKD normalization, do NOT decompose appropriately for your
 * language (e.g., SENĆOŦEN, Plains Cree in syllabics).
 *
 * Use this in early iterations of the model. For a production lexical model,
 * you will probably write/generate your own key function, tailored to your
 * language. There is a chance the default will work properly out of the box.
 */
export declare function defaultSearchTermToKey(wordform: string): string;
/**
 * Detects the encoding of a text file.
 *
 * Supported encodings are:
 *
 *  - UTF-8, with or without BOM
 *  - UTF-16, little endian, with BOM
 *
 * UTF-16 in big endian is explicitly NOT supported! The reason is two-fold:
 * 1) Node does not support it without resorting to an external library (or
 * swapping every byte in the file!); and 2) I'm not sure anything actually
 * outputs in this format anyway!
 *
 * @param filename filename of the file to detect encoding
 */
export declare function detectEncodingFromBuffer(buffer: Int8Array | Buffer): 'utf8' | 'utf16le';