@predictive-text-studio/lexical-model-compiler/dist/lexical-model-compiler/build-trie/with-node-fs.js

Keyman Developer lexical model compiler

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
/**
 * Extends build-trie with utilities that depend upon Node's fs module.
 *
 * @file with-node-fs
 */
const fs_1 = require("fs");
const index_1 = require("./index");
const parse_wordlist_1 = require("../parse-wordlist");
/**
 * Returns a data structure that can be loaded by the TrieModel.
 *
 * It implements a **weighted** trie, whose indices (paths down the trie) are
 * generated by a search key, and not concrete wordforms themselves.
 *
 * @param sourceFiles an array of source files that will be read to generate the trie.
 */
function createTrieDataStructureFromFilenames(filenames, searchTermToKey) {
    return index_1.compileTrieFromWordlist(wordListFromFilenames(filenames), searchTermToKey);
}
exports.createTrieDataStructureFromFilenames = createTrieDataStructureFromFilenames;
/**
 * Make one big word list out of all of the filenames provided.
 */
function wordListFromFilenames(filenames) {
    let wordlist = {};
    filenames.forEach(filename => parseWordListFromFilename(wordlist, filename));
    return wordlist;
}
/**
 * Parses a word list from a file, merging duplicate entries.
 *
 * The word list may be encoded in:
 *
 *  - UTF-8, with or without BOM [exported by most software]
 *  - UTF-16, little endian, with BOM [exported by Microsoft Excel]
 *
 * @param wordlist word list to merge entries into (may have existing entries)
 * @param filename filename of the word list
 */
function parseWordListFromFilename(wordlist, filename) {
    parse_wordlist_1.parseWordList(wordlist, new WordListFromFilename(filename));
}
exports.parseWordListFromFilename = parseWordListFromFilename;
/**
 * Given a real filesystem path, this implements the WordList source.
 */
class WordListFromFilename {
    constructor(filename) {
        this.name = filename;
    }
    *lines() {
        let contents = fs_1.readFileSync(this.name, detectEncodingFromFilename(this.name));
        yield* index_1.enumerateLines(contents.split(index_1.NEWLINE_SEPARATOR));
    }
}
exports.WordListFromFilename = WordListFromFilename;
function detectEncodingFromFilename(filename) {
    let buffer = fs_1.readFileSync(filename);
    return index_1.detectEncodingFromBuffer(buffer);
}
//# sourceMappingURL=with-node-fs.js.map