@cognigy/rest-api-client
Version:
Cognigy REST-Client
436 lines • 22.4 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.getTextCleaner = exports.TextCleaner = void 0;
const dicts_1 = require("./dicts/dicts");
;
/**
* A class to clean up text
* Mainly written to deal with text coming in from STT
*/
class TextCleaner {
/**
* Creates an instance of CleanText
* @param locale The locale to use (e.g. "de")
* @param additionalAllowedCharacters Extra allowed symbols for this instance (e.g. ["-", "*"])
* @param additionalMappedSymbols Extra allowed symbol words for this instance (e.g. {"minus": "-"})
* @param additionalSpecialPhrases Special words which may need replacing (e.g. if you want to detect cities, you might want to replace "färben" with "verden")
* @param additionalPhoneticAlphabet Extra spelling alphabet for this instance (e.g. {"alfa": "a", "bravo": "b"})
*/
constructor(locale, additionalAllowedCharacters = [], additionalMappedSymbols = {}, additionalSpecialPhrases = {}, additionalPhoneticAlphabet = null) {
this.locale = locale;
this.allowedCharacters = dicts_1.LOCALE_SPECIFIC_SYMBOLS[locale].concat(additionalAllowedCharacters);
if (additionalPhoneticAlphabet === null) {
additionalPhoneticAlphabet = dicts_1.EXTRA_SPELLING_ALPHABET[locale];
}
this.dictAlphabet = dicts_1.ALPHABET[locale];
this.dictAlphabet = Object.assign(Object.assign({}, this.dictAlphabet), additionalPhoneticAlphabet);
this.dictSymbolWords = dicts_1.SYMBOL_WORDS[locale];
this.dictSymbolWords = Object.assign(Object.assign({}, this.dictSymbolWords), additionalMappedSymbols);
this.dictSpecials = Object.assign({}, additionalSpecialPhrases);
this.dictPunctuation = dicts_1.PUNCTUATION;
this.listExcludedWords = dicts_1.EXCLUDE[locale];
this.dictNumbers = dicts_1.NUMBERS_DICT[locale];
this.dictNumberRepeaters = dicts_1.NUMBERS_REPEAT[locale];
this.dictNumberMultipliers = dicts_1.NUMBERS_MULTIPLIERS[locale];
this.dictPhoneticConnectors = dicts_1.PHONETIC_CONNECTORS[locale];
if (this.dictPhoneticConnectors.length > 0) {
// sort in descending order for the regex to work properly
this.dictPhoneticConnectors.sort((a, b) => b.length - a.length);
}
// create the regex of allowed characters.
// the .replace portion escapes the additional allowed symbols so they can be used in the regex
this.allowedSymbolsRegex = new RegExp(`[^A-Za-z0-9${this.allowedCharacters.join("").replace(/[-.*+?^${}()|[\]\\]/g, '\\$&')}\\s]+`, "g");
}
/**
* Function to executed all or selected functions of this class on a phrase
* @param phrase The phrase to clean
* @param options Optional toggle for cleaning functions
* @returns The cleaned phrase
*/
cleanAll(phrase, options, detailedSlots) {
if (!options || options.cleanDisallowedSymbols)
phrase = this.cleanDisallowedSymbols(phrase);
if (!options || options.resolveSpelledOutNumbers)
phrase = this.resolveSpelledOutNumbers(phrase, detailedSlots, false);
if (!options || options.resolvePhoneticAlphabet)
phrase = this.resolvePhoneticAlphabet(phrase);
if (!options || options.replaceSpecialWords || options.replaceSpecialPhrases)
phrase = this.replaceSpecialPhrases(phrase);
if (!options || options.resolveSpelledOutAlphabet)
phrase = this.resolveSpelledOutAlphabet(phrase);
if (!options || options.resolvePhoneticCounters)
phrase = this.resolvePhoneticCounters(phrase);
if (!options || options.contractSingleCharacters)
phrase = this.contractSingleCharacters(phrase);
if (!options || options.contractNumberGroups)
phrase = this.contractNumberGroups(phrase);
if (!options || options.trimResult)
phrase = this.trimResult(phrase);
return phrase;
}
/**
* Remove all symbols which are not allowed
*/
cleanDisallowedSymbols(phrase) {
phrase = phrase.toLowerCase().replace(this.allowedSymbolsRegex, '');
phrase = phrase.replace(/\s+/g, ' ');
const words = phrase.split(' ');
const cleanedWords = words.map((word) => {
var _a, _b;
// figure out if last character is a full stop
const lastChar = word.slice(-1);
const hasPunctuation = this.dictPunctuation.indexOf(lastChar) !== -1;
let replacement = word;
if (hasPunctuation) {
word = word.slice(0, -1);
replacement = ((_a = this.dictSymbolWords[word]) !== null && _a !== void 0 ? _a : word) + lastChar;
}
else {
replacement = (_b = this.dictSymbolWords[word]) !== null && _b !== void 0 ? _b : word;
}
// if word was found in the dictionary, add it to the list of additional allowed characters
if (this.dictSymbolWords[word] && this.allowedCharacters.indexOf(this.dictSymbolWords[word]) === -1) {
this.allowedCharacters.push(this.dictSymbolWords[word]);
this.allowedSymbolsRegex = new RegExp(`[^A-Za-z0-9${this.allowedCharacters.join("").replace(/[-.*+?^${}()|[\]\\]/g, '\\$&')}\\s]+`, "g");
}
return replacement;
});
phrase = cleanedWords.join(' ');
return phrase;
}
/**
* Replaces all number words with their numerical representation
* e.g. "eins zwei drei" -> "1 2 3"
* @param preserveExcludedWords If true, words which are in the listExcludedWords will not be replaced
*/
resolveSpelledOutNumbers(phrase, detailedSlots, preserveExcludedWords) {
// check if we have a detailedSlots NUMBER object
if (detailedSlots && detailedSlots.NUMBER && Array.isArray(detailedSlots.NUMBER) && detailedSlots.NUMBER.length > 0) {
detailedSlots.NUMBER.forEach((num) => {
// special rule for de-DE, as our NLU captures "mal" as part of the number words
// if we replace this, we can't resolve phonetic counters anymore
// in addition the German NLU parser finds IPv4 addresses with full length (123.123.123.132) as numbers, which we don't want to replace
// also avoid replacing the phrase if the string has leading zeros, as data.value will have the string without them
// ex: 0000 should be 0000 not 0
if ((this.locale !== "de" || (!num.text.includes("mal") && !num.text.match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/))) && num.text[0] !== "0"
&& num.data.value != null && Math.abs(Number(num.data.value)) <= Number.MAX_SAFE_INTEGER) {
phrase = phrase.replace(num.text, num.data.value.toString());
}
});
}
phrase = phrase.toLowerCase().split(' ')
.filter((i) => preserveExcludedWords || !this.listExcludedWords.includes(i))
.map((w, i, splits) => {
if (this.dictNumbers[w] !== undefined) {
if (splits[i + 1] && this.dictNumbers[w + " " + splits[i + 1]] !== undefined) {
const result = this.dictNumbers[w + " " + splits[i + 1]];
delete splits[i + 1];
return result;
}
else {
return this.dictNumbers[w];
}
}
else
return w;
}).join(' ').replace(/\s\s/g, ' ');
return phrase;
}
/**
* Replaces specifically determined words with their replacements
* e.g. "färben" -> "verden"
* @deprecated Use replaceSpecialPhrases instead
*/
replaceSpecialWords(phrase) {
phrase = phrase.toLowerCase().replace(this.allowedSymbolsRegex, '');
phrase = phrase.replace(/\s+/g, ' ');
const words = phrase.split(' ');
const cleanedWords = words.map((word) => {
var _a, _b;
// figure out if last character is a punctuation symbol
const lastChar = word.slice(-1);
const hasPunctuation = this.dictPunctuation.indexOf(lastChar) !== -1;
let replacement = word;
if (hasPunctuation) {
word = word.slice(0, -1);
replacement = ((_a = this.dictSpecials[word]) !== null && _a !== void 0 ? _a : word) + lastChar;
}
else {
replacement = (_b = this.dictSpecials[word]) !== null && _b !== void 0 ? _b : word;
}
return replacement;
});
phrase = cleanedWords.join(' ');
return phrase;
}
/**
* Replaces specifically determined phrases with their replacements
* e.g. "swiss airlines international" -> "SWISS"
*/
replaceSpecialPhrases(phrase) {
// lowercase the whole dictSpecials
for (const [key, value] of Object.entries(this.dictSpecials)) {
this.dictSpecials[key.toLowerCase()] = value;
}
// Replace all double spaces with single spaces
let normalizedPhrase = phrase.replace(/\s+/g, ' ').toLowerCase();
for (let phraseToDetect of Object.keys(this.dictSpecials)) {
// make phrase regex-safe
phraseToDetect = phraseToDetect.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
// check if phraseToDetect is a single special character (e.g., *, ., +)
const isSpecialChar = /^[-\/\\^$*+?.()|[\]{}#]$/.test(phraseToDetect.replace(/\\/g, '')); // Remove escaping before testing
// for special characters like '*', don't use word boundaries since they can appear between numbers
// for normal words like "hello", use word boundaries (\b) to match whole words only
const regexPattern = isSpecialChar
? `(${phraseToDetect})` // Match special char anywhere (e.g., in "8*4")
: `\\b(${phraseToDetect})\\b`; // Match whole words only (e.g., not "hello" in "hello123")
const matches = normalizedPhrase.match(new RegExp(regexPattern, 'gi'));
if (matches) {
normalizedPhrase = normalizedPhrase.replace(new RegExp(regexPattern, "gi"), this.dictSpecials[phraseToDetect.replace(/\\/g, '')] // Remove escaping to get original key
);
}
}
return normalizedPhrase;
}
/**
* Detects and replaces all words which are known in the spelling alphabet
* e.g. "alpha tango lima" -> "a t l" or "anton ida otto" -> "a i o"
*/
resolvePhoneticAlphabet(phrase) {
phrase = phrase.toLowerCase().replace(this.allowedSymbolsRegex, '');
phrase = phrase.replace(/\s+/g, ' ');
const words = phrase.split(' ');
const cleanedWords = words.map((word) => {
var _a;
let lastChar = word.slice(-1);
if (this.dictPunctuation.indexOf(lastChar) !== -1) {
// remove punctuation
word = word.slice(0, -1);
}
else
lastChar = "";
const replacement = (_a = this.dictAlphabet[word]) !== null && _a !== void 0 ? _a : word;
return replacement + lastChar;
});
phrase = cleanedWords.join(' ');
return phrase;
}
/**
* Resolves string like "a for anton b for bertram" to "a b"
*/
resolveSpelledOutAlphabet(phrase) {
phrase = phrase.toLowerCase();
while (true) {
// Use \u00C0-\u017E (Latin Extended) in both positions to support German special characters (ä, ö, ü, ß)
// in both the spelled letter (e.g. "Ä wie Ärger") and the reference word (e.g. "K wie Köln")
const regex = new RegExp(`(\\s|^)([a-z\\u00C0-\\u017E]{1,3}) (${this.dictPhoneticConnectors.join("|")}) [\\w\\u00C0-\\u017E]+`, "i");
const match = phrase.match(regex);
if (match) {
const start = match.index;
const stop = start + match[0].length;
const letter = this._validateReduce(phrase, match[2], start, stop, match[3]);
phrase = phrase.slice(0, start) + ' ' + letter + phrase.slice(stop);
}
else {
break;
}
}
// Replace letter multiplication
for (const repeat in this.dictNumbers) {
while (true) {
const match = phrase.match(new RegExp(`(${repeat} [a-z]\\b)`));
if (match) {
const start = match.index;
const stop = start + match[0].length;
const letter = `${this.dictNumbers[repeat]}`.repeat(Number(phrase.slice(start, stop).slice(-1)));
phrase = phrase.slice(0, start) + letter + phrase.slice(stop);
}
else {
break;
}
}
}
return phrase;
}
/**
* Resolves strings like "three times 2" to "222" or "double 4" to "44"
* Only takes the last number of the counter and first of the repeater
* e.g. ("352 mal 355" becomes "2 mal 3" > "333")
*/
resolvePhoneticCounters(phrase) {
phrase = phrase.toLowerCase();
// first we check for "X times Y"-style counters
let matches;
do {
const allowedRepeaters = this.dictNumberMultipliers.join("|");
const xMalYRegex = new RegExp(`(\\d+){1}\\s*(${allowedRepeaters}|\\*)\\s*[die\\s]*(\\d+|\\w+)`, "gi");
phrase = phrase.replace(/\s\s/gi, ' ');
matches = xMalYRegex.exec(phrase);
if (matches && !isNaN(Number(matches[1])) && !isNaN(Number(matches[3]))) {
// string is like "3 mal 4"
const counter = Number(matches[1]);
let replacer = "";
let remainer = "";
let adder = matches[3];
if (adder.length > 1 && !isNaN(Number(adder)) && Number(adder) > 12) {
// if number is > 12, we assume that the user meant to write "3 times 1 and 3" instead of "3 times 13"
// otherwise we assume "2 times 11" means "1111"
remainer = adder.substring(1, adder.length);
adder = adder.substring(0, 1);
}
for (let i = 0; i < counter; i++) {
replacer += adder;
}
phrase = phrase.replace(matches[0].replace(remainer, ""), replacer);
}
else if (matches && !isNaN(Number(matches[1])) && isNaN(Number(matches[3]))) {
// string is like "3 mal x"
const counter = Number(matches[1]);
let replacer = "";
let remainer = "";
let adder = matches[3];
// this happens if adder is a string and starts with a number
if (adder.length > 1 && !isNaN(Number(adder.substring(0, 1)))) {
remainer = adder.substring(1, adder.length);
adder = adder.substring(0, 1);
}
// the next checks happen only if the adder is a string
if (isNaN(Number(adder)) && typeof adder === "string" && adder.length > 1 && dicts_1.ALPHABET.de[adder]) {
// adder is part of the spelling alphabet (e.g. "alpha"), so we use the corresponding letter
adder = dicts_1.ALPHABET.de[matches[3]];
}
else if (isNaN(Number(adder)) && typeof adder === "string" && adder.length > 1 && !dicts_1.ALPHABET.de[adder]) {
// adder is NOT part of the spelling alphabet (e.g. "alpha"), so we use the first letter only
remainer = adder.substring(1, adder.length);
adder = adder.substring(0, 1);
}
for (let i = 0; i < counter; i++) {
replacer += adder;
}
phrase = phrase.replace(matches[0].replace(remainer, ""), replacer);
}
} while (matches);
// here we check for "double X"-style counters
let matches2;
do {
const allowedRepeaters = Object.keys(this.dictNumberRepeaters).join("|");
const reg2 = new RegExp(`(${allowedRepeaters})\\s*(?:die|the)?\\s*(\\d+|\\w+)`, "gi");
matches2 = reg2.exec(phrase);
if (matches2 && this.dictNumberRepeaters[matches2[1]]) {
const counter = Number(this.dictNumberRepeaters[matches2[1]]);
let replacer = "";
let remainder = "";
let adder = matches2[2];
if (adder.length > 1 && !isNaN(Number(adder)) && Number(adder) > 12) {
// if number is > 12, we assume that the user meant to write "double 1 and 3" instead of "double 13"
// otherwise we assume "double 11" means "1111"
remainder = adder.substring(1, adder.length);
adder = adder.substring(0, 1);
}
// TODO: We only check for entries in the German Alphabet. Currently, this appears to be working also
// for English, because most English words were included in the German alphabet, too. Other languages
// are not supported.
if (isNaN(Number(adder)) && typeof adder === "string" && adder.length > 1 && dicts_1.ALPHABET.de[adder]) {
// adder is part of the spelling alphabet (e.g. "alpha"), so we use the corresponding letter
adder = dicts_1.ALPHABET.de[matches2[2]];
}
else if (isNaN(Number(adder)) && typeof adder === "string" && adder.length > 1 && !dicts_1.ALPHABET.de[adder]) {
// adder is NOT part of the spelling alphabet (e.g. "alpha"), so we use the first letter only
remainder = adder.substring(1, adder.length);
adder = adder.substring(0, 1);
}
for (let i = 0; i < counter; i++) {
replacer += adder;
}
// Replace the last occurrence of the remainder with ""
let matchReplace = matches2[0];
const lastIndex = matchReplace.lastIndexOf(remainder);
if (lastIndex !== -1) {
matchReplace = matchReplace.substring(0, lastIndex) + matchReplace.substring(lastIndex + remainder.length);
}
// Replace the first occurrence of matchReplace with the replacer
phrase = phrase.replace(matchReplace, replacer);
}
} while (matches2);
return phrase;
}
/**
* Joins all single characters standing alone into a full string
* e.g. "my name is h e l t e w i g" > "my name is heltewig"
*/
contractSingleCharacters(phrase) {
const words = phrase.split(' ');
const result = [];
for (let i = 0; i < words.length; i++) {
const word = words[i];
// we check whether the word is either a single character or a single character followed by a punctuation mark
// if so, we check whether the previous word is a single character and if yes, we join them
if ((word.length === 1 || (word.length === 2 && [".", ",", "!", "?"].indexOf(word[1]) > -1)) && i > 0 && words[i - 1].length === 1) {
result[result.length - 1] += word;
}
else {
result.push(word);
}
}
// Join the words back together with spaces
return result.join(' ');
}
/**
* Joins all numbers standing next to each other
* e.g. "his number is 333 43 22 44" > "his number is 333432244"
*/
contractNumberGroups(phrase) {
// Use a regular expression to match all the number groups in the input string
const numberGroups = phrase.match(/(\d[\d\s]*\d|\d)/g);
if (!numberGroups) {
// If there are no number groups, return the original phrase
return phrase;
}
// Join the number groups, removing spaces in between
const contractedNumberGroups = numberGroups.map(group => group.replace(/\s+/g, ''));
// Replace the original number groups with the contracted ones
let result = phrase;
for (let i = 0; i < numberGroups.length; i++) {
result = result.replace(numberGroups[i], contractedNumberGroups[i]);
}
return result;
}
/**
* Trims the start and end of the string and replaces double spaces with single spaces
*/
trimResult(phrase) {
phrase = phrase.trim();
phrase = phrase.replace(/\s+/g, ' ');
return phrase;
}
/**
* Helper function for resolveSpelledOutAlphabet
* @param phrase The phrase to resolve
* @param letter The letter found in LETTER as in THIRDWORD
* @param start The start index of the detected partial phrase
* @param stop The stop index of the detected partial phrase
* @param separator The separator (e.g. "as in" or "wie")
* @returns string
*/
_validateReduce(phrase, letter, start, stop, separator) {
if (letter.length === 1) {
const thirdWord = phrase
.substring(start, stop)
.trim()
.split(` ${separator} `)[1];
if (letter !== thirdWord[0]) {
letter = thirdWord[0];
}
}
return letter;
}
}
exports.TextCleaner = TextCleaner;
/**
* Returns a new instance of TextCleaner
*/
function getTextCleaner(locale, options) {
return new TextCleaner(locale, options === null || options === void 0 ? void 0 : options.additionalAllowedCharacters, options === null || options === void 0 ? void 0 : options.additionalMappedSymbols, options === null || options === void 0 ? void 0 : options.additionalSpecialPhrases, options === null || options === void 0 ? void 0 : options.additionalPhoneticAlphabet);
}
exports.getTextCleaner = getTextCleaner;
//# sourceMappingURL=textCleaner.js.map