classifier.js
Version:
:robot: Natural language processing with Javascript
170 lines (169 loc) • 7.63 kB
JavaScript
;
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Classifier = void 0;
const category_1 = require("./category");
const lib_1 = require("./lib");
const promises_1 = require("fs/promises");
const fs_1 = require("fs");
const path_1 = require("path");
const js_yaml_1 = __importDefault(require("js-yaml"));
class Classifier {
constructor(options) {
this.categories = [];
this.options = {};
this.options = options !== null && options !== void 0 ? options : {};
}
calculateAccuracy(trainData, validationData) {
const validationSentencesQuantity = validationData.length;
let correctInferences = 0;
for (const { sentence, categories } of trainData) {
this.learn(sentence, categories);
}
for (const { sentence, categories } of validationData) {
const classification = this.classify(sentence);
const match = (this.options.returnType === 'BEST_MATCH'
? classification
: this.extractBestMatch(classification));
const categoriesToCheck = (0, lib_1.isArray)(categories)
? categories
: [categories];
if (categoriesToCheck.includes(match))
correctInferences++;
}
return (0, lib_1.returnTypeParser)(correctInferences / validationSentencesQuantity, this.options);
}
learn(sentence, inputs) {
const classifications = Array.isArray(inputs) ? inputs : [inputs];
classifications.forEach((classification) => {
let category = this.categories.find((categorie) => categorie.name === classification);
const relatedCategories = classifications.filter((item) => item !== classification);
if (category)
category.addSentence(sentence, relatedCategories);
else {
category = new category_1.Category(classification).addSentence(sentence, relatedCategories);
this.categories.push(category);
}
});
}
normalizeData(sentence) {
return sentence.toLowerCase();
}
getUnknownScore(sentence) {
return (this.normalizeData(sentence)
.split(' ')
.filter((word) => !this.getTokens().includes(word)).length /
sentence.split(' ').length);
}
classify(sentence) {
var _a;
let classification = {};
this.categories.forEach((category) => {
classification[category.name] = category.classify(sentence, this.categories);
});
let result = {};
const unknownScore = this.getUnknownScore(sentence);
const relevancySum = Object.values(classification).reduce(lib_1.sumFunc, 0) + unknownScore;
result.unknown =
((_a = this.options) === null || _a === void 0 ? void 0 : _a.returnType) === 'PERCENTAGE'
? (0, lib_1.toPercent)(!!relevancySum ? unknownScore / relevancySum : unknownScore)
: !!relevancySum
? unknownScore / relevancySum
: unknownScore;
for (const [name, relevancy] of Object.entries(classification)) {
const value = relevancySum ? relevancy / relevancySum : 0;
result[name] = (0, lib_1.returnTypeParser)(value !== null && value !== void 0 ? value : 0, this.options);
}
this.freeMemory();
if (this.options.returnType === 'BEST_MATCH') {
return this.extractBestMatch(result);
}
return result;
}
extractBestMatch(result) {
const descendingSortedResult = Object.entries(result).sort((a, b) => (0, lib_1.toNumber)(b[1]) - (0, lib_1.toNumber)(a[1]));
const [[bestMatch]] = descendingSortedResult;
return bestMatch;
}
getTokens() {
return [
...new Set(this.categories.flatMap((category) => category.getTokens().map((token) => {
const [tokenName] = token;
return tokenName;
}))),
];
}
resetKnowledge() {
this.categories = [];
}
ensureAttributesForCreation(filepath, format, allowedExtentions) {
return __awaiter(this, void 0, void 0, function* () {
const { ext, dir } = (0, path_1.parse)(filepath);
if (!allowedExtentions.includes(ext))
throw new Error(`'${filepath}' is an invalid filepath! The file must be a valid ${format}.`);
if (dir && !(0, fs_1.existsSync)(dir)) {
yield (0, promises_1.mkdir)(dir, { recursive: true });
}
});
}
getModel() {
this.analize();
const json = {
options: this.options,
categories: [],
};
this.categories.forEach((category) => json.categories.push({
name: category.name,
tokens: category.getTokens(),
}));
return json;
}
toYAML(filepath) {
return __awaiter(this, void 0, void 0, function* () {
yield this.ensureAttributesForCreation(filepath, 'YAML', ['.yml', '.yaml']);
yield (0, promises_1.writeFile)(filepath, js_yaml_1.default.dump(this.getModel()));
});
}
toJSON(filepath) {
return __awaiter(this, void 0, void 0, function* () {
yield this.ensureAttributesForCreation(filepath, 'JSON', ['.json']);
yield (0, promises_1.writeFile)(filepath, JSON.stringify(this.getModel(), null, 2));
});
}
fromYAML(filePath, options) {
return __awaiter(this, void 0, void 0, function* () {
const file = yield (0, promises_1.readFile)(filePath);
const classifierProps = js_yaml_1.default.load(file.toString());
this.resetKnowledge();
this.options = Object.assign(Object.assign({}, classifierProps.options), options);
classifierProps.categories.forEach((category) => this.categories.push(new category_1.Category(category.name, category.tokens)));
});
}
fromJSON(filePath, options) {
return __awaiter(this, void 0, void 0, function* () {
const file = yield (0, promises_1.readFile)(filePath);
const classifierProps = JSON.parse(file.toString());
this.resetKnowledge();
this.options = Object.assign(Object.assign({}, classifierProps.options), options);
classifierProps.categories.forEach((category) => this.categories.push(new category_1.Category(category.name, category.tokens)));
});
}
freeMemory() {
this.categories.forEach((category) => (category.sentences = []));
}
analize() {
this.categories.forEach((category) => category.analize(this.categories));
}
}
exports.Classifier = Classifier;