anylang
Version:
A translator's kit that uses the free APIs of Google Translate, Yandex, Bing, ChatGPT, and other LLMs
62 lines (60 loc) • 8.65 kB
JavaScript
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
import { pipeline } from '@xenova/transformers';
function cosineSimilarity(a, b) {
const dot = a.reduce((sum, val, i) => sum + val * b[i], 0);
const normA = Math.sqrt(a.reduce((sum, val) => sum + val * val, 0));
const normB = Math.sqrt(b.reduce((sum, val) => sum + val * val, 0));
return dot / (normA * normB);
}
class SimilarityScore {
constructor() {
this.pipe = null;
}
getPipe() {
if (this.pipe)
return this.pipe;
this.pipe = pipeline('feature-extraction', 'Xenova/all-MiniLM-L6-v2').catch((error) => {
this.pipe = null;
throw error;
});
return this.pipe;
}
getEmbedding(text) {
return __awaiter(this, void 0, void 0, function* () {
const pipe = yield this.getPipe();
const output = yield pipe(text, { pooling: 'mean', normalize: true });
return Array.from(output.data);
});
}
score(text1, text2) {
return __awaiter(this, void 0, void 0, function* () {
return cosineSimilarity(yield this.getEmbedding(text1), yield this.getEmbedding(text2));
});
}
}
export function getTranslatorsScore(translators, reference) {
return __awaiter(this, void 0, void 0, function* () {
const similarity = new SimilarityScore();
const translatorsScore = {};
for (const translator of translators) {
console.log(`Score translator ${translator.translatorName}...`);
const translatorInstance = new translator();
const translation = yield translatorInstance.translate(reference.text, 'en', 'ru');
const score = yield similarity.score(reference.translation, translation);
console.log(`Score is ${score}`);
translatorsScore[translator.translatorName] = score;
}
return Object.entries(translatorsScore)
.map(([name, score]) => ({ name, score }))
.sort((a, b) => b.score - a.score);
});
}
//# sourceMappingURL=data:application/json;charset=utf8;base64,eyJ2ZXJzaW9uIjozLCJzb3VyY2VzIjpbInRyYW5zbGF0b3JzL19fdGVzdHNfXy9iZW5jaG1hcmsudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6Ijs7Ozs7Ozs7O0FBQUEsT0FBTyxFQUE2QixRQUFRLEVBQUUsTUFBTSxzQkFBc0IsQ0FBQztBQUkzRSxTQUFTLGdCQUFnQixDQUFDLENBQVcsRUFBRSxDQUFXO0lBQ2pELE1BQU0sR0FBRyxHQUFHLENBQUMsQ0FBQyxNQUFNLENBQUMsQ0FBQyxHQUFHLEVBQUUsR0FBRyxFQUFFLENBQUMsRUFBRSxFQUFFLENBQUMsR0FBRyxHQUFHLEdBQUcsR0FBRyxDQUFDLENBQUMsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBQUM7SUFDM0QsTUFBTSxLQUFLLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsTUFBTSxDQUFDLENBQUMsR0FBRyxFQUFFLEdBQUcsRUFBRSxFQUFFLENBQUMsR0FBRyxHQUFHLEdBQUcsR0FBRyxHQUFHLEVBQUUsQ0FBQyxDQUFDLENBQUMsQ0FBQztJQUNwRSxNQUFNLEtBQUssR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUMsQ0FBQyxNQUFNLENBQUMsQ0FBQyxHQUFHLEVBQUUsR0FBRyxFQUFFLEVBQUUsQ0FBQyxHQUFHLEdBQUcsR0FBRyxHQUFHLEdBQUcsRUFBRSxDQUFDLENBQUMsQ0FBQyxDQUFDO0lBQ3BFLE9BQU8sR0FBRyxHQUFHLENBQUMsS0FBSyxHQUFHLEtBQUssQ0FBQyxDQUFDO0FBQzlCLENBQUM7QUFFRCxNQUFNLGVBQWU7SUFBckI7UUFDUyxTQUFJLEdBQ1gsSUFBSSxDQUFDO0lBMEJQLENBQUM7SUF6QlEsT0FBTztRQUNkLElBQUksSUFBSSxDQUFDLElBQUk7WUFBRSxPQUFPLElBQUksQ0FBQyxJQUFJLENBQUM7UUFFaEMsSUFBSSxDQUFDLElBQUksR0FBRyxRQUFRLENBQUMsb0JBQW9CLEVBQUUseUJBQXlCLENBQUMsQ0FBQyxLQUFLLENBQzFFLENBQUMsS0FBYyxFQUFFLEVBQUU7WUFDbEIsSUFBSSxDQUFDLElBQUksR0FBRyxJQUFJLENBQUM7WUFDakIsTUFBTSxLQUFLLENBQUM7UUFDYixDQUFDLENBQ0QsQ0FBQztRQUNGLE9BQU8sSUFBSSxDQUFDLElBQUksQ0FBQztJQUNsQixDQUFDO0lBRVksWUFBWSxDQUFDLElBQVk7O1lBQ3JDLE1BQU0sSUFBSSxHQUFHLE1BQU0sSUFBSSxDQUFDLE9BQU8sRUFBRSxDQUFDO1lBRWxDLE1BQU0sTUFBTSxHQUFHLE1BQU0sSUFBSSxDQUFDLElBQUksRUFBRSxFQUFFLE9BQU8sRUFBRSxNQUFNLEVBQUUsU0FBUyxFQUFFLElBQUksRUFBRSxDQUFDLENBQUM7WUFDdEUsT0FBTyxLQUFLLENBQUMsSUFBSSxDQUFDLE1BQU0sQ0FBQyxJQUFJLENBQWEsQ0FBQztRQUM1QyxDQUFDO0tBQUE7SUFFWSxLQUFLLENBQUMsS0FBYSxFQUFFLEtBQWE7O1lBQzlDLE9BQU8sZ0JBQWdCLENBQ3RCLE1BQU0sSUFBSSxDQUFDLFlBQVksQ0FBQyxLQUFLLENBQUMsRUFDOUIsTUFBTSxJQUFJLENBQUMsWUFBWSxDQUFDLEtBQUssQ0FBQyxDQUM5QixDQUFDO1FBQ0gsQ0FBQztLQUFBO0NBQ0Q7QUFFRCxNQUFNLFVBQWdCLG1CQUFtQixDQUN4QyxXQUFvQyxFQUNwQyxTQUdDOztRQUVELE1BQU0sVUFBVSxHQUFHLElBQUksZUFBZSxFQUFFLENBQUM7UUFFekMsTUFBTSxnQkFBZ0IsR0FBMkIsRUFBRSxDQUFDO1FBQ3BELEtBQUssTUFBTSxVQUFVLElBQUksV0FBVyxFQUFFLENBQUM7WUFDdEMsT0FBTyxDQUFDLEdBQUcsQ0FBQyxvQkFBb0IsVUFBVSxDQUFDLGNBQWMsS0FBSyxDQUFDLENBQUM7WUFFaEUsTUFBTSxrQkFBa0IsR0FBRyxJQUFJLFVBQVUsRUFBRSxDQUFDO1lBQzVDLE1BQU0sV0FBVyxHQUFHLE1BQU0sa0JBQWtCLENBQUMsU0FBUyxDQUNyRCxTQUFTLENBQUMsSUFBSSxFQUNkLElBQUksRUFDSixJQUFJLENBQ0osQ0FBQztZQUVGLE1BQU0sS0FBSyxHQUFHLE1BQU0sVUFBVSxDQUFDLEtBQUssQ0FBQyxTQUFTLENBQUMsV0FBVyxFQUFFLFdBQVcsQ0FBQyxDQUFDO1lBQ3pFLE9BQU8sQ0FBQyxHQUFHLENBQUMsWUFBWSxLQUFLLEVBQUUsQ0FBQyxDQUFDO1lBRWpDLGdCQUFnQixDQUFDLFVBQVUsQ0FBQyxjQUFjLENBQUMsR0FBRyxLQUFLLENBQUM7UUFDckQsQ0FBQztRQUVELE9BQU8sTUFBTSxDQUFDLE9BQU8sQ0FBQyxnQkFBZ0IsQ0FBQzthQUNyQyxHQUFHLENBQUMsQ0FBQyxDQUFDLElBQUksRUFBRSxLQUFLLENBQUMsRUFBRSxFQUFFLENBQUMsQ0FBQyxFQUFFLElBQUksRUFBRSxLQUFLLEVBQUUsQ0FBQyxDQUFDO2FBQ3pDLElBQUksQ0FBQyxDQUFDLENBQUMsRUFBRSxDQUFDLEVBQUUsRUFBRSxDQUFDLENBQUMsQ0FBQyxLQUFLLEdBQUcsQ0FBQyxDQUFDLEtBQUssQ0FBQyxDQUFDO0lBQ3JDLENBQUM7Q0FBQSIsImZpbGUiOiJ0cmFuc2xhdG9ycy9fX3Rlc3RzX18vYmVuY2htYXJrLmpzIiwic291cmNlc0NvbnRlbnQiOlsiaW1wb3J0IHsgRmVhdHVyZUV4dHJhY3Rpb25QaXBlbGluZSwgcGlwZWxpbmUgfSBmcm9tICdAeGVub3ZhL3RyYW5zZm9ybWVycyc7XG5cbmltcG9ydCB7IFRyYW5zbGF0b3JDb25zdHJ1Y3RvciB9IGZyb20gJy4uL1RyYW5zbGF0b3InO1xuXG5mdW5jdGlvbiBjb3NpbmVTaW1pbGFyaXR5KGE6IG51bWJlcltdLCBiOiBudW1iZXJbXSk6IG51bWJlciB7XG5cdGNvbnN0IGRvdCA9IGEucmVkdWNlKChzdW0sIHZhbCwgaSkgPT4gc3VtICsgdmFsICogYltpXSwgMCk7XG5cdGNvbnN0IG5vcm1BID0gTWF0aC5zcXJ0KGEucmVkdWNlKChzdW0sIHZhbCkgPT4gc3VtICsgdmFsICogdmFsLCAwKSk7XG5cdGNvbnN0IG5vcm1CID0gTWF0aC5zcXJ0KGIucmVkdWNlKChzdW0sIHZhbCkgPT4gc3VtICsgdmFsICogdmFsLCAwKSk7XG5cdHJldHVybiBkb3QgLyAobm9ybUEgKiBub3JtQik7XG59XG5cbmNsYXNzIFNpbWlsYXJpdHlTY29yZSB7XG5cdHByaXZhdGUgcGlwZTogUHJvbWlzZTxGZWF0dXJlRXh0cmFjdGlvblBpcGVsaW5lPiB8IEZlYXR1cmVFeHRyYWN0aW9uUGlwZWxpbmUgfCBudWxsID1cblx0XHRudWxsO1xuXHRwcml2YXRlIGdldFBpcGUoKSB7XG5cdFx0aWYgKHRoaXMucGlwZSkgcmV0dXJuIHRoaXMucGlwZTtcblxuXHRcdHRoaXMucGlwZSA9IHBpcGVsaW5lKCdmZWF0dXJlLWV4dHJhY3Rpb24nLCAnWGVub3ZhL2FsbC1NaW5pTE0tTDYtdjInKS5jYXRjaChcblx0XHRcdChlcnJvcjogdW5rbm93bikgPT4ge1xuXHRcdFx0XHR0aGlzLnBpcGUgPSBudWxsO1xuXHRcdFx0XHR0aHJvdyBlcnJvcjtcblx0XHRcdH0sXG5cdFx0KTtcblx0XHRyZXR1cm4gdGhpcy5waXBlO1xuXHR9XG5cblx0cHVibGljIGFzeW5jIGdldEVtYmVkZGluZyh0ZXh0OiBzdHJpbmcpOiBQcm9taXNlPG51bWJlcltdPiB7XG5cdFx0Y29uc3QgcGlwZSA9IGF3YWl0IHRoaXMuZ2V0UGlwZSgpO1xuXG5cdFx0Y29uc3Qgb3V0cHV0ID0gYXdhaXQgcGlwZSh0ZXh0LCB7IHBvb2xpbmc6ICdtZWFuJywgbm9ybWFsaXplOiB0cnVlIH0pO1xuXHRcdHJldHVybiBBcnJheS5mcm9tKG91dHB1dC5kYXRhKSBhcyBudW1iZXJbXTtcblx0fVxuXG5cdHB1YmxpYyBhc3luYyBzY29yZSh0ZXh0MTogc3RyaW5nLCB0ZXh0Mjogc3RyaW5nKSB7XG5cdFx0cmV0dXJuIGNvc2luZVNpbWlsYXJpdHkoXG5cdFx0XHRhd2FpdCB0aGlzLmdldEVtYmVkZGluZyh0ZXh0MSksXG5cdFx0XHRhd2FpdCB0aGlzLmdldEVtYmVkZGluZyh0ZXh0MiksXG5cdFx0KTtcblx0fVxufVxuXG5leHBvcnQgYXN5bmMgZnVuY3Rpb24gZ2V0VHJhbnNsYXRvcnNTY29yZShcblx0dHJhbnNsYXRvcnM6IFRyYW5zbGF0b3JDb25zdHJ1Y3RvcltdLFxuXHRyZWZlcmVuY2U6IHtcblx0XHR0ZXh0OiBzdHJpbmc7XG5cdFx0dHJhbnNsYXRpb246IHN0cmluZztcblx0fSxcbikge1xuXHRjb25zdCBzaW1pbGFyaXR5ID0gbmV3IFNpbWlsYXJpdHlTY29yZSgpO1xuXG5cdGNvbnN0IHRyYW5zbGF0b3JzU2NvcmU6IFJlY29yZDxzdHJpbmcsIG51bWJlcj4gPSB7fTtcblx0Zm9yIChjb25zdCB0cmFuc2xhdG9yIG9mIHRyYW5zbGF0b3JzKSB7XG5cdFx0Y29uc29sZS5sb2coYFNjb3JlIHRyYW5zbGF0b3IgJHt0cmFuc2xhdG9yLnRyYW5zbGF0b3JOYW1lfS4uLmApO1xuXG5cdFx0Y29uc3QgdHJhbnNsYXRvckluc3RhbmNlID0gbmV3IHRyYW5zbGF0b3IoKTtcblx0XHRjb25zdCB0cmFuc2xhdGlvbiA9IGF3YWl0IHRyYW5zbGF0b3JJbnN0YW5jZS50cmFuc2xhdGUoXG5cdFx0XHRyZWZlcmVuY2UudGV4dCxcblx0XHRcdCdlbicsXG5cdFx0XHQncnUnLFxuXHRcdCk7XG5cblx0XHRjb25zdCBzY29yZSA9IGF3YWl0IHNpbWlsYXJpdHkuc2NvcmUocmVmZXJlbmNlLnRyYW5zbGF0aW9uLCB0cmFuc2xhdGlvbik7XG5cdFx0Y29uc29sZS5sb2coYFNjb3JlIGlzICR7c2NvcmV9YCk7XG5cblx0XHR0cmFuc2xhdG9yc1Njb3JlW3RyYW5zbGF0b3IudHJhbnNsYXRvck5hbWVdID0gc2NvcmU7XG5cdH1cblxuXHRyZXR1cm4gT2JqZWN0LmVudHJpZXModHJhbnNsYXRvcnNTY29yZSlcblx0XHQubWFwKChbbmFtZSwgc2NvcmVdKSA9PiAoeyBuYW1lLCBzY29yZSB9KSlcblx0XHQuc29ydCgoYSwgYikgPT4gYi5zY29yZSAtIGEuc2NvcmUpO1xufVxuIl19