ai-zero-shot-classifier
Version:
🧠powerful JavaScript library that leverages advanced AI embeddings to perform zero-shot text classification. Whether you're dealing with unlabelled data or seeking to classify text against dynamic and user-defined labels, this library provides a seamles
378 lines (367 loc) • 24.7 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.default = void 0;
var _lodash = _interopRequireDefault(require("lodash.chunk"));
var _pMap = _interopRequireDefault(require("../utils/p-map"));
var _providers = require("../providers");
var _similarityFunctions = require("../utils/similarityFunctions");
function _interopRequireDefault(e) { return e && e.__esModule ? e : { default: e }; }
function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
function _toConsumableArray(r) { return _arrayWithoutHoles(r) || _iterableToArray(r) || _unsupportedIterableToArray(r) || _nonIterableSpread(); }
function _nonIterableSpread() { throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
function _iterableToArray(r) { if ("undefined" != typeof Symbol && null != r[Symbol.iterator] || null != r["@@iterator"]) return Array.from(r); }
function _arrayWithoutHoles(r) { if (Array.isArray(r)) return _arrayLikeToArray(r); }
function _regeneratorRuntime() { "use strict"; var r = _regenerator(), e = r.m(_regeneratorRuntime), t = (Object.getPrototypeOf ? Object.getPrototypeOf(e) : e.__proto__).constructor; function n(r) { var e = "function" == typeof r && r.constructor; return !!e && (e === t || "GeneratorFunction" === (e.displayName || e.name)); } var o = { throw: 1, return: 2, break: 3, continue: 3 }; function a(r) { var e, t; return function (n) { e || (e = { stop: function stop() { return t(n.a, 2); }, catch: function _catch() { return n.v; }, abrupt: function abrupt(r, e) { return t(n.a, o[r], e); }, delegateYield: function delegateYield(r, o, a) { return e.resultName = o, t(n.d, _regeneratorValues(r), a); }, finish: function finish(r) { return t(n.f, r); } }, t = function t(r, _t, o) { n.p = e.prev, n.n = e.next; try { return r(_t, o); } finally { e.next = n.n; } }), e.resultName && (e[e.resultName] = n.v, e.resultName = void 0), e.sent = n.v, e.next = n.n; try { return r.call(this, e); } finally { n.p = e.prev, n.n = e.next; } }; } return (_regeneratorRuntime = function _regeneratorRuntime() { return { wrap: function wrap(e, t, n, o) { return r.w(a(e), t, n, o && o.reverse()); }, isGeneratorFunction: n, mark: r.m, awrap: function awrap(r, e) { return new _OverloadYield(r, e); }, AsyncIterator: _regeneratorAsyncIterator, async: function async(r, e, t, o, u) { return (n(e) ? _regeneratorAsyncGen : _regeneratorAsync)(a(r), e, t, o, u); }, keys: _regeneratorKeys, values: _regeneratorValues }; })(); }
function _regeneratorValues(e) { if (null != e) { var t = e["function" == typeof Symbol && Symbol.iterator || "@@iterator"], r = 0; if (t) return t.call(e); if ("function" == typeof e.next) return e; if (!isNaN(e.length)) return { next: function next() { return e && r >= e.length && (e = void 0), { value: e && e[r++], done: !e }; } }; } throw new TypeError(_typeof(e) + " is not iterable"); }
function _regeneratorKeys(e) { var n = Object(e), r = []; for (var t in n) r.unshift(t); return function e() { for (; r.length;) if ((t = r.pop()) in n) return e.value = t, e.done = !1, e; return e.done = !0, e; }; }
function _regeneratorAsync(n, e, r, t, o) { var a = _regeneratorAsyncGen(n, e, r, t, o); return a.next().then(function (n) { return n.done ? n.value : a.next(); }); }
function _regeneratorAsyncGen(r, e, t, o, n) { return new _regeneratorAsyncIterator(_regenerator().w(r, e, t, o), n || Promise); }
function _regeneratorAsyncIterator(t, e) { function n(r, o, i, f) { try { var c = t[r](o), u = c.value; return u instanceof _OverloadYield ? e.resolve(u.v).then(function (t) { n("next", t, i, f); }, function (t) { n("throw", t, i, f); }) : e.resolve(u).then(function (t) { c.value = t, i(c); }, function (t) { return n("throw", t, i, f); }); } catch (t) { f(t); } } var r; this.next || (_regeneratorDefine2(_regeneratorAsyncIterator.prototype), _regeneratorDefine2(_regeneratorAsyncIterator.prototype, "function" == typeof Symbol && Symbol.asyncIterator || "@asyncIterator", function () { return this; })), _regeneratorDefine2(this, "_invoke", function (t, o, i) { function f() { return new e(function (e, r) { n(t, i, e, r); }); } return r = r ? r.then(f, f) : f(); }, !0); }
function _regenerator() { /*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/babel/babel/blob/main/packages/babel-helpers/LICENSE */ var e, t, r = "function" == typeof Symbol ? Symbol : {}, n = r.iterator || "@@iterator", o = r.toStringTag || "@@toStringTag"; function i(r, n, o, i) { var c = n && n.prototype instanceof Generator ? n : Generator, u = Object.create(c.prototype); return _regeneratorDefine2(u, "_invoke", function (r, n, o) { var i, c, u, f = 0, p = o || [], y = !1, G = { p: 0, n: 0, v: e, a: d, f: d.bind(e, 4), d: function d(t, r) { return i = t, c = 0, u = e, G.n = r, a; } }; function d(r, n) { for (c = r, u = n, t = 0; !y && f && !o && t < p.length; t++) { var o, i = p[t], d = G.p, l = i[2]; r > 3 ? (o = l === n) && (c = i[4] || 3, u = i[5] === e ? i[3] : i[5], i[4] = 3, i[5] = e) : i[0] <= d && ((o = r < 2 && d < i[1]) ? (c = 0, G.v = n, G.n = i[1]) : d < l && (o = r < 3 || i[0] > n || n > l) && (i[4] = r, i[5] = n, G.n = l, c = 0)); } if (o || r > 1) return a; throw y = !0, n; } return function (o, p, l) { if (f > 1) throw TypeError("Generator is already running"); for (y && 1 === p && d(p, l), c = p, u = l; (t = c < 2 ? e : u) || !y;) { i || (c ? c < 3 ? (c > 1 && (G.n = -1), d(c, u)) : G.n = u : G.v = u); try { if (f = 2, i) { if (c || (o = "next"), t = i[o]) { if (!(t = t.call(i, u))) throw TypeError("iterator result is not an object"); if (!t.done) return t; u = t.value, c < 2 && (c = 0); } else 1 === c && (t = i.return) && t.call(i), c < 2 && (u = TypeError("The iterator does not provide a '" + o + "' method"), c = 1); i = e; } else if ((t = (y = G.n < 0) ? u : r.call(n, G)) !== a) break; } catch (t) { i = e, c = 1, u = t; } finally { f = 1; } } return { value: t, done: y }; }; }(r, o, i), !0), u; } var a = {}; function Generator() {} function GeneratorFunction() {} function GeneratorFunctionPrototype() {} t = Object.getPrototypeOf; var c = [][n] ? t(t([][n]())) : (_regeneratorDefine2(t = {}, n, function () { return this; }), t), u = GeneratorFunctionPrototype.prototype = Generator.prototype = Object.create(c); function f(e) { return Object.setPrototypeOf ? Object.setPrototypeOf(e, GeneratorFunctionPrototype) : (e.__proto__ = GeneratorFunctionPrototype, _regeneratorDefine2(e, o, "GeneratorFunction")), e.prototype = Object.create(u), e; } return GeneratorFunction.prototype = GeneratorFunctionPrototype, _regeneratorDefine2(u, "constructor", GeneratorFunctionPrototype), _regeneratorDefine2(GeneratorFunctionPrototype, "constructor", GeneratorFunction), GeneratorFunction.displayName = "GeneratorFunction", _regeneratorDefine2(GeneratorFunctionPrototype, o, "GeneratorFunction"), _regeneratorDefine2(u), _regeneratorDefine2(u, o, "Generator"), _regeneratorDefine2(u, n, function () { return this; }), _regeneratorDefine2(u, "toString", function () { return "[object Generator]"; }), (_regenerator = function _regenerator() { return { w: i, m: f }; })(); }
function _regeneratorDefine2(e, r, n, t) { var i = Object.defineProperty; try { i({}, "", {}); } catch (e) { i = 0; } _regeneratorDefine2 = function _regeneratorDefine(e, r, n, t) { if (r) i ? i(e, r, { value: n, enumerable: !t, configurable: !t, writable: !t }) : e[r] = n;else { var o = function o(r, n) { _regeneratorDefine2(e, r, function (e) { return this._invoke(r, n, e); }); }; o("next", 0), o("throw", 1), o("return", 2); } }, _regeneratorDefine2(e, r, n, t); }
function _OverloadYield(e, d) { this.v = e, this.k = d; }
function asyncGeneratorStep(n, t, e, r, o, a, c) { try { var i = n[a](c), u = i.value; } catch (n) { return void e(n); } i.done ? t(u) : Promise.resolve(u).then(r, o); }
function _asyncToGenerator(n) { return function () { var t = this, e = arguments; return new Promise(function (r, o) { var a = n.apply(t, e); function _next(n) { asyncGeneratorStep(a, r, o, _next, _throw, "next", n); } function _throw(n) { asyncGeneratorStep(a, r, o, _next, _throw, "throw", n); } _next(void 0); }); }; }
function _slicedToArray(r, e) { return _arrayWithHoles(r) || _iterableToArrayLimit(r, e) || _unsupportedIterableToArray(r, e) || _nonIterableRest(); }
function _nonIterableRest() { throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
function _unsupportedIterableToArray(r, a) { if (r) { if ("string" == typeof r) return _arrayLikeToArray(r, a); var t = {}.toString.call(r).slice(8, -1); return "Object" === t && r.constructor && (t = r.constructor.name), "Map" === t || "Set" === t ? Array.from(r) : "Arguments" === t || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(t) ? _arrayLikeToArray(r, a) : void 0; } }
function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length); for (var e = 0, n = Array(a); e < a; e++) n[e] = r[e]; return n; }
function _iterableToArrayLimit(r, l) { var t = null == r ? null : "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"]; if (null != t) { var e, n, i, u, a = [], f = !0, o = !1; try { if (i = (t = t.call(r)).next, 0 === l) { if (Object(t) !== t) return; f = !1; } else for (; !(f = (e = i.call(t)).done) && (a.push(e.value), a.length !== l); f = !0); } catch (r) { o = !0, n = r; } finally { try { if (!f && null != t.return && (u = t.return(), Object(u) !== u)) return; } finally { if (o) throw n; } } return a; } }
function _arrayWithHoles(r) { if (Array.isArray(r)) return r; }
function _classCallCheck(a, n) { if (!(a instanceof n)) throw new TypeError("Cannot call a class as a function"); }
function _defineProperties(e, r) { for (var t = 0; t < r.length; t++) { var o = r[t]; o.enumerable = o.enumerable || !1, o.configurable = !0, "value" in o && (o.writable = !0), Object.defineProperty(e, _toPropertyKey(o.key), o); } }
function _createClass(e, r, t) { return r && _defineProperties(e.prototype, r), t && _defineProperties(e, t), Object.defineProperty(e, "prototype", { writable: !1 }), e; }
function _defineProperty(e, r, t) { return (r = _toPropertyKey(r)) in e ? Object.defineProperty(e, r, { value: t, enumerable: !0, configurable: !0, writable: !0 }) : e[r] = t, e; }
function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == _typeof(i) ? i : i + ""; }
function _toPrimitive(t, r) { if ("object" != _typeof(t) || !t) return t; var e = t[Symbol.toPrimitive]; if (void 0 !== e) { var i = e.call(t, r || "default"); if ("object" != _typeof(i)) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === r ? String : Number)(t); }
/**
* Supported providers
*/
var SUPPORTED_PROVIDERS = ['openai', 'groq'];
/**
* Validate provider
*
* @param provider
* @throws Error if provider is not supported
*/
var _validateProvider = function _validateProvider(provider) {
if (!SUPPORTED_PROVIDERS.includes(provider)) {
throw new Error("Unsupported provider \"".concat(provider, "\". Must be one of ").concat(SUPPORTED_PROVIDERS.join(', ')));
}
};
/**
* Zero-shot classifier
*/
var ZeroShotClassifier = /*#__PURE__*/function () {
function ZeroShotClassifier(config) {
_classCallCheck(this, ZeroShotClassifier);
/** Provider used for API */
_defineProperty(this, "provider", void 0);
/** Model used for classification */
_defineProperty(this, "model", void 0);
/** Provider API Key */
_defineProperty(this, "apiKey", void 0);
/** Labels to classify against */
_defineProperty(this, "labels", void 0);
/** Dimensions used for embeddings */
_defineProperty(this, "dimensions", void 0);
/** Labels cache */
_defineProperty(this, "labelsCache", void 0);
/** Data cache */
_defineProperty(this, "dataCache", void 0);
/** API client */
_defineProperty(this, "client", void 0);
var _config$model = config.model,
model = _config$model === void 0 ? 'text-embedding-3-small' : _config$model,
_config$provider = config.provider,
provider = _config$provider === void 0 ? 'openai' : _config$provider,
apiKey = config.apiKey,
_config$labels = config.labels,
labels = _config$labels === void 0 ? [] : _config$labels,
dimensions = config.dimensions,
_config$labelsCache = config.labelsCache,
labelsCache = _config$labelsCache === void 0 ? {} : _config$labelsCache,
_config$dataCache = config.dataCache,
dataCache = _config$dataCache === void 0 ? {} : _config$dataCache;
_validateProvider(provider);
this.model = model;
this.provider = provider;
this.apiKey = apiKey;
this.labels = labels || [];
this.dimensions = dimensions;
this.labelsCache = labelsCache || {};
this.dataCache = dataCache || {};
this._createAndSetClient({
model: model,
provider: provider,
apiKey: apiKey
});
}
/**
* Create and set client based on provider, model and config
* @param config
*/
return _createClass(ZeroShotClassifier, [{
key: "_createAndSetClient",
value: function _createAndSetClient(config) {
this.client = (0, _providers.getProvider)(config.provider).createClient(config);
}
/**
* Set labels to classify against and clear labels cache
* @param labels string[]
*/
}, {
key: "setLabels",
value: function setLabels(labels) {
this.labels = labels;
// Clear cache from stale entries based on updated labels
this.labelsCache = Object.fromEntries(Object.entries(this.labelsCache).filter(function (_ref) {
var _ref2 = _slicedToArray(_ref, 1),
key = _ref2[0];
return labels.includes(key);
}));
}
/**
* Set model and provider used for classification
* @param model string
* @param provider string
*/
}, {
key: "setProviderAndModel",
value: function setProviderAndModel() {
var provider = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 'openai';
var model = arguments.length > 1 ? arguments[1] : undefined;
var apiKey = arguments.length > 2 ? arguments[2] : undefined;
var dimensions = arguments.length > 3 ? arguments[3] : undefined;
_validateProvider(provider);
// clear cache if provider or model changes
if (provider !== this.provider || model !== this.model || dimensions !== this.dimensions) {
this.clearAllCaches();
}
this.provider = provider;
this.model = model;
this.apiKey = apiKey;
this.dimensions = dimensions;
this._createAndSetClient({
model: this.model,
provider: this.provider,
apiKey: this.apiKey
});
}
/**
* Get embeddings in batches with concurrency
*/
}, {
key: "getEmbeddings",
value: (function () {
var _getEmbeddings = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee2(texts, batchSize, concurrency) {
var _this = this;
var type,
cache,
uncachedTexts,
chunks,
_args2 = arguments;
return _regeneratorRuntime().wrap(function _callee2$(_context2) {
while (1) switch (_context2.prev = _context2.next) {
case 0:
type = _args2.length > 3 && _args2[3] !== undefined ? _args2[3] : 'data';
cache = type === 'label' ? this.labelsCache : this.dataCache;
uncachedTexts = texts.filter(function (text) {
return !cache[text];
});
chunks = (0, _lodash.default)(uncachedTexts, batchSize);
_context2.next = 6;
return (0, _pMap.default)(chunks, /*#__PURE__*/function () {
var _ref3 = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(currChunk) {
var provider, response;
return _regeneratorRuntime().wrap(function _callee$(_context) {
while (1) switch (_context.prev = _context.next) {
case 0:
provider = (0, _providers.getProvider)(_this.provider);
_context.next = 3;
return provider.createEmbedding(_this.client, {
model: _this.model,
input: currChunk,
dimensions: _this.dimensions
});
case 3:
response = _context.sent;
currChunk.forEach(function (text, index) {
cache[text] = response[index].embedding;
});
return _context.abrupt("return", response.map(function (r) {
return r.embedding;
}));
case 6:
case "end":
return _context.stop();
}
}, _callee);
}));
return function (_x4) {
return _ref3.apply(this, arguments);
};
}(), {
concurrency: concurrency
});
case 6:
return _context2.abrupt("return", texts.map(function (text) {
return cache[text];
}));
case 7:
case "end":
return _context2.stop();
}
}, _callee2, this);
}));
function getEmbeddings(_x, _x2, _x3) {
return _getEmbeddings.apply(this, arguments);
}
return getEmbeddings;
}()
/**
* Clear cache for data embeddings
*/
)
}, {
key: "clearDataCache",
value: function clearDataCache() {
this.dataCache = {};
}
/**
* Clear cache for label embeddings
*/
}, {
key: "clearLabelsCache",
value: function clearLabelsCache() {
this.labelsCache = {};
}
/**
* Clear all caches
*/
}, {
key: "clearAllCaches",
value: function clearAllCaches() {
this.clearDataCache();
this.clearLabelsCache();
}
/**
* Classify data
*
* @param data string[]
* @param config object
* @returns Object[]
*/
}, {
key: "classify",
value: (function () {
var _classify = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee5(data) {
var _this2 = this;
var config,
_config$similarity,
similarity,
_config$embeddingBatc,
embeddingBatchSizeData,
_config$embeddingBatc2,
embeddingBatchSizeLabels,
_config$embeddingConc,
embeddingConcurrencyData,
_config$embeddingConc2,
embeddingConcurrencyLabels,
_config$comparingConc,
comparingConcurrencyTop,
_config$comparingConc2,
comparingConcurrencyBottom,
_yield$Promise$all,
_yield$Promise$all2,
labelsEmbeddings,
dataEmbeddings,
getSimilarity,
result,
_args5 = arguments;
return _regeneratorRuntime().wrap(function _callee5$(_context5) {
while (1) switch (_context5.prev = _context5.next) {
case 0:
config = _args5.length > 1 && _args5[1] !== undefined ? _args5[1] : {};
if (this.labels.length) {
_context5.next = 3;
break;
}
throw new Error('Labels must be set.');
case 3:
_config$similarity = config.similarity, similarity = _config$similarity === void 0 ? 'cosine' : _config$similarity, _config$embeddingBatc = config.embeddingBatchSizeData, embeddingBatchSizeData = _config$embeddingBatc === void 0 ? 50 : _config$embeddingBatc, _config$embeddingBatc2 = config.embeddingBatchSizeLabels, embeddingBatchSizeLabels = _config$embeddingBatc2 === void 0 ? 50 : _config$embeddingBatc2, _config$embeddingConc = config.embeddingConcurrencyData, embeddingConcurrencyData = _config$embeddingConc === void 0 ? 5 : _config$embeddingConc, _config$embeddingConc2 = config.embeddingConcurrencyLabels, embeddingConcurrencyLabels = _config$embeddingConc2 === void 0 ? 5 : _config$embeddingConc2, _config$comparingConc = config.comparingConcurrencyTop, comparingConcurrencyTop = _config$comparingConc === void 0 ? 10 : _config$comparingConc, _config$comparingConc2 = config.comparingConcurrencyBottom, comparingConcurrencyBottom = _config$comparingConc2 === void 0 ? 10 : _config$comparingConc2; // Parallel embedding computation for labels and data
_context5.next = 6;
return Promise.all([this.getEmbeddings(this.labels, embeddingBatchSizeLabels, embeddingConcurrencyLabels, 'label'), this.getEmbeddings(data, embeddingBatchSizeData, embeddingConcurrencyData, 'data')]);
case 6:
_yield$Promise$all = _context5.sent;
_yield$Promise$all2 = _slicedToArray(_yield$Promise$all, 2);
labelsEmbeddings = _yield$Promise$all2[0];
dataEmbeddings = _yield$Promise$all2[1];
/** similarity getter function */
getSimilarity = (0, _similarityFunctions.getSimilarityFunction)(similarity);
_context5.next = 13;
return (0, _pMap.default)(dataEmbeddings, /*#__PURE__*/function () {
var _ref4 = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee4(dataEmbedding) {
var similarities, bestIndex;
return _regeneratorRuntime().wrap(function _callee4$(_context4) {
while (1) switch (_context4.prev = _context4.next) {
case 0:
_context4.next = 2;
return (0, _pMap.default)(labelsEmbeddings, /*#__PURE__*/function () {
var _ref5 = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee3(labelEmbedding) {
return _regeneratorRuntime().wrap(function _callee3$(_context3) {
while (1) switch (_context3.prev = _context3.next) {
case 0:
return _context3.abrupt("return", getSimilarity(dataEmbedding, labelEmbedding));
case 1:
case "end":
return _context3.stop();
}
}, _callee3);
}));
return function (_x7) {
return _ref5.apply(this, arguments);
};
}(), {
concurrency: comparingConcurrencyBottom
});
case 2:
similarities = _context4.sent;
// find closest label based on similarity
bestIndex = similarities.indexOf(Math[similarity === 'euclidean' ? 'min' : 'max'].apply(Math, _toConsumableArray(similarities)));
return _context4.abrupt("return", {
label: _this2.labels[bestIndex],
confidence: similarities[bestIndex] // Include confidence score
});
case 5:
case "end":
return _context4.stop();
}
}, _callee4);
}));
return function (_x6) {
return _ref4.apply(this, arguments);
};
}(), {
concurrency: comparingConcurrencyTop
});
case 13:
result = _context5.sent;
return _context5.abrupt("return", result);
case 15:
case "end":
return _context5.stop();
}
}, _callee5, this);
}));
function classify(_x5) {
return _classify.apply(this, arguments);
}
return classify;
}())
}]);
}();
var _default = exports.default = ZeroShotClassifier;
//# sourceMappingURL=index.js.map