cozy-dataproxy-lib
Version:
Library meant to be by Cozy Cloud's DataProxy apps for data manipulation
1,199 lines (971 loc) • 45 kB
JavaScript
"use strict";
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault");
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.SearchEngine = void 0;
var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));
var _slicedToArray2 = _interopRequireDefault(require("@babel/runtime/helpers/slicedToArray"));
var _toConsumableArray2 = _interopRequireDefault(require("@babel/runtime/helpers/toConsumableArray"));
var _asyncToGenerator2 = _interopRequireDefault(require("@babel/runtime/helpers/asyncToGenerator"));
var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/classCallCheck"));
var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));
var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));
var _cozyClient = require("cozy-client");
var _cozyMinilog = _interopRequireDefault(require("cozy-minilog"));
var _cozyRealtime = _interopRequireWildcard(require("cozy-realtime"));
var _consts = require("./consts");
var _client = require("./helpers/client");
var _normalizeSearchResult = require("./helpers/normalizeSearchResult");
var _utils = require("./helpers/utils");
var _indexDocs = require("./indexDocs");
var _queries = require("./queries");
var _storage = require("./storage");
var _types = require("./types");
function _getRequireWildcardCache(nodeInterop) { if (typeof WeakMap !== "function") return null; var cacheBabelInterop = new WeakMap(); var cacheNodeInterop = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(nodeInterop) { return nodeInterop ? cacheNodeInterop : cacheBabelInterop; })(nodeInterop); }
function _interopRequireWildcard(obj, nodeInterop) { if (!nodeInterop && obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { default: obj }; } var cache = _getRequireWildcardCache(nodeInterop); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (key !== "default" && Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj.default = obj; if (cache) { cache.set(obj, newObj); } return newObj; }
function _createForOfIteratorHelper(o, allowArrayLike) { var it = typeof Symbol !== "undefined" && o[Symbol.iterator] || o["@@iterator"]; if (!it) { if (Array.isArray(o) || (it = _unsupportedIterableToArray(o)) || allowArrayLike && o && typeof o.length === "number") { if (it) o = it; var i = 0; var F = function F() {}; return { s: F, n: function n() { if (i >= o.length) return { done: true }; return { done: false, value: o[i++] }; }, e: function e(_e) { throw _e; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var normalCompletion = true, didErr = false, err; return { s: function s() { it = it.call(o); }, n: function n() { var step = it.next(); normalCompletion = step.done; return step; }, e: function e(_e2) { didErr = true; err = _e2; }, f: function f() { try { if (!normalCompletion && it.return != null) it.return(); } finally { if (didErr) throw err; } } }; }
function _unsupportedIterableToArray(o, minLen) { if (!o) return; if (typeof o === "string") return _arrayLikeToArray(o, minLen); var n = Object.prototype.toString.call(o).slice(8, -1); if (n === "Object" && o.constructor) n = o.constructor.name; if (n === "Map" || n === "Set") return Array.from(o); if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _arrayLikeToArray(o, minLen); }
function _arrayLikeToArray(arr, len) { if (len == null || len > arr.length) len = arr.length; for (var i = 0, arr2 = new Array(len); i < len; i++) { arr2[i] = arr[i]; } return arr2; }
function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); enumerableOnly && (symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; })), keys.push.apply(keys, symbols); } return keys; }
function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = null != arguments[i] ? arguments[i] : {}; i % 2 ? ownKeys(Object(source), !0).forEach(function (key) { (0, _defineProperty2.default)(target, key, source[key]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)) : ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } return target; }
var log = (0, _cozyMinilog.default)('🗂️ [Indexing]');
var SearchEngine = /*#__PURE__*/function () {
function SearchEngine(client, storage, performanceApi) {
var _this = this;
var engineOptions = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : {};
(0, _classCallCheck2.default)(this, SearchEngine);
(0, _defineProperty2.default)(this, "client", void 0);
(0, _defineProperty2.default)(this, "searchIndexes", void 0);
(0, _defineProperty2.default)(this, "debouncedReplication", void 0);
(0, _defineProperty2.default)(this, "isLocalSearch", void 0);
(0, _defineProperty2.default)(this, "storage", void 0);
(0, _defineProperty2.default)(this, "performanceApi", void 0);
(0, _defineProperty2.default)(this, "engineOptions", void 0);
(0, _defineProperty2.default)(this, "sharedDrivesRealtimes", void 0);
this.client = client;
this.searchIndexes = {};
this.storage = storage;
this.performanceApi = performanceApi !== null && performanceApi !== void 0 ? performanceApi : _cozyClient.defaultPerformanceApi;
this.engineOptions = _objectSpread({
shouldInit: true
}, engineOptions);
this.sharedDrivesRealtimes = {};
this.isLocalSearch = !!(0, _client.getPouchLink)(this.client);
log.info('Use local data on trusted device: ', this.isLocalSearch);
this.debouncedReplication = function () {
var pouchLink = (0, _client.getPouchLink)(client);
if (pouchLink) {
pouchLink.startReplicationWithDebounce();
}
};
if (this.client.isLogged) {
this.afterLogin();
} else {
this.client.on('login', function () {
_this.afterLogin();
});
}
this.client.on('logout', function () {
_this.stopSharedDrivesRealtimes();
});
}
(0, _createClass2.default)(SearchEngine, [{
key: "afterLogin",
value: function afterLogin() {
if (this.engineOptions.shouldInit) {
void this.init();
} // Use replication events to have up-to-date search indexes, based on local data
this.handleReplicationEvents();
}
}, {
key: "indexDocumentsAtInit",
value: function () {
var _indexDocumentsAtInit = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee() {
var markName, lastExportDate, doctypes, _iterator, _step, doctype, searchIndex;
return _regenerator.default.wrap(function _callee$(_context) {
while (1) {
switch (_context.prev = _context.next) {
case 0:
if (this.client) {
_context.next = 2;
break;
}
return _context.abrupt("return");
case 2:
log.info('Initialize indexes...');
markName = this.performanceApi.mark('indexDocuments');
_context.next = 6;
return (0, _storage.getExportDate)(this.storage);
case 6:
lastExportDate = _context.sent;
if (!(!lastExportDate || !this.isLocalSearch)) {
_context.next = 30;
break;
}
// No persisted index: let's create them
doctypes = _consts.SEARCHABLE_DOCTYPES.concat(this.getSharedDrivesDoctypes());
_iterator = _createForOfIteratorHelper(doctypes);
_context.prev = 10;
_iterator.s();
case 12:
if ((_step = _iterator.n()).done) {
_context.next = 20;
break;
}
doctype = _step.value;
_context.next = 16;
return this.indexDocsForSearch(doctype);
case 16:
searchIndex = _context.sent;
if (searchIndex) {
this.searchIndexes[doctype] = searchIndex;
}
case 18:
_context.next = 12;
break;
case 20:
_context.next = 25;
break;
case 22:
_context.prev = 22;
_context.t0 = _context["catch"](10);
_iterator.e(_context.t0);
case 25:
_context.prev = 25;
_iterator.f();
return _context.finish(25);
case 28:
_context.next = 32;
break;
case 30:
_context.next = 32;
return this.initSearchWithIndexImport();
case 32:
this.performanceApi.measure({
markName: markName,
category: 'Search'
});
case 33:
case "end":
return _context.stop();
}
}
}, _callee, this, [[10, 22, 25, 28]]);
}));
function indexDocumentsAtInit() {
return _indexDocumentsAtInit.apply(this, arguments);
}
return indexDocumentsAtInit;
}()
}, {
key: "handleReplicationEvents",
value: function handleReplicationEvents() {
var _this2 = this;
if (!this.isLocalSearch) {
// Nothing to do here, we do not want to replicate for non-local search
return;
}
var startReplicationTime = 0,
endReplicationTime = 0;
this.client.on('pouchlink:doctypesync:end', /*#__PURE__*/function () {
var _ref = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee2(doctype) {
var normalizedDoctype, searchIndex;
return _regenerator.default.wrap(function _callee2$(_context2) {
while (1) {
switch (_context2.prev = _context2.next) {
case 0:
normalizedDoctype = (0, _utils.normalizeDoctype)(doctype);
if (!(0, _types.isSearchedDoctype)(normalizedDoctype)) {
_context2.next = 6;
break;
}
_context2.next = 4;
return _this2.indexDocsForSearch(doctype);
case 4:
searchIndex = _context2.sent;
if (searchIndex) {
_this2.searchIndexes[doctype] = searchIndex;
}
case 6:
case "end":
return _context2.stop();
}
}
}, _callee2);
}));
return function (_x) {
return _ref.apply(this, arguments);
};
}());
this.client.on('pouchlink:sync:start', function () {
log.debug('Started pouch replication');
startReplicationTime = performance.now();
});
this.client.on('pouchlink:sync:end', /*#__PURE__*/(0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee3() {
return _regenerator.default.wrap(function _callee3$(_context3) {
while (1) {
switch (_context3.prev = _context3.next) {
case 0:
log.debug('Ended pouch replication');
endReplicationTime = performance.now();
if (startReplicationTime > 0) {
// Log only if the start replication event was correctly get
log.debug("Replication took ".concat((endReplicationTime - startReplicationTime).toFixed(2), " ms"));
}
if (!(Object.keys(_this2.searchIndexes).length < 1)) {
_context3.next = 7;
break;
}
log.info('No search index found: start indexing'); // This happens at first replication, so init the indexes
_context3.next = 7;
return _this2.indexDocumentsAtInit();
case 7:
// Save up-to-date index on storage to be later imported
void (0, _storage.exportSearchIndexes)(_this2.storage, _this2.searchIndexes);
case 8:
case "end":
return _context3.stop();
}
}
}, _callee3);
})));
}
}, {
key: "init",
value: function () {
var _init = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee4() {
var realtimePlugin, pouchLink, sharedDrivesDoctypes, _iterator2, _step2, sharedDrivesDoctype, driveId;
return _regenerator.default.wrap(function _callee4$(_context4) {
while (1) {
switch (_context4.prev = _context4.next) {
case 0:
// Ensure login is done before plugin register
realtimePlugin = _cozyRealtime.RealtimePlugin;
if (!this.client.plugins[realtimePlugin.pluginName]) {
this.client.registerPlugin(realtimePlugin, {});
} // Realtime subscription
this.handleUpdatedOrCreatedDoc = this.handleUpdatedOrCreatedDoc.bind(this);
this.handleDeletedDoc = this.handleDeletedDoc.bind(this);
this.subscribeDoctype(this.client, _consts.FILES_DOCTYPE);
this.subscribeDoctype(this.client, _consts.CONTACTS_DOCTYPE);
this.subscribeDoctype(this.client, _consts.APPS_DOCTYPE);
pouchLink = (0, _client.getPouchLink)(this.client);
if (pouchLink) {
sharedDrivesDoctypes = pouchLink.getSharedDriveDoctypes();
_iterator2 = _createForOfIteratorHelper(sharedDrivesDoctypes);
try {
for (_iterator2.s(); !(_step2 = _iterator2.n()).done;) {
sharedDrivesDoctype = _step2.value;
driveId = sharedDrivesDoctype.split('-').pop();
if (driveId) {
this.addSharedDriveRealtime(driveId);
}
}
} catch (err) {
_iterator2.e(err);
} finally {
_iterator2.f();
}
}
if (this.isLocalSearch) {
this.debouncedReplication();
} // The document indexing should be performed once everything is setup
_context4.next = 12;
return this.indexDocumentsAtInit();
case 12:
case "end":
return _context4.stop();
}
}
}, _callee4, this);
}));
function init() {
return _init.apply(this, arguments);
}
return init;
}()
}, {
key: "addSharedDrive",
value: function addSharedDrive(driveId) {
var doctype = "".concat(_consts.SHARED_DRIVE_FILES_DOCTYPE, "-").concat(driveId); // Seed an index right away so realtime events for this drive's files are
// not dropped while its first replication is still running. Without this,
// a drive received over realtime stays unsearchable until the next full
// init (a page refresh), because the replication event that would build
// its index does not fire for a dynamically-added doctype.
if (!this.searchIndexes[doctype]) {
this.searchIndexes[doctype] = {
index: (0, _indexDocs.initSearchIndex)(doctype),
lastSeq: 0,
lastUpdated: new Date().toISOString()
};
}
this.addSharedDriveRealtime(driveId);
if (this.isLocalSearch) {
this.debouncedReplication();
} // Index the documents already replicated for this drive (existing files
// predate the realtime subscription) instead of waiting for a refresh.
void this.indexSharedDriveDocs(doctype);
}
}, {
key: "indexSharedDriveDocs",
value: function () {
var _indexSharedDriveDocs = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee5(doctype) {
var searchIndex;
return _regenerator.default.wrap(function _callee5$(_context5) {
while (1) {
switch (_context5.prev = _context5.next) {
case 0:
_context5.next = 2;
return this.indexDocsForSearch(doctype);
case 2:
searchIndex = _context5.sent;
if (searchIndex) {
this.searchIndexes[doctype] = searchIndex;
}
case 4:
case "end":
return _context5.stop();
}
}
}, _callee5, this);
}));
function indexSharedDriveDocs(_x2) {
return _indexSharedDriveDocs.apply(this, arguments);
}
return indexSharedDriveDocs;
}()
}, {
key: "removeSharedDrive",
value: function removeSharedDrive(driveId) {
var _this$sharedDrivesRea;
(_this$sharedDrivesRea = this.sharedDrivesRealtimes[driveId]) === null || _this$sharedDrivesRea === void 0 ? void 0 : _this$sharedDrivesRea.stop();
delete this.sharedDrivesRealtimes[driveId];
if (this.searchIndexes && this.searchIndexes["".concat(_consts.SHARED_DRIVE_FILES_DOCTYPE, "-").concat(driveId)]) {
delete this.searchIndexes["".concat(_consts.SHARED_DRIVE_FILES_DOCTYPE, "-").concat(driveId)];
}
void (0, _storage.exportSearchIndexes)(this.storage, this.searchIndexes);
}
}, {
key: "addSharedDriveRealtime",
value: function addSharedDriveRealtime(sharedDriveId) {
var _this$sharedDrivesRea2;
(_this$sharedDrivesRea2 = this.sharedDrivesRealtimes[sharedDriveId]) === null || _this$sharedDrivesRea2 === void 0 ? void 0 : _this$sharedDrivesRea2.stop(); // background: the indexer watches the drive without the user looking at
// it, so the stack must not mark the sharing as seen
var realtime = new _cozyRealtime.default({
client: this.client,
sharedDriveId: sharedDriveId,
background: true
});
this.subscribeDoctype(this.client, _consts.FILES_DOCTYPE, realtime, sharedDriveId);
this.sharedDrivesRealtimes[sharedDriveId] = realtime;
}
}, {
key: "stopSharedDrivesRealtimes",
value: function stopSharedDrivesRealtimes() {
for (var _i = 0, _Object$keys = Object.keys(this.sharedDrivesRealtimes); _i < _Object$keys.length; _i++) {
var _this$sharedDrivesRea3;
var driveId = _Object$keys[_i];
(_this$sharedDrivesRea3 = this.sharedDrivesRealtimes[driveId]) === null || _this$sharedDrivesRea3 === void 0 ? void 0 : _this$sharedDrivesRea3.stop();
}
this.sharedDrivesRealtimes = {};
}
}, {
key: "subscribeDoctype",
value: function subscribeDoctype(client, doctype, realtime, sharedDriveId) {
var _this3 = this;
/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call */
var realtimeInstance = realtime ? realtime : client.plugins.realtime;
realtimeInstance.subscribe('created', doctype, function (doc) {
return _this3.handleUpdatedOrCreatedDoc(doc, sharedDriveId);
});
realtimeInstance.subscribe('updated', doctype, function (doc) {
return _this3.handleUpdatedOrCreatedDoc(doc, sharedDriveId);
});
realtimeInstance.subscribe('deleted', doctype, function (doc) {
return _this3.handleDeletedDoc(doc, sharedDriveId);
});
/* eslint-enable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call */
}
}, {
key: "handleUpdatedOrCreatedDoc",
value: function handleUpdatedOrCreatedDoc(doc, sharedDriveId) {
var _this$searchIndexes;
var doctype = sharedDriveId ? "".concat(_consts.SHARED_DRIVE_FILES_DOCTYPE, "-").concat(sharedDriveId) : doc._type;
if (!doctype || !(0, _types.isSearchedDoctype)((0, _utils.normalizeDoctype)(doctype)) || (0, _types.isTrashedSharedDrive)(doc, sharedDriveId)) {
return;
}
var searchIndex = (_this$searchIndexes = this.searchIndexes) === null || _this$searchIndexes === void 0 ? void 0 : _this$searchIndexes[doctype];
if (!searchIndex) {
// No index yet: it will be done by querying the local db after first replication
return;
}
log.debug('[REALTIME] Update doc from index after update : ', doc._id);
if (this.isLocalSearch) {
this.debouncedReplication();
}
(0, _indexDocs.indexSingleDoc)(searchIndex.index, doc);
}
}, {
key: "handleDeletedDoc",
value: function handleDeletedDoc(doc, sharedDriveId) {
var _this$searchIndexes2;
var doctype = sharedDriveId ? "".concat(_consts.SHARED_DRIVE_FILES_DOCTYPE, "-").concat(sharedDriveId) : doc._type;
if (!doctype || !(0, _types.isSearchedDoctype)((0, _utils.normalizeDoctype)(doctype)) || (0, _types.isInSharedDrivesDir)(doc)) {
return;
}
var searchIndex = (_this$searchIndexes2 = this.searchIndexes) === null || _this$searchIndexes2 === void 0 ? void 0 : _this$searchIndexes2[doctype];
if (!searchIndex) {
// No index yet: it will be done by querying the local db after first replication
return;
}
log.debug('[REALTIME] Remove doc from index after update : ', doc._id);
this.searchIndexes[doctype].index.remove(doc._id);
if (this.isLocalSearch) {
this.debouncedReplication();
}
}
/**
* Initialize indexes by:
* - importing persisted indexes
* - Requesting remote changes if the indexes are stale
* - Query the local database to load the redux store
*/
}, {
key: "initSearchWithIndexImport",
value: function () {
var _initSearchWithIndexImport = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee6() {
var startImport, endImport, _iterator3, _step3, doctype, searchIndex, newSearchIndex;
return _regenerator.default.wrap(function _callee6$(_context6) {
while (1) {
switch (_context6.prev = _context6.next) {
case 0:
startImport = performance.now();
_context6.next = 3;
return (0, _storage.importSearchIndexes)(this.storage);
case 3:
this.searchIndexes = _context6.sent;
endImport = performance.now();
if ((0, _utils.isDebug)()) {
log.debug("Index import took ".concat((endImport - startImport).toFixed(2), " ms"));
}
_iterator3 = _createForOfIteratorHelper(_consts.SEARCHABLE_DOCTYPES);
_context6.prev = 7;
_iterator3.s();
case 9:
if ((_step3 = _iterator3.n()).done) {
_context6.next = 22;
break;
}
doctype = _step3.value;
searchIndex = this.searchIndexes[doctype];
if (searchIndex) {
_context6.next = 17;
break;
}
_context6.next = 15;
return this.indexDocsForSearch(doctype);
case 15:
newSearchIndex = _context6.sent;
if (newSearchIndex) {
this.searchIndexes[doctype] = newSearchIndex;
}
case 17:
_context6.next = 19;
return (0, _indexDocs.indexOnChanges)(this, this.searchIndexes[doctype], doctype);
case 19:
// The doctype needs some extra treatment to initialize everything correctly
void (0, _indexDocs.initDoctypeAfterIndexImport)(this.client, doctype);
case 20:
_context6.next = 9;
break;
case 22:
_context6.next = 27;
break;
case 24:
_context6.prev = 24;
_context6.t0 = _context6["catch"](7);
_iterator3.e(_context6.t0);
case 27:
_context6.prev = 27;
_iterator3.f();
return _context6.finish(27);
case 30:
case "end":
return _context6.stop();
}
}
}, _callee6, this, [[7, 24, 27, 30]]);
}));
function initSearchWithIndexImport() {
return _initSearchWithIndexImport.apply(this, arguments);
}
return initSearchWithIndexImport;
}()
}, {
key: "buildSearchIndex",
value: function buildSearchIndex(doctype, docs) {
var startTimeIndex = performance.now();
var flexsearchIndex = (0, _indexDocs.initSearchIndex)(doctype);
(0, _indexDocs.indexAllDocs)(flexsearchIndex, docs);
var endTimeIndex = performance.now();
log.debug("Create ".concat(doctype, " index took ").concat((endTimeIndex - startTimeIndex).toFixed(2), " ms"));
return flexsearchIndex;
}
}, {
key: "getLocalLastSeq",
value: function () {
var _getLocalLastSeq = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee7(doctype) {
var pouchLink, info;
return _regenerator.default.wrap(function _callee7$(_context7) {
while (1) {
switch (_context7.prev = _context7.next) {
case 0:
if (!this.isLocalSearch) {
_context7.next = 11;
break;
}
pouchLink = (0, _client.getPouchLink)(this.client);
if (!pouchLink) {
_context7.next = 8;
break;
}
_context7.next = 5;
return pouchLink.getDbInfo(doctype);
case 5:
_context7.t0 = _context7.sent;
_context7.next = 9;
break;
case 8:
_context7.t0 = null;
case 9:
info = _context7.t0;
return _context7.abrupt("return", (info === null || info === void 0 ? void 0 : info.update_seq) || 0);
case 11:
return _context7.abrupt("return", -1);
case 12:
case "end":
return _context7.stop();
}
}
}, _callee7, this);
}));
function getLocalLastSeq(_x3) {
return _getLocalLastSeq.apply(this, arguments);
}
return getLocalLastSeq;
}()
}, {
key: "initialIndexation",
value: function () {
var _initialIndexation = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee8(doctype) {
var docs, index, lastSeq;
return _regenerator.default.wrap(function _callee8$(_context8) {
while (1) {
switch (_context8.prev = _context8.next) {
case 0:
_context8.next = 2;
return (0, _queries.queryLocalOrRemoteDocs)(this.client, doctype, {
isLocalSearch: this.isLocalSearch
});
case 2:
docs = _context8.sent;
if (!(docs.length < 1)) {
_context8.next = 5;
break;
}
return _context8.abrupt("return", null);
case 5:
index = this.buildSearchIndex(doctype, docs);
_context8.next = 8;
return this.getLocalLastSeq(doctype);
case 8:
lastSeq = _context8.sent;
this.searchIndexes[doctype] = {
index: index,
lastSeq: lastSeq,
lastUpdated: new Date().toISOString()
};
return _context8.abrupt("return", this.searchIndexes[doctype]);
case 11:
case "end":
return _context8.stop();
}
}
}, _callee8, this);
}));
function initialIndexation(_x4) {
return _initialIndexation.apply(this, arguments);
}
return initialIndexation;
}()
}, {
key: "incrementalIndexation",
value: function () {
var _incrementalIndexation = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee9(doctype, searchIndex) {
var updatedSearchIndex;
return _regenerator.default.wrap(function _callee9$(_context9) {
while (1) {
switch (_context9.prev = _context9.next) {
case 0:
_context9.next = 2;
return (0, _indexDocs.indexOnChanges)(this, searchIndex, doctype);
case 2:
updatedSearchIndex = _context9.sent;
return _context9.abrupt("return", updatedSearchIndex);
case 4:
case "end":
return _context9.stop();
}
}
}, _callee9, this);
}));
function incrementalIndexation(_x5, _x6) {
return _incrementalIndexation.apply(this, arguments);
}
return incrementalIndexation;
}()
}, {
key: "indexDocsForSearch",
value: function () {
var _indexDocsForSearch = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee10(doctype) {
var markeNameIndex, searchIndex, startIndexing, index, endIndexing;
return _regenerator.default.wrap(function _callee10$(_context10) {
while (1) {
switch (_context10.prev = _context10.next) {
case 0:
markeNameIndex = this.performanceApi.mark("indexDocsForSearch ".concat(doctype));
searchIndex = this.searchIndexes[doctype];
startIndexing = performance.now();
if (searchIndex) {
_context10.next = 12;
break;
}
_context10.next = 6;
return this.initialIndexation(doctype);
case 6:
index = _context10.sent;
if (index) {
_context10.next = 10;
break;
}
this.performanceApi.measure({
markName: markeNameIndex,
measureName: "".concat(markeNameIndex, " initial indexation"),
category: 'Search'
});
return _context10.abrupt("return", null);
case 10:
_context10.next = 15;
break;
case 12:
_context10.next = 14;
return this.incrementalIndexation(doctype, searchIndex);
case 14:
index = _context10.sent;
case 15:
endIndexing = performance.now();
if ((0, _utils.isDebug)()) {
log.debug("Indexing ".concat(doctype, " took ").concat((endIndexing - startIndexing).toFixed(2), " ms"));
}
this.performanceApi.measure({
markName: markeNameIndex,
measureName: "".concat(markeNameIndex, " incremental indexation"),
category: 'Search'
});
return _context10.abrupt("return", index);
case 19:
case "end":
return _context10.stop();
}
}
}, _callee10, this);
}));
function indexDocsForSearch(_x7) {
return _indexDocsForSearch.apply(this, arguments);
}
return indexDocsForSearch;
}()
}, {
key: "search",
value: function () {
var _search = (0, _asyncToGenerator2.default)( /*#__PURE__*/_regenerator.default.mark(function _callee11(query, options) {
var _pouchLink$doctypes;
var pouchLink, optionsDoctypes, sharedDrivesDoctypes, markeNameIndex, allResults, dedupResults, enrichedResults, filteredByAttributes, sortedResults, results, normResults, _iterator4, _step4, res, normalizedRes, output;
return _regenerator.default.wrap(function _callee11$(_context11) {
while (1) {
switch (_context11.prev = _context11.next) {
case 0:
if (!(!this.searchIndexes || Object.keys(this.searchIndexes).length < 1)) {
_context11.next = 3;
break;
}
// The indexing might be running but not finished yet
log.warn('[SEARCH] No search index available');
return _context11.abrupt("return", null);
case 3:
pouchLink = (0, _client.getPouchLink)(this.client);
if (pouchLink !== null && pouchLink !== void 0 && (_pouchLink$doctypes = pouchLink.doctypes) !== null && _pouchLink$doctypes !== void 0 && _pouchLink$doctypes.length) {
// FIXME: doing this at search time is probably not the right place
// FIXME: we do not handle index cleanup for non-pouch search
this.cleanIndexes(pouchLink.doctypes);
}
optionsDoctypes = (options === null || options === void 0 ? void 0 : options.doctypes) || [];
if (optionsDoctypes.includes(_consts.FILES_DOCTYPE) || optionsDoctypes.length === 0) {
sharedDrivesDoctypes = Object.keys(this.searchIndexes).filter(function (doctype) {
return doctype.includes(_consts.SHARED_DRIVE_FILES_DOCTYPE);
});
optionsDoctypes.push.apply(optionsDoctypes, (0, _toConsumableArray2.default)(sharedDrivesDoctypes));
}
markeNameIndex = this.performanceApi.mark('search');
allResults = this.searchOnIndexes(query, optionsDoctypes);
dedupResults = this.deduplicateAndFlatten(allResults);
_context11.next = 12;
return (0, _normalizeSearchResult.enrichResultsWithDocs)(this.client, dedupResults);
case 12:
enrichedResults = _context11.sent;
filteredByAttributes = options !== null && options !== void 0 && options.excludeFilters ? enrichedResults.filter(function (result) {
return Object.entries(options.excludeFilters).every(function (_ref3) {
var _ref4 = (0, _slicedToArray2.default)(_ref3, 2),
key = _ref4[0],
value = _ref4[1];
return result.doc[key] !== value;
});
}) : enrichedResults;
sortedResults = this.sortSearchResults(filteredByAttributes, optionsDoctypes);
results = this.limitSearchResults(sortedResults);
normResults = [];
_iterator4 = _createForOfIteratorHelper(results);
try {
for (_iterator4.s(); !(_step4 = _iterator4.n()).done;) {
res = _step4.value;
normalizedRes = (0, _normalizeSearchResult.normalizeSearchResult)(this.client, res, query);
normResults.push(normalizedRes);
}
} catch (err) {
_iterator4.e(err);
} finally {
_iterator4.f();
}
output = normResults.filter(function (res) {
return res.title;
});
this.performanceApi.measure({
markName: markeNameIndex,
category: 'Search'
});
return _context11.abrupt("return", output);
case 22:
case "end":
return _context11.stop();
}
}
}, _callee11, this);
}));
function search(_x8, _x9) {
return _search.apply(this, arguments);
}
return search;
}()
}, {
key: "searchOnIndexes",
value: function searchOnIndexes(query, searchOnDoctypes) {
var _this4 = this;
var searchResults = [];
var _loop = function _loop(key) {
var doctype = key; // XXX - Should not be necessary
var isSearchOnDoctypesDefined = searchOnDoctypes.length > 0;
if (searchOnDoctypes && isSearchOnDoctypesDefined && !searchOnDoctypes.includes(doctype)) {
// Search only on specified doctypes
return "continue";
}
var index = _this4.searchIndexes[doctype];
if (!index) {
log.warn('[SEARCH] No search index available for ', doctype);
return "continue";
} // XXX - The limit is specified twice because of a flexsearch inconstency
// that does not enforce the limit if only given in second argument, and
// does not return the correct type is only given in third options
//
// XXX - The given limit here is arbitrary because flexsearch enforce it on matching
// field, which can cause issue related to the sort: if we search on name+path for files,
// and limit on 100, the 101th result on name will be skipped, but might appear on path,
// which will make it appear in the search results, but in incorrect order.
//
// Search result example:
// [
// {
// "field": "displayName",
// "result": [
// "604627c6bafee013ec5f27f7f72029f6"
// ]
// },
// {
// "field": "fullname",
// "result": [
// "604627c6bafee013ec5f27f7f72029f6", "604627c6bafee013ec5f27f3f714568"
// ]
// }
// ]
var FLEXSEARCH_LIMIT = 10000;
var indexResults = index.index.search(query, FLEXSEARCH_LIMIT, {
limit: FLEXSEARCH_LIMIT,
enrich: false
});
var newResults = indexResults.map(function (res) {
return _objectSpread(_objectSpread({}, res), {}, {
doctype: doctype
});
});
searchResults = searchResults.concat(newResults);
};
for (var key in this.searchIndexes) {
var _ret = _loop(key);
if (_ret === "continue") continue;
}
return searchResults;
}
}, {
key: "deduplicateAndFlatten",
value: function deduplicateAndFlatten(searchResults) {
var combinedResults = searchResults.flatMap(function (item) {
return item.result.map(function (id) {
return {
id: id.toString(),
// Because of flexsearch Id typing
doctype: item.doctype,
field: item.field
};
});
});
var resultMap = new Map();
combinedResults.forEach(function (_ref5) {
var id = _ref5.id,
field = _ref5.field,
doctype = _ref5.doctype;
if (resultMap.has(id)) {
var _resultMap$get;
(_resultMap$get = resultMap.get(id)) === null || _resultMap$get === void 0 ? void 0 : _resultMap$get.fields.push(field);
} else {
resultMap.set(id, {
id: id,
fields: [field],
doctype: doctype
});
}
});
return (0, _toConsumableArray2.default)(resultMap.values());
}
}, {
key: "compareStrings",
value: function compareStrings(str1, str2) {
if (!str1 && !str2) {
return 0;
} else if (!str1) {
return 1;
} else if (!str2) {
return -1;
}
return str1.localeCompare(str2, undefined, {
numeric: true
});
}
}, {
key: "sortSearchResults",
value: function sortSearchResults(searchResults, doctypesOrder) {
var _this5 = this;
return searchResults.sort(function (a, b) {
var doctypeComparison;
if (doctypesOrder && doctypesOrder.length > 0) {
doctypeComparison = doctypesOrder.findIndex(function (dt) {
return dt === a.doctype;
}) - doctypesOrder.findIndex(function (dt) {
return dt === b.doctype;
});
} else {
doctypeComparison = _consts.DOCTYPE_DEFAULT_ORDER[a.doctype] - _consts.DOCTYPE_DEFAULT_ORDER[b.doctype];
}
if (doctypeComparison !== 0) return doctypeComparison;
if (a.doctype === _consts.APPS_DOCTYPE && (0, _types.isIOCozyApp)(a.doc) && (0, _types.isIOCozyApp)(b.doc)) {
var _a$doc, _b$doc;
return _this5.compareStrings((_a$doc = a.doc) === null || _a$doc === void 0 ? void 0 : _a$doc.slug, (_b$doc = b.doc) === null || _b$doc === void 0 ? void 0 : _b$doc.slug);
} else if (a.doctype === _consts.CONTACTS_DOCTYPE && (0, _types.isIOCozyContact)(a.doc) && (0, _types.isIOCozyContact)(b.doc)) {
var _a$doc2, _b$doc2;
return _this5.compareStrings((_a$doc2 = a.doc) === null || _a$doc2 === void 0 ? void 0 : _a$doc2.displayName, (_b$doc2 = b.doc) === null || _b$doc2 === void 0 ? void 0 : _b$doc2.displayName);
} else if (a.doctype === _consts.FILES_DOCTYPE && (0, _types.isIOCozyFile)(a.doc) && (0, _types.isIOCozyFile)(b.doc)) {
return _this5.sortFiles(a, b);
}
return 0;
});
}
}, {
key: "sortFiles",
value: function sortFiles(aRes, bRes) {
var _aRes$doc, _bRes$doc;
if (!(0, _types.isIOCozyFile)(aRes.doc) || !(0, _types.isIOCozyFile)(bRes.doc)) {
return 0;
}
if (!aRes.fields.includes('name') || !bRes.fields.includes('name')) {
// First, sort docs with a match on the name field
return aRes.fields.includes('name') ? -1 : 1;
}
if (aRes.doc.type !== bRes.doc.type) {
// Then, directories
return aRes.doc.type === 'directory' ? -1 : 1;
} // Then name
return this.compareStrings((_aRes$doc = aRes.doc) === null || _aRes$doc === void 0 ? void 0 : _aRes$doc.name, (_bRes$doc = bRes.doc) === null || _bRes$doc === void 0 ? void 0 : _bRes$doc.name);
}
}, {
key: "limitSearchResults",
value: function limitSearchResults(searchResults) {
var doctypesCount = {};
return searchResults.filter(function (result) {
var doctype = result.doctype;
if (doctypesCount[doctype]) {
doctypesCount[doctype] += 1;
} else {
doctypesCount[doctype] = 1;
}
return doctypesCount[doctype] <= _consts.LIMIT_DOCTYPE_SEARCH;
});
}
}, {
key: "getSharedDrivesDoctypes",
value: function getSharedDrivesDoctypes() {
var pouchLink = (0, _client.getPouchLink)(this.client);
if (!pouchLink) {
return [];
}
return pouchLink.doctypes.filter(function (dtype) {
return dtype.includes(_consts.SHARED_DRIVE_FILES_DOCTYPE);
});
}
/**
* Clean up search indexes for doctypes that no longer exist
* @param currentDoctypes - List of currently valid doctypes
*/
}, {
key: "cleanIndexes",
value: function cleanIndexes(currentDoctypes) {
var existingDoctypes = Object.keys(this.searchIndexes);
var nonExistingDoctypes = existingDoctypes.filter(function (doctype) {
return !currentDoctypes.includes(doctype);
});
var _iterator5 = _createForOfIteratorHelper(nonExistingDoctypes),
_step5;
try {
for (_iterator5.s(); !(_step5 = _iterator5.n()).done;) {
var doctype = _step5.value;
delete this.searchIndexes[doctype];
log.debug('[SEARCH] Delete index for non-existing doctype', doctype);
}
} catch (err) {
_iterator5.e(err);
} finally {
_iterator5.f();
}
}
}]);
return SearchEngine;
}();
exports.SearchEngine = SearchEngine;