kit-fullsearch-electron
Version:
全文搜索组件，基于网易云信web端im sdk，用于Electron
967 lines (966 loc) • 56 kB
JavaScript
"use strict";
var __extends = (this && this.__extends) || (function () {
var extendStatics = function (d, b) {
extendStatics = Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
function (d, b) { for (var p in b) if (Object.prototype.hasOwnProperty.call(b, p)) d[p] = b[p]; };
return extendStatics(d, b);
};
return function (d, b) {
if (typeof b !== "function" && b !== null)
throw new TypeError("Class extends value " + String(b) + " is not a constructor or null");
extendStatics(d, b);
function __() { this.constructor = d; }
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
})();
var __assign = (this && this.__assign) || function () {
__assign = Object.assign || function(t) {
for (var s, i = 1, n = arguments.length; i < n; i++) {
s = arguments[i];
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
t[p] = s[p];
}
return t;
};
return __assign.apply(this, arguments);
};
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
return new (P || (P = Promise))(function (resolve, reject) {
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
step((generator = generator.apply(thisArg, _arguments || [])).next());
});
};
var __generator = (this && this.__generator) || function (thisArg, body) {
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g;
return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
function verb(n) { return function (v) { return step([n, v]); }; }
function step(op) {
if (f) throw new TypeError("Generator is already executing.");
while (_) try {
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
if (y = 0, t) op = [op[0] & 2, t.value];
switch (op[0]) {
case 0: case 1: t = op; break;
case 4: _.label++; return { value: op[1], done: false };
case 5: _.label++; y = op[1]; op = [0]; continue;
case 7: op = _.ops.pop(); _.trys.pop(); continue;
default:
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
if (t[2]) _.ops.pop();
_.trys.pop(); continue;
}
op = body.call(thisArg, _);
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
}
};
var __read = (this && this.__read) || function (o, n) {
var m = typeof Symbol === "function" && o[Symbol.iterator];
if (!m) return o;
var i = m.call(o), r, ar = [], e;
try {
while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
}
catch (error) { e = { error: error }; }
finally {
try {
if (r && !r.done && (m = i["return"])) m.call(i);
}
finally { if (e) throw e.error; }
}
return ar;
};
var __spreadArray = (this && this.__spreadArray) || function (to, from) {
for (var i = 0, il = from.length, j = to.length; i < il; i++, j++)
to[j] = from[i];
return to;
};
Object.defineProperty(exports, "__esModule", { value: true });
/* eslint-disable @typescript-eslint/explicit-function-return-type */
/* eslint-disable @typescript-eslint/no-this-alias */
var log4js_1 = require("log4js");
var utils_1 = require("./utils");
var type_1 = require("./type");
var path = require("path");
var os = require("os");
// import fs from 'fs'
var fs = require('fs');
var sqlite3 = require('sqlite3').verbose();
var tableColumn = [
'id',
'text',
'sessionId',
'from',
'time',
'target',
'to',
'type',
'scene',
'idServer',
'fromNick',
'content',
];
/**
* 全文搜索扩展函数
* @param NimSdk im sdk的类
*/
var fullText = function (NimSdk) {
return /** @class */ (function (_super) {
__extends(FullTextNim, _super);
function FullTextNim(initOpt) {
var _this = _super.call(this, initOpt) || this;
var account = initOpt.account, appKey = initOpt.appKey, queryOption = initOpt.queryOption, enablePinyin = initOpt.enablePinyin, searchDBName = initOpt.searchDBName, searchDBPath = initOpt.searchDBPath, debug = initOpt.debug;
if (!account || !appKey) {
throw new Error('invalid init params!');
}
_this.queryOption = queryOption || type_1.QueryOption.kDefault;
_this.enablePinyin = enablePinyin || false;
_this.searchDBName = searchDBName || account + "-" + appKey;
_this.searchDBPath = searchDBPath || '';
_this.msgQueue = [];
_this.msgStockQueue = [];
_this.timeout = 0;
// 初始化logger
log4js_1.configure({
appenders: {
ftsLog: {
type: 'file',
filename: path.join(_this.searchDBPath, _this.searchDBName + '.log')
},
console: {
type: 'console'
}
},
categories: {
default: {
appenders: debug ? ['ftsLog', 'console'] : ['ftsLog'],
level: "ALL"
}
}
});
_this.logger4j = log4js_1.getLogger();
return _this;
}
FullTextNim.prototype.initDB = function () {
return __awaiter(this, void 0, void 0, function () {
var finalName, that, _a;
return __generator(this, function (_b) {
switch (_b.label) {
case 0:
finalName = this.searchDBPath
? path.join(this.searchDBPath, this.searchDBName + ".sqlite")
: this.searchDBName + ".sqlite";
that = this;
_a = this;
return [4 /*yield*/, new Promise(function (resolve, reject) {
var db = new sqlite3.Database(finalName, function (err) {
if (err) {
that.logger4j.info('failed to open database: ', err);
reject(err);
return;
}
that.logger4j.info('open database successfully');
resolve(db);
});
})];
case 1:
_a.searchDB = _b.sent();
this.searchDB.run = utils_1.promisify(this.searchDB.run, this.searchDB);
this.searchDB.all = utils_1.promisify(this.searchDB.all, this.searchDB);
// this.searchDB.close = promisify(this.searchDB.close, this.searchDB)
return [4 /*yield*/, this.loadExtension()];
case 2:
// this.searchDB.close = promisify(this.searchDB.close, this.searchDB)
_b.sent();
return [4 /*yield*/, this.createTable()];
case 3:
_b.sent();
return [4 /*yield*/, this.loadDict()];
case 4:
_b.sent();
return [4 /*yield*/, this.backupDBFile()];
case 5:
_b.sent();
this.checkDbSafe();
return [2 /*return*/];
}
});
});
};
FullTextNim.prototype.loadExtension = function (filePath) {
return __awaiter(this, void 0, void 0, function () {
var type, arch, libName;
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
if (!filePath) {
type = os.type();
arch = os.arch();
libName = 'libsimple';
if (type === 'Darwin') {
arch = '';
libName = 'libsimple';
}
else {
libName = 'simple.dll';
}
filePath = path.resolve(path
.join(__dirname, arch, libName)
.replace(/^(.+)asar(.node_modules.+)$/, '$1asar.unpacked$2'));
}
return [4 /*yield*/, new Promise(function (resolve, reject) {
_this.logger4j.info("Load extension from file: " + filePath);
_this.searchDB.loadExtension(filePath, function (err) {
if (err) {
_this.logger4j.info("Load extension failed \uFF01from file: " + filePath);
reject(err);
return;
}
_this.logger4j.info("Load extension success \uFF01from file: " + filePath);
resolve({});
});
})];
case 1:
_a.sent();
return [2 /*return*/];
}
});
});
};
FullTextNim.prototype.loadDict = function () {
return __awaiter(this, void 0, void 0, function () {
var resourcePath, dictPath;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
this.logger4j.info('load dict start');
resourcePath = path.resolve(path
.join(__dirname)
.replace(/^(.+)asar(.node_modules.+)$/, '$1asar.unpacked$2'));
dictPath = path
.join(resourcePath, 'dict')
.concat(process.platform === 'win32' ? '\\' : '/');
return [4 /*yield*/, this.searchDB.run("SELECT jieba_dict(\"" + dictPath + "\")")];
case 1:
_a.sent();
this.logger4j.info('load dict success');
return [2 /*return*/];
}
});
});
};
FullTextNim.prototype.backupDBFile = function () {
return __awaiter(this, void 0, void 0, function () {
var srcFile, dstFile;
return __generator(this, function (_a) {
srcFile = path.join(this.searchDBPath, this.searchDBName + ".sqlite");
dstFile = path.join(this.searchDBPath, this.searchDBName + ".sqlite.backup");
try {
// remove exists backup DB file
if (fs.existsSync(dstFile)) {
fs.unlinkSync(dstFile);
}
fs.copyFileSync(srcFile, dstFile);
this.logger4j.info("backup DB file from " + srcFile + " to " + dstFile);
}
catch (err) {
this.logger4j.info("failed to backup DB file from " + srcFile + " to " + dstFile + ", error: " + err);
}
return [2 /*return*/];
});
});
};
FullTextNim.prototype.restoreDBFile = function () {
return __awaiter(this, void 0, void 0, function () {
var srcFile, dstFile;
var _this = this;
return __generator(this, function (_a) {
srcFile = path.join(this.searchDBPath, this.searchDBName + ".sqlite.backup");
dstFile = path.join(this.searchDBPath, this.searchDBName + ".sqlite");
return [2 /*return*/, new Promise(function (resolve, reject) {
_this.searchDB.close(function (err) {
if (err) {
reject(err);
}
else {
resolve({});
}
});
}).then(function () {
if (fs.existsSync(dstFile)) {
fs.unlinkSync(dstFile);
}
fs.copyFileSync(srcFile, dstFile);
// remove backup file
fs.unlinkSync(srcFile);
_this.logger4j.info("restore DB file from " + srcFile + " to " + dstFile);
}).catch(function () {
_this.logger4j.info("failed to restore DB file.");
})];
});
});
};
FullTextNim.prototype.checkDbSafe = function () {
return __awaiter(this, void 0, void 0, function () {
var err_1;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 2, , 3]);
return [4 /*yield*/, this.searchDB.run("INSERT INTO nim_msglog_fts(nim_msglog_fts) VALUES('integrity-check');")];
case 1:
_a.sent();
return [3 /*break*/, 3];
case 2:
err_1 = _a.sent();
this.emit('ftsDamaged', err_1);
this.logger4j.error(err_1);
throw err_1;
case 3: return [2 /*return*/];
}
});
});
};
FullTextNim.prototype.rebuildDbIndex = function () {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0: return [4 /*yield*/, this.searchDB.run("INSERT INTO nim_msglog_fts(nim_msglog_fts) VALUES('rebuild');")];
case 1: return [2 /*return*/, _a.sent()];
}
});
});
};
FullTextNim.prototype.formatSQLText = function (src) {
// sqlite 语法，语句中一个单引号转为两个单引号，还是当作一个转义好的单引号插入
return src.replace(/\'/gi, "''");
};
FullTextNim.prototype.createTable = function () {
return __awaiter(this, void 0, void 0, function () {
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
// simple 0 是为了禁止拼音
this.logger4j.info('create table start');
return [4 /*yield*/, this.searchDB.run("\n CREATE TABLE IF NOT EXISTS \"nim_msglog\" (\n \"id\" INTEGER PRIMARY KEY AUTOINCREMENT,\n \"idClient\" TEXT NOT NULL UNIQUE,\n \"text\" TEXT,\n \"sessionId\" TEXT NOT NULL,\n \"from\" TEXT NOT NULL,\n \"time\" INTEGER NOT NULL,\n \"target\" TEXT NOT NULL,\n \"to\" TEXT NOT NULL,\n \"type\" TEXT,\n \"scene\" TEXT,\n \"idServer\" TEXT NOT NULL,\n \"fromNick\" TEXT,\n \"content\" TEXT\n );")];
case 1:
_a.sent();
this.logger4j.info('create table nim_msglog success');
return [4 /*yield*/, this.searchDB.run("\n CREATE VIRTUAL TABLE IF NOT EXISTS nim_msglog_fts USING fts5(\n [idClient] UNINDEXED,\n [text],\n [sessionId] UNINDEXED,\n [from] UNINDEXED,\n [time] UNINDEXED,\n [target] UNINDEXED,\n [to] UNINDEXED,\n [type] UNINDEXED,\n [scene] UNINDEXED,\n [idServer] UNINDEXED,\n [fromNick] UNINDEXED,\n [content] UNINDEXED,\n content = [nim_msglog], content_rowid = id, tokenize = 'simple 0'\n );")];
case 2:
_a.sent();
this.logger4j.info('create table nim_msglog_fts success');
return [4 /*yield*/, this.searchDB.run("\n CREATE TRIGGER IF NOT EXISTS nim_msglog_ai AFTER INSERT ON nim_msglog \n BEGIN \n INSERT INTO nim_msglog_fts (\n rowid,idClient,text,sessionId,[from],time,target,[to],type,scene,idServer,fromNick,content\n ) VALUES (\n new.id,new.idClient,new.text,new.sessionId,new.[from],new.time,new.target,\n new.[to],new.type,new.scene,new.idServer,new.fromNick,new.content\n );\n END;")];
case 3:
_a.sent();
this.logger4j.info('create table nim_msglog_ai success');
return [4 /*yield*/, this.searchDB.run("\n CREATE TRIGGER IF NOT EXISTS nim_msglog_ad AFTER DELETE ON nim_msglog\n BEGIN\n INSERT INTO nim_msglog_fts (\n nim_msglog_fts,rowid,idClient,text,sessionId,[from],\n time,target,[to],type,scene,idServer,fromNick,content\n ) VALUES (\n 'delete',old.id,old.idClient,old.text,old.sessionId,old.[from],old.time,old.target,\n old.[to],old.type,old.scene,old.idServer,old.fromNick,old.content\n );\n END;")];
case 4:
_a.sent();
this.logger4j.info('create table nim_msglog_ad success');
return [4 /*yield*/, this.searchDB.run("\n CREATE TRIGGER IF NOT EXISTS nim_msglog_au AFTER UPDATE ON nim_msglog\n BEGIN\n INSERT INTO nim_msglog_fts (\n nim_msglog_fts,rowid,idClient,text,sessionId,[from],time,target,[to],type,scene,idServer,fromNick,content\n ) VALUES (\n 'delete',old.id,old.idClient,old.text,old.sessionId,old.[from],old.time,\n old.target,old.[to],old.type,old.scene,old.idServer,old.fromNick,old.content\n );\n INSERT INTO nim_msglog_fts (\n rowid,idClient,text,sessionId,[from],time,target,[to],type,scene,idServer,fromNick,content\n ) VALUES (\n new.id,new.idClient,new.text,new.sessionId,new.[from],\n new.time,new.target,new.[to],new.type,new.scene,new.idServer,new.fromNick,new.content\n );\n END;")];
case 5:
_a.sent();
this.logger4j.info('create table nim_msglog_au success');
this.logger4j.info('create tables successfully');
return [2 /*return*/];
}
});
});
};
FullTextNim.prototype.sendText = function (opt) {
var _this = this;
return _super.prototype.sendText.call(this, __assign(__assign({}, opt), { done: function (err, obj) {
if (!err && obj.idClient) {
_this.putFts(obj);
}
opt.done && opt.done(err, obj);
} }));
};
FullTextNim.prototype.sendCustomMsg = function (opt) {
var _this = this;
return _super.prototype.sendCustomMsg.call(this, __assign(__assign({}, opt), { done: function (err, obj) {
if (!err && obj.idClient) {
_this.putFts(obj);
}
opt.done && opt.done(err, obj);
} }));
};
FullTextNim.prototype.saveMsgsToLocal = function (opt) {
var _this = this;
return _super.prototype.saveMsgsToLocal.call(this, __assign(__assign({}, opt), { done: function (err, obj) {
if (!err) {
_this.putFts(obj);
}
opt.done && opt.done(err, obj);
} }));
};
FullTextNim.prototype.deleteMsg = function (opt) {
var _this = this;
return _super.prototype.deleteMsg.call(this, __assign(__assign({}, opt), { done: function (err, obj) {
if (!err && opt.msg && opt.msg.idClient) {
_this.deleteFts(opt.msg.idClient);
}
opt.done && opt.done(err, obj);
} }));
};
FullTextNim.prototype.deleteLocalMsg = function (opt) {
var _this = this;
return _super.prototype.deleteLocalMsg.call(this, __assign(__assign({}, opt), { done: function (err, obj) {
if (!err && opt.msg && opt.msg.idClient) {
_this.deleteFts(opt.msg.idClient);
}
opt.done && opt.done(err, obj);
} }));
};
FullTextNim.prototype.deleteLocalMsgs = function (opt) {
return __awaiter(this, void 0, void 0, function () {
var getLocalMsgs, deleteLocalMsgs, obj, result, idClients, err_2;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
getLocalMsgs = utils_1.promisifyForDone(_super.prototype.getLocalMsgs, this);
deleteLocalMsgs = utils_1.promisifyForDone(_super.prototype.deleteLocalMsgs, this);
_a.label = 1;
case 1:
_a.trys.push([1, 6, , 7]);
return [4 /*yield*/, getLocalMsgs(__assign(__assign({}, opt), { limit: Infinity }))];
case 2:
obj = _a.sent();
return [4 /*yield*/, deleteLocalMsgs(__assign({}, opt))];
case 3:
result = _a.sent();
if (!(obj.msgs && obj.msgs.length > 0)) return [3 /*break*/, 5];
idClients = obj.msgs.map(function (msg) { return msg.idClient; });
return [4 /*yield*/, this.deleteFts(idClients)];
case 4:
_a.sent();
_a.label = 5;
case 5:
opt.done && opt.done(null, result);
return [3 /*break*/, 7];
case 6:
err_2 = _a.sent();
opt.done && opt.done(err_2);
return [3 /*break*/, 7];
case 7: return [2 /*return*/];
}
});
});
};
FullTextNim.prototype.deleteLocalMsgsBySession = function (opt) {
return __awaiter(this, void 0, void 0, function () {
var getLocalMsgs, deleteLocalMsgsBySession, obj, result, idClients, err_3;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
getLocalMsgs = utils_1.promisifyForDone(_super.prototype.getLocalMsgs, this);
deleteLocalMsgsBySession = utils_1.promisifyForDone(_super.prototype.deleteLocalMsgsBySession, this);
_a.label = 1;
case 1:
_a.trys.push([1, 6, , 7]);
return [4 /*yield*/, getLocalMsgs({
sessionId: opt.scene + "-" + opt.to,
limit: Infinity,
})];
case 2:
obj = _a.sent();
return [4 /*yield*/, deleteLocalMsgsBySession(__assign({}, opt))];
case 3:
result = _a.sent();
if (!(obj.msgs && obj.msgs.length > 0)) return [3 /*break*/, 5];
idClients = obj.msgs.map(function (msg) { return msg.idClient; });
return [4 /*yield*/, this.deleteFts(idClients)];
case 4:
_a.sent();
_a.label = 5;
case 5:
opt.done && opt.done(null, result);
return [3 /*break*/, 7];
case 6:
err_3 = _a.sent();
opt.done && opt.done(err_3);
return [3 /*break*/, 7];
case 7: return [2 /*return*/];
}
});
});
};
FullTextNim.prototype.deleteAllLocalMsgs = function (opt) {
var _this = this;
return _super.prototype.deleteAllLocalMsgs.call(this, __assign(__assign({}, opt), { done: function (err, obj) {
if (!err) {
_this.clearAllFts();
}
opt.done && opt.done(err, obj);
} }));
};
FullTextNim.prototype.deleteMsgSelf = function (opt) {
var _this = this;
return _super.prototype.deleteMsgSelf.call(this, __assign(__assign({}, opt), { done: function (err, obj) {
if (!err && opt.msg && opt.msg.idClient) {
_this.deleteFts(opt.msg.idClient);
}
opt.done && opt.done(err, obj);
} }));
};
FullTextNim.prototype.deleteMsgSelfBatch = function (opt) {
var _this = this;
return _super.prototype.deleteMsgSelfBatch.call(this, __assign(__assign({}, opt), { done: function (err, obj) {
if (!err && opt.msgs && opt.msgs.length) {
var ids = opt.msgs.map(function (item) { return item.idClient; });
_this.deleteFts(ids);
}
opt.done && opt.done(err, obj);
} }));
};
FullTextNim.prototype.getLocalMsgsToFts = function (opt) {
return __awaiter(this, void 0, void 0, function () {
var msgs, err_4, length, lastTime;
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 2, , 3]);
return [4 /*yield*/, new Promise(function (resolve, reject) {
_super.prototype.getLocalMsgs.call(_this, __assign(__assign({}, opt), { done: function (err, obj) {
if (err) {
reject(err);
return;
}
var msgs = obj.msgs && obj.msgs.filter(function (item) { return item.text && item.idClient; });
resolve(msgs);
} }));
})];
case 1:
msgs = _a.sent();
return [3 /*break*/, 3];
case 2:
err_4 = _a.sent();
opt.done && opt.done(err_4, null);
return [3 /*break*/, 3];
case 3:
length = msgs.length;
lastTime = length > 0 ? msgs[length - 1].time : undefined;
if (msgs && msgs.length > 0) {
this.putFts(msgs || [], true);
}
// 解除引用
msgs = null;
opt.done && opt.done(null, {
length: length,
lastTime: lastTime
});
return [2 /*return*/];
}
});
});
};
FullTextNim.prototype.queryFts = function (params) {
return __awaiter(this, void 0, void 0, function () {
var reg, sql, records, error_1;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 2, , 3]);
// 入参过滤，去除多余的符号，text 替换 ' 字符
if (params.text) {
reg = /[^\u4e00-\u9fa5^a-z^A-Z^0-9]/g;
params.text = params.text.replace(reg, ' ').trim();
}
if (!params.sessionIds) {
params.sessionIds = [];
}
if (!params.froms) {
params.froms = [];
}
// 如果 text 过滤后为空，并且此时不存在 froms，sessionIds，那么直接返回空把。
if (!(params.text || params.froms.length > 0 ||
params.sessionIds.length > 0 || params.start || params.end)) {
return [2 /*return*/, []];
}
sql = this._handleQueryParams(params);
return [4 /*yield*/, this.searchDB.all(sql)];
case 1:
records = _a.sent();
return [2 /*return*/, records];
case 2:
error_1 = _a.sent();
this.logger4j.info('queryFts fail: ', error_1);
throw error_1;
case 3: return [2 /*return*/];
}
});
});
};
FullTextNim.prototype.putFts = function (msgs, isStock) {
if (isStock === void 0) { isStock = false; }
if (!Array.isArray(msgs)) {
msgs = [msgs];
}
if (isStock) {
this.msgStockQueue = this.msgStockQueue.concat(msgs);
}
else {
this.msgQueue = this.msgQueue.concat(msgs);
}
// @ts-ignore
msgs = null;
// 设置定时器，开始同步
if (!this.timeout) {
this.timeout = setTimeout(this._putFts.bind(this, isStock), 0);
}
};
FullTextNim.prototype._putFts = function (isStock) {
if (isStock === void 0) { isStock = false; }
return __awaiter(this, void 0, void 0, function () {
var msgs, fts, ftsLength, ftsLastTime;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
// 当 msgQueue 为 null 或者 undefined 时，当作实例已经销毁，此定时触发的任务直接作废
if (!this.msgQueue) {
return [2 /*return*/];
}
msgs = isStock
? this.msgStockQueue.splice(0, 3000)
: this.msgQueue.splice(0, 3000);
fts = msgs;
// 解除引用
msgs = null;
ftsLength = fts.length;
ftsLastTime = fts[fts.length - 1].time;
if (!(fts.length > 0)) return [3 /*break*/, 2];
return [4 /*yield*/, this._doInsert(fts)
// 解除引用
];
case 1:
_a.sent();
// 解除引用
fts = null;
// 当 msgQueue 为 null 或者 undefined 时，当作实例已经销毁，此定时触发的任务直接作废
if (!this.msgQueue) {
return [2 /*return*/];
}
isStock
? this.emit('ftsStockUpsert', ftsLength, this.msgQueue.length, this.msgStockQueue.length, ftsLastTime)
: this.emit('ftsUpsert', ftsLength, this.msgQueue.length);
_a.label = 2;
case 2:
if (this.msgStockQueue.length === 0 && this.msgQueue.length === 0) {
// clearTimeout(this.timeout)
this.timeout = 0;
return [2 /*return*/];
}
// 队列里还存在未同步的，那么继续定时执行
this.timeout = this.msgStockQueue.length > 0 ?
setTimeout(this._putFts.bind(this, true), 0) :
setTimeout(this._putFts.bind(this), 0);
return [2 /*return*/];
}
});
});
};
// async _getMsgsWithInsertAndUpdate(msgs: IMsg[]): Promise<IMsg[]> {
// // 去重
// // const map = msgs.reduce((total, next) => {
// // if (next.idClient) {
// // total[next.idClient] = next
// // }
// // return total
// // }, {})
// // msgs = Object.keys(map).map((key) => map[key])
// // const fts = msgs
// // .filter((msg) => msg.text && msg.idClient)
// // .map((msg) => {
// // return {
// // _id: msg.idClient,
// // text: msg.text,
// // sessionId: msg.sessionId,
// // from: msg.from,
// // time: msg.time,
// // target: msg.target,
// // to: msg.to,
// // type: msg.type,
// // scene: msg.scene,
// // idServer: msg.idServer,
// // fromNick: msg.fromNick,
// // content: msg.content,
// // }
// // })
// let map: any = {}
// const fts: IMsg[] = [];
// msgs.forEach((msg: any) => {
// // text 内容存在，idClient 存在，在去重的 map 中找不到已经存在过的
// if (msg && msg.text && msg.idClient && !map[msg.idClient]) {
// fts.push({
// _id: msg.idClient,
// text: msg.text,
// sessionId: msg.sessionId,
// from: msg.from,
// time: msg.time,
// target: msg.target,
// to: msg.to,
// type: msg.type,
// scene: msg.scene,
// idServer: msg.idServer,
// fromNick: msg.fromNick,
// content: msg.content,
// })
// map[msg.idClient] = true
// Object.keys(msg).forEach((item: any) => { item = null })
// msg = null
// }
// })
// // 解除引用.
// map = null
// return fts
// }
FullTextNim.prototype._doInsert = function (msgs) {
return __awaiter(this, void 0, void 0, function () {
var that;
var _this = this;
return __generator(this, function (_a) {
that = this;
this.logger4j.info("insert data to database, length: " + msgs.length + ", timetag from " + msgs[0].time + " to " + msgs[msgs.length - 1].time + ", " + JSON.stringify(msgs[0]));
return [2 /*return*/, new Promise(function (resolve, reject) {
var column = tableColumn.map(function () { return '?'; }).join(',');
_this.searchDB.serialize(function () {
try {
_this.searchDB.exec('BEGIN TRANSACTION;');
msgs.forEach(function (msg) {
_this.searchDB.run("INSERT OR IGNORE INTO `nim_msglog` VALUES(NULL," + column + ");", [
msg.idClient,
msg.text,
msg.sessionId,
msg.from,
msg.time,
msg.target,
msg.to,
msg.type,
msg.scene,
msg.idServer,
msg.fromNick,
msg.content
])
.catch(function (err) { that.emit('ftsError', err); });
Object.keys(msg).forEach(function (item) { item = null; });
msg = null;
});
_this.searchDB.exec('COMMIT;', function (err) {
if (err) {
that.emit('ftsError', err);
reject(err);
return;
}
resolve();
});
}
catch (err) {
_this.searchDB.exec('ROLLBACK TRANSACTION;', function (err) {
that.logger4j.info('rollback: ', err);
});
reject(err);
}
});
})];
});
});
};
FullTextNim.prototype.deleteFts = function (ids) {
return __awaiter(this, void 0, void 0, function () {
var idsString, error_2;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
idsString = '';
if (Array.isArray(ids)) {
idsString = ids.map(function (id) { return "\"" + id + "\""; }).join(',');
}
else {
idsString = "\"" + ids + "\"";
}
_a.label = 1;
case 1:
_a.trys.push([1, 3, , 4]);
this.logger4j.info("delete data from database, ids: " + idsString);
return [4 /*yield*/, this.searchDB.run("DELETE FROM nim_msglog WHERE idClient in (" + idsString + ");")];
case 2:
_a.sent();
this.logger4j.info('deleteFts success', ids);
return [3 /*break*/, 4];
case 3:
error_2 = _a.sent();
this.logger4j.info('deleteFts fail: ', error_2);
throw error_2;
case 4: return [2 /*return*/];
}
});
});
};
FullTextNim.prototype.clearAllFts = function () {
return __awaiter(this, void 0, void 0, function () {
var error_3;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 2, , 3]);
return [4 /*yield*/, this.searchDB.run('DELETE FROM `nim_msglog`;')];
case 1:
_a.sent();
return [3 /*break*/, 3];
case 2:
error_3 = _a.sent();
this.logger4j.info('clearAllFts fail: ', error_3);
throw error_3;
case 3: return [2 /*return*/];
}
});
});
};
FullTextNim.prototype.dropAllFts = function () {
return __awaiter(this, void 0, void 0, function () {
var error_4;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
_a.trys.push([0, 7, , 8]);
return [4 /*yield*/, this.searchDB.run('drop table if exists nim_msglog;')];
case 1:
_a.sent();
return [4 /*yield*/, this.searchDB.run('drop table if exists nim_msglog_fts;')];
case 2:
_a.sent();
return [4 /*yield*/, this.searchDB.run('drop trigger if exists nim_msglog_au;')];
case 3:
_a.sent();
return [4 /*yield*/, this.searchDB.run('drop trigger if exists nim_msglog_ai;')];
case 4:
_a.sent();
return [4 /*yield*/, this.searchDB.run('drop trigger if exists nim_msglog_ad;')];
case 5:
_a.sent();
return [4 /*yield*/, this.createTable()];
case 6:
_a.sent();
this.logger4j.info('dropAllFts success');
return [3 /*break*/, 8];
case 7:
error_4 = _a.sent();
this.logger4j.info('dropAllFts fail: ', error_4);
throw error_4;
case 8: return [2 /*return*/];
}
});
});
};
FullTextNim.prototype.destroy = function (options) {
var _this = this;
// 清空队列和定时器，关闭 db。
this.timeout && clearTimeout(this.timeout);
this.msgStockQueue = [];
this.msgQueue = [];
new Promise(function (resolve, reject) {
_this.searchDB.close(function (err) {
if (err) {
reject(err);
return;
}
resolve({});
});
})
.then(function () {
_this.logger4j.info && _this.logger4j.info('close searchDB success');
FullTextNim.instance = null;
_super.prototype.destroy.call(_this, options);
})
.catch(function (error) {
_this.logger4j.info && _this.logger4j.info('close searchDB fail: ', error);
FullTextNim.instance = null;
_super.prototype.destroy.call(_this, options);
});
};
// 处理QUERY参数
FullTextNim.prototype._handleQueryParams = function (_a) {
var text = _a.text, sessionIds = _a.sessionIds, froms = _a.froms, timeDirection = _a.timeDirection, _b = _a.limit, limit = _b === void 0 ? 100 : _b, _c = _a.offset, offset = _c === void 0 ? 0 : _c, start = _a.start, end = _a.end, _d = _a.queryOption, queryOption = _d === void 0 ? this.queryOption : _d;
var where = [];
if (text) {
var matchRegex = new RegExp(/^[0-9a-zA-Z]+$/);
var queryText = this.formatSQLText(text);
if (matchRegex.test(text)) {
where.push("`text` GLOB '*" + text + "*'");
}
else {
where.push("`text` MATCH query('" + queryText + "', " + queryOption + ", " + this.enablePinyin + ")");
}
}
if (sessionIds && sessionIds.length > 0) {
var temp = sessionIds.map(function (id) { return "'" + id + "'"; }).join(',');
where.push("`sessionId` IN (" + temp + ")");
}
if (froms && froms.length > 0) {
var temp = froms.map(function (from) { return "'" + from + "'"; }).join(',');
where.push("`from` IN (" + temp + ")");
}
if (start) {
where.push("`time` >= " + start);
}
if (end) {
where.push("`time` < " + end);
}
var order = '';
if (timeDirection === 'ascend') {
order = "ORDER BY time ASC";
}
else if (timeDirection === 'descend') {
order = "ORDER BY time DESC";
}
var limitQuery = '';
if (limit !== Infinity) {
limitQuery = "LIMIT " + limit + " OFFSET " + offset;
}
var column = tableColumn
.slice(1)
.map(function (item) { return '`' + item + '`'; })
.join(', ');
var whereSQL = where.length > 0 ? "where " + where.join(' AND ') : '';
var sql = "SELECT `idClient`, " + column + " from nim_msglog_fts " + whereSQL + " " + order + " " + limitQuery;
this.logger4j.info('_handleQueryParams: ', sql);
return sql;
};
FullTextNim.getInstance = function (initOpt) {
return __awaiter(this, void 0, void 0, function () {
var error, err_5, err_6;
var _this = this;
return __generator(this, function (_a) {
switch (_a.label) {
case 0:
if (!!this.instance) return [3 /*break*/, 9];
this.instance = new FullTextNim(initOpt);
error = null;
_a.label = 1;
case 1:
_a.trys.push([1, 3, , 4]);
return [4 /*yield*/, this.instance.initDB()];
case 2:
_a.sent();
console.log("Init DB successfully");
return [3 /*break*/, 4];
case 3:
err_5 = _a.sent();
console.error("Failed to initialize database, error: " + (err_5 && err_5.message));
error = err_5;
return [3 /*break*/, 4];
case 4:
if (!error) return [3 /*break*/, 9];
_a.label = 5;
case 5:
_a.trys.push([5, 8, , 9]);