mongoosastic-ts
Version:
A mongoose plugin that indexes models into elastic search
597 lines • 20.8 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.mongoosastic = void 0;
const mapping_generator_1 = require("./mapping-generator");
const serialize_1 = require("./serialize");
const elasticsearch_1 = require("elasticsearch");
const events_1 = __importDefault(require("events"));
function isString(subject) {
return typeof subject === 'string';
}
function isStringArray(arr) {
return arr.filter && arr.length === arr.filter((item) => typeof item === 'string').length;
}
function createEsClient(options) {
const esOptions = {};
const { host = 'localhost', port = 9200, protocol = 'http', auth = null, keepAlive = false, hosts, log = null, } = options;
if (Array.isArray(hosts)) {
esOptions.host = hosts;
}
else {
esOptions.host = {
host,
port,
protocol,
auth,
keepAlive,
};
}
esOptions.log = log;
return new elasticsearch_1.Client(esOptions);
}
function filterMappingFromMixed(props) {
const filteredMapping = {};
Object.keys(props).map((key) => {
const field = props[key];
if (field.type !== 'mixed') {
filteredMapping[key] = field;
if (field.properties) {
filteredMapping[key].properties = filterMappingFromMixed(field.properties);
if (!Object.keys(filteredMapping[key].properties).length) {
delete filteredMapping[key].properties;
}
}
}
});
return filteredMapping;
}
async function createMappingIfNotPresent(options) {
const { client, indexName, typeName, schema, properties, mappings, settings } = options;
const completeMapping = {};
if (!mappings) {
completeMapping[typeName] = mapping_generator_1.Generator.generateMapping(schema);
completeMapping[typeName].properties = filterMappingFromMixed(completeMapping[typeName].properties);
if (properties) {
Object.keys(properties).map((key) => {
completeMapping[typeName].properties[key] = properties[key];
});
}
}
else {
completeMapping[typeName] = mappings;
}
const inputMapping = completeMapping[typeName];
const exists = await client.indices.exists({
index: indexName,
});
if (exists) {
return await client.indices.putMapping({
type: undefined,
index: indexName,
body: inputMapping,
});
}
return await client.indices.create({
index: indexName,
body: { settings, mappings: inputMapping },
});
}
async function hydrate(res, model, options) {
const results = res.hits;
const resultsMap = {};
const ids = results.hits.map((result, idx) => {
resultsMap[result._id] = idx;
return result._id;
});
const query = model.find({
_id: {
$in: ids,
},
});
const hydrateOptions = options.hydrateOptions;
// Build Mongoose query based on hydrate options
// Example: {lean: true, sort: '-name', select: 'address name'}
Object.keys(hydrateOptions).forEach((option) => {
query[option](hydrateOptions[option]);
});
const docs = await query.exec();
let hits;
const docsMap = {};
if (!docs || docs.length === 0) {
results.hits = [];
res.hits = results;
return res;
}
if (hydrateOptions.sort) {
// Hydrate sort has precedence over ES result order
hits = docs;
}
else {
// Preserve ES result ordering
docs.forEach((doc) => {
docsMap[doc._id] = doc;
});
hits = results.hits.map((result) => docsMap[result._id]);
}
if (options.highlight || options.hydrateWithESResults) {
hits.forEach((doc) => {
const idx = resultsMap[doc._id];
if (options.highlight) {
doc._highlight = results.hits[idx].highlight;
}
if (options.hydrateWithESResults) {
// Add to doc ES raw result (with, e.g., _score value)
doc._esResult = results.hits[idx];
if (!options.hydrateWithESResults.source) {
// Remove heavy load
delete doc._esResult._source;
}
}
});
}
results.hits = hits;
res.hits = results;
return res;
}
async function deleteByMongoId(options) {
const { index, client, model, tries = 0, routing } = options;
return client
.delete({
maxRetries: tries,
type: undefined,
index: index,
id: model._id.toString(),
routing: routing,
})
.then((res) => {
if (res.result === 'deleted') {
model.emit('es-removed', undefined, res);
}
else {
model.emit('es-removed', res, res);
}
})
.catch((e) => {
model.emit('es-removed', e, undefined);
});
}
function mongoosastic(schema, pluginOpts) {
var _a;
const options = pluginOpts || {};
let bulkTimeout;
let bulkBuffer = [];
const { populate, hydrate: alwaysHydrate, hydrateOptions: defaultHydrateOptions, filter, transform, routing, customProperties, customSerialize, forceIndexRefresh, } = options;
const mapping = mapping_generator_1.Generator.generateMapping(schema);
const indexAutomatically = !(options && options.indexAutomatically === false);
const saveOnSynchronize = !(options && options.saveOnSynchronize === false);
const bulkErrEm = new events_1.default.EventEmitter();
const esClient = (_a = options.esClient) !== null && _a !== void 0 ? _a : createEsClient(options);
let { index: indexName, type: typeName, bulk } = options;
const { esVersion = '8' } = options;
function setIndexNameIfUnset(model) {
const modelName = model.toLowerCase();
if (!indexName) {
indexName = `${modelName}s`;
}
if (!typeName) {
typeName = modelName;
}
}
async function postSave(doc) {
let _doc;
function onIndex(err, res) {
if (!filter || !filter(doc)) {
doc.emit('es-indexed', err, res);
}
else {
doc.emit('es-filtered', err, res);
}
}
if (doc) {
// todo check populate and fix constructor typing
// @ts-expect-error ts-migrate(2351) FIXME: This expression is not constructable.
_doc = new doc.constructor(doc);
if (populate && populate.length) {
const popDoc = await _doc.populate(populate);
popDoc
.index()
.then((res) => {
onIndex(undefined, res);
})
.catch((e) => {
onIndex(e, undefined);
});
}
else {
return _doc
.index()
.then((res) => {
onIndex(undefined, res);
})
.catch((e) => {
onIndex(e, undefined);
});
}
}
}
function clearBulkTimeout() {
clearTimeout(bulkTimeout);
bulkTimeout = undefined;
}
async function bulkAdd(instruction) {
var _a;
bulkBuffer.push(instruction);
// Return because we need the doc being indexed
// Before we start inserting
if ((_a = instruction === null || instruction === void 0 ? void 0 : instruction.index) === null || _a === void 0 ? void 0 : _a._index) {
return;
}
if (bulkBuffer.length >= ((bulk === null || bulk === void 0 ? void 0 : bulk.size) || 1000)) {
await schema.statics.flush();
clearBulkTimeout();
}
else if (bulkTimeout === undefined) {
bulkTimeout = setTimeout(async () => {
await schema.statics.flush();
clearBulkTimeout();
}, (bulk === null || bulk === void 0 ? void 0 : bulk.delay) || 1000);
}
}
async function bulkDelete(opts) {
return await bulkAdd({
delete: {
_index: opts.index || indexName,
_id: opts.model._id.toString(),
routing: opts.routing,
},
});
}
async function bulkIndex(opts) {
await bulkAdd({
index: {
_index: opts.index || indexName,
_id: opts._id.toString(),
routing: opts.routing,
},
});
await bulkAdd(opts.model);
}
/**
* ElasticSearch Client
*/
schema.statics.esClient = esClient;
/**
* Create the mapping. Takes an optional settings parameter
* and a callback that will be called once the mapping is created
*/
schema.statics.createMapping = async function createMapping(inSettings, inMappings) {
setIndexNameIfUnset(this.modelName);
return await createMappingIfNotPresent({
client: esClient,
indexName: indexName,
typeName: typeName,
schema: schema,
settings: inSettings,
mappings: inMappings,
properties: customProperties,
});
};
/**
* Get the mapping.
*/
schema.statics.getMapping = function getMapping() {
return mapping_generator_1.Generator.generateMapping(schema);
};
/**
* Get clean tree.
*/
schema.statics.getCleanTree = function getCleanTree() {
return mapping_generator_1.Generator.getCleanTree(schema);
};
schema.methods.index = async function schemaIndex(inOpts = {}) {
let serialModel;
const opts = inOpts;
if (filter && filter(this)) {
return this.unIndex();
}
setIndexNameIfUnset(this.constructor['modelName']);
const index = opts.index || indexName;
/**
* Serialize the model, and apply transformation
*/
if (typeof customSerialize === 'function') {
serialModel = customSerialize(this, mapping);
}
else {
serialModel = (0, serialize_1.serialize)(this.toObject(), mapping);
}
if (transform)
serialModel = transform(serialModel, this);
const _opts = {
model: undefined,
index: index,
refresh: forceIndexRefresh,
};
if (routing) {
_opts.routing = routing(this);
}
if (bulk) {
_opts.model = serialModel;
_opts._id = this._id;
await bulkIndex(_opts);
return this;
}
else {
_opts.id = this._id.toString();
_opts.body = serialModel;
// indexing log in-case of slow queries in elasticsearch
return esClient.index(_opts);
}
};
/**
* Unset elasticsearch index
*/
schema.methods.unIndex = async function unIndex(inOpts = {}) {
setIndexNameIfUnset(this.constructor['modelName']);
inOpts.index = inOpts.index || indexName;
inOpts.type = inOpts.type || typeName;
inOpts.model = this;
inOpts.client = esClient;
inOpts.tries = inOpts.tries || 3;
if (routing) {
inOpts.routing = routing(this);
}
if (bulk) {
return await bulkDelete(inOpts);
}
else {
return await deleteByMongoId(inOpts);
}
};
/**
* Delete all documents from a type/index
*/
schema.statics.esTruncate = async function esTruncate(inOpts = {}) {
var _a, _b, _c, _d;
setIndexNameIfUnset(this.modelName);
// todo fix pagination and only get ids
// or recreate index better?
inOpts.index = inOpts.index || indexName;
const settingsRes = await esClient.indices.getSettings(inOpts);
const indexSettings = (settingsRes === null || settingsRes === void 0 ? void 0 : settingsRes[indexName].settings) || {};
(_a = indexSettings === null || indexSettings === void 0 ? void 0 : indexSettings.index) === null || _a === void 0 ? true : delete _a.creation_date;
(_b = indexSettings === null || indexSettings === void 0 ? void 0 : indexSettings.index) === null || _b === void 0 ? true : delete _b.provided_name;
(_c = indexSettings === null || indexSettings === void 0 ? void 0 : indexSettings.index) === null || _c === void 0 ? true : delete _c.uuid;
(_d = indexSettings === null || indexSettings === void 0 ? void 0 : indexSettings.index) === null || _d === void 0 ? true : delete _d.version;
// pass this to override the mapping from default // todo
// const mappingsRes = await esClient.indices.getMapping(opts);
// const indexMappings = mappingsRes?.[indexName].mappings || {};
try {
await esClient.indices.delete(inOpts);
}
catch (e) { }
return await this.createMapping(indexSettings);
};
/**
* Synchronize an existing collection
*/
schema.statics.synchronize = function synchronize(inQuery, inOpts) {
const em = new events_1.default.EventEmitter();
let closeValues = [];
let counter = 0;
const query = inQuery || {};
const close = function close() {
em.emit.apply(em, ['close'].concat(closeValues));
};
const _saveOnSynchronize = inOpts && inOpts.saveOnSynchronize !== undefined ? inOpts.saveOnSynchronize : saveOnSynchronize;
// Set indexing to be bulk when synchronizing to make synchronizing faster
// Set default values when not present
bulk = {
delay: (bulk && bulk.delay) || 1000,
size: (bulk && bulk.size) || 1000,
batch: (bulk && bulk.batch) || 50,
};
setIndexNameIfUnset(this.modelName);
const stream = this.find(query).batchSize(bulk.batch).cursor();
stream.on('data', (doc) => {
stream.pause();
counter++;
function onIndex(indexErr, inDoc) {
counter--;
if (indexErr) {
em.emit('error', indexErr);
}
else {
em.emit('data', undefined, inDoc);
}
stream.resume();
}
doc.on('es-indexed', onIndex);
doc.on('es-filtered', onIndex);
if (_saveOnSynchronize) {
// Save document with Mongoose first
doc.save((err) => {
if (err) {
counter--;
em.emit('error', err);
return stream.resume();
}
});
}
else {
postSave(doc).then();
}
});
stream.on('close', (pA, pB) => {
closeValues = [pA, pB];
const closeInterval = setInterval(() => {
if (counter === 0 && bulkBuffer.length === 0) {
clearInterval(closeInterval);
close();
bulk = options && options.bulk;
}
}, 1000);
});
stream.on('error', (err) => {
em.emit('error', err);
});
return em;
};
/**
* ElasticSearch search function
* Wrapping schema.statics.es_search().
*/
schema.statics.search = async function search(inQuery, inOpts) {
const opts = inOpts;
const query = inQuery === null ? undefined : inQuery;
const fullQuery = {
query: query,
};
const esSearch = schema.statics.esSearch.bind(this);
return esSearch(fullQuery, opts);
};
/**
* ElasticSearch true/raw search function
*
* Elastic search query: provide full query object.
* Useful, e.g., for paged requests.
*
* @param inQuery - **full** query object to perform search with
* @param inOpts - (optional) special search options, such as hydrate
*/
schema.statics.esSearch = async function (inQuery, inOpts) {
const opts = inOpts !== null && inOpts !== void 0 ? inOpts : {};
const query = inQuery === null ? undefined : inQuery;
opts.hydrateOptions = (opts === null || opts === void 0 ? void 0 : opts.hydrateOptions) || defaultHydrateOptions || {};
setIndexNameIfUnset(this.modelName);
const esQuery = {
body: query,
index: opts.index || indexName,
};
if (opts.routing) {
esQuery.routing = opts.routing;
}
if (opts.highlight) {
esQuery.body.highlight = opts.highlight;
}
if (opts.suggest) {
esQuery.body.suggest = opts.suggest;
}
if (opts.aggs) {
esQuery.body.aggs = opts.aggs;
}
if (opts.min_score) {
esQuery.body.min_score = opts.min_score;
}
Object.keys(opts).forEach((opt) => {
if (!opt.match(/(hydrate|sort|aggs|highlight|suggest)/) && opts.hasOwnProperty(opt)) {
esQuery[opt] = opts[opt];
}
if (opts.sort) {
if (isString(opts.sort) || isStringArray(opts.sort)) {
esQuery.sort = opts.sort;
}
else {
esQuery.body.sort = opts.sort;
}
}
});
// search query for elasticsearch
const res = await esClient.search(esQuery);
const resp = reformatESTotalNumber(res);
if (alwaysHydrate || opts.hydrate) {
return hydrate(resp, this, opts);
}
return resp;
};
function reformatESTotalNumber(res) {
Object.assign(res.hits, {
total: res.hits.total.value,
extTotal: res.hits.total,
});
return res;
}
schema.statics.esCount = async function esCount(query) {
setIndexNameIfUnset(this.modelName);
const esQuery = {
body: {
query: query !== null && query !== void 0 ? query : { match_all: {} },
},
index: indexName,
};
return await esClient.count(esQuery);
};
schema.statics.flush = async function flush() {
const res = await esClient.bulk({
body: bulkBuffer,
});
if (res.errors)
bulkErrEm.emit('error', res.errors, res);
if (res.items && res.items.length) {
for (let i = 0; i < res.items.length; i++) {
const info = res.items[i];
if (info && info.index && info.index.error) {
bulkErrEm.emit('error', undefined, info.index);
}
}
}
bulkBuffer = [];
};
schema.statics.refresh = async function refresh(inOpts = {}) {
setIndexNameIfUnset(this.modelName);
return await esClient.indices.refresh({
index: inOpts.index || indexName,
});
};
async function postRemove(doc) {
if (!doc) {
return;
}
const opts = {
index: indexName,
tries: 3,
model: doc,
client: esClient,
};
if (routing) {
opts.routing = routing(doc);
}
setIndexNameIfUnset(doc.constructor.modelName);
if (bulk) {
await bulkDelete(opts);
}
else {
await deleteByMongoId(opts);
}
}
schema.statics.bulkError = function bulkError() {
return bulkErrEm;
};
/**
* Use standard Mongoose Middleware hooks
* to persist to Elasticsearch
*/
function setUpMiddlewareHooks(inSchema) {
/**
* Remove in elasticsearch on remove
*/
inSchema.post('remove', postRemove);
inSchema.post('findOneAndRemove', postRemove);
/**
* Save in elasticsearch on save.
*/
inSchema.post('save', postSave);
inSchema.post('findOneAndUpdate', postSave);
inSchema.post('insertMany', (docs) => {
docs.forEach((doc) => postSave(doc));
});
}
if (indexAutomatically) {
setUpMiddlewareHooks(schema);
}
}
exports.mongoosastic = mongoosastic;
//# sourceMappingURL=mongoosastic.js.map