epub-full-text-search
Version:
275 lines (207 loc) • 9.64 kB
JavaScript
;
// TODOs:
// [ ] separate api feature in own modules
// [X] logging
// [X] rest api del
// [ ] possibility to choose other backend search engines -> interface
var _searchIndex = require('search-index');
var _searchIndex2 = _interopRequireDefault(_searchIndex);
var _q = require('q');
var _q2 = _interopRequireDefault(_q);
var _lodash = require('lodash');
var _lodash2 = _interopRequireDefault(_lodash);
var _validUrl = require('valid-url');
var _validUrl2 = _interopRequireDefault(_validUrl);
var _Logger = require('./Logger');
var _Logger2 = _interopRequireDefault(_Logger);
var _Normalize = require('./Normalize');
var _Normalize2 = _interopRequireDefault(_Normalize);
var _CFI = require('./CFI.js');
var _CFI2 = _interopRequireDefault(_CFI);
var _Constants = require('./Constants');
var _Constants2 = _interopRequireDefault(_Constants);
function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
var DEFAULT_EPUB_TITLE = '*';
var searchIndex = _q2.default.denodeify(_searchIndex2.default);
module.exports = function (options) {
var SearchEngine = {};
var defaultOption = { 'indexPath': _Constants2.default.DATA_FOLDER };
options = _lodash2.default.isEmpty(options) ? defaultOption : options;
SearchEngine.indexing = function (EPUBLocation, uuid) {
_Logger2.default.log('info', "******************************************************");
_Logger2.default.log('info', "Start normalize EPUB content");
if (_validUrl2.default.isUri(EPUBLocation)) {
//TODO: fix url if '/' lost at the end of uri
_Logger2.default.log('info', 'Indexing URI'.blue);
uuid = _Normalize2.default.normalizeUUID(uuid);
return _Normalize2.default.url(EPUBLocation, function (dataSet) {
dataSet.map(function (doc) {
doc.id = doc.id.split(':')[0] + ':' + uuid;
doc.uuid = uuid;
});
_Logger2.default.log('info', "Ready with normalize EPUB");
_Logger2.default.log('info', "Start normalize EPUB.");
// var fs = require('fs');
// fs.writeFileSync('/tmp/fs.tmp', JSON.stringify(dataSet));
return SearchEngine.add(dataSet);
});
} else {
_Logger2.default.log('info', 'Indexing local path'.blue);
return _Normalize2.default.local(EPUBLocation, options, function (dataSet) {
_Logger2.default.log('info', "Ready with normalize EPUB");
if (dataSet.length > 0) {
_Logger2.default.log('info', "Start normalize EPUB-data.");
return SearchEngine.add(dataSet);
} else {
_Logger2.default.log('info', "DONE! Nothing to do, EPUBs already indexed.");
return;
}
});
}
};
SearchEngine.add = function (jsonDoc) {
return SearchEngine._add(jsonDoc, getIndexOptions());
};
SearchEngine.del = function (id) {
if (!id || id.length < 1) return _q2.default.reject('Del function: Id have to be set!');
var q = { query: { AND: { 'uuid': [_Normalize2.default.normalizeUUID(id)] } } };
return SearchEngine._search(q).then(function (result) {
if (!result.hits) {
return q.reject('No hits!');
}
return SearchEngine._del(result.hits);
}).catch(function (err) {
_Logger2.default.log('error', err);
return _q2.default.reject(err);
});
};
SearchEngine.get = function (id) {
var q = { query: { AND: { 'uuid': [_Normalize2.default.normalizeUUID(id)] } } };
return SearchEngine._search(q).then(function (result) {
if (!result.hits) {
return q.reject('No hits!');
}
return result.hits;
}).catch(function (err) {
_Logger2.default.log('error', err);
return _q2.default.reject(err);
});
};
SearchEngine.flush = function () {
return SearchEngine._flush();
};
SearchEngine.search = function (searchFor, bookTitle, uuid) {
var title = bookTitle || DEFAULT_EPUB_TITLE; // * if bookTitle undefined return all hits
var q = {};
q.searchFor = searchFor;
q.query = { AND: { 'body': [searchFor] } };
// Attention AND property have to be an array!!!
if (uuid && uuid.length > 0 && uuid !== '-1') q.query.AND.uuid = [_Normalize2.default.normalizeUUID(uuid)];else q.query.AND.epubTitle = [_Normalize2.default.normalizeEpupTitle(title)];
return SearchEngine.query(q);
};
SearchEngine.query = function (query) {
return SearchEngine._search(query).then(function (result) {
var hits = [];
if (!result.hits) {
return hits;
}
var cfis = result.hits.map(function (hit) {
var document = hit.document;
return _CFI2.default.generate({
"searchFor": query.searchFor,
"spineItemPath": document.spineItemPath,
"baseCfi": document.baseCfi
});
});
return Promise.all(cfis).then(function (cfis) {
cfis.forEach(function (cfiList, i) {
var document = result.hits[i].document,
idData = document.id.split(':');
document.id = idData[0];
document.cfis = cfiList;
delete document['*'];
delete document.spineItemPath;
hits.push(document);
});
return hits;
}).catch(function (err) {
_Logger2.default.log('error', err);
});
}).catch(function (err) {
_Logger2.default.log('error', err);
});
};
SearchEngine.match = function (beginsWith, EPUBTitle, uuid) {
if (!_lodash2.default.isString(EPUBTitle) && !_lodash2.default.isNull(EPUBTitle)) _Logger2.default.log('error', 'EPUBTitle should be null or type string');
if (beginsWith.length < 3) {
//match string must be longer than threshold (3)
var deferred = _q2.default.defer();
deferred.resolve([]);
return deferred.promise;
}
var title = EPUBTitle || DEFAULT_EPUB_TITLE;
return SearchEngine._match({ beginsWith: beginsWith, type: 'ID' }).then(function (matches) {
return filterMatches(matches, title, !uuid ? uuid : _Normalize2.default.normalizeUUID(uuid));
});
};
SearchEngine.close = function () {
return SearchEngine._close();
};
// private
function getIndexOptions() {
return {
fieldOptions: [{ fieldName: 'epubTitle', searchable: false, store: true }, { fieldName: 'spineItemPath', searchable: false, store: true }, { fieldName: 'href', searchable: false, store: true }, { fieldName: 'baseCfi', searchable: false, store: true }, { fieldName: 'id', searchable: false, store: true }, { fieldName: 'filename', searchable: true, store: true }, { fieldName: 'title', searchable: true, store: false }, { fieldName: 'body', searchable: true, store: false }, { fieldName: 'uuid', searchable: false, store: true }]
};
}
function filterMatches(matches, EPUBTitle, uuid) {
return matches.map(function (match) {
if (uuid && uuid !== '-1') {
var titles = match[1].map(function (id) {
return id.split(':')[1];
});
return _lodash2.default.include(titles, uuid) ? match[0] : '';
} else if (EPUBTitle === '*') {
// if EPUBTitle undefined return all matches
return match[0];
} else {
var _titles = match[1].map(function (id) {
// id = spineitemid:epubtitle
return id.split(':')[1];
});
return _lodash2.default.include(_titles, EPUBTitle) ? match[0] : '';
}
}).filter(Boolean); // filter ["", "", ""] -> []
}
return searchIndex(options).then(function (si) {
SearchEngine.si = si;
SearchEngine._search = _q2.default.nbind(si.search, si);
SearchEngine._close = _q2.default.nbind(si.close, si);
SearchEngine._match = _q2.default.nbind(si.match, si);
SearchEngine._add = _q2.default.nbind(si.add, si);
SearchEngine._del = _q2.default.nbind(si.del, si);
SearchEngine._get = _q2.default.nbind(si.get, si);
SearchEngine._flush = _q2.default.nbind(si.flush, si);
return SearchEngine;
});
};
/* require('search-index')({
nGramLength: [1, 2, 3], // allow phrase search on phrases of length 1, 2 and 3 words
stopwords: [] // dont strip out stopwords
}, function(err, si) {
si.add([
{date: 1464122926, text: 'some text'},
{date: 1464122916, text: 'some another text'}
], err => {
if (! err) {
const q = {};
q.query = { AND: [{'*': ['text']}] };
si.search(q, (err, results) => {
winston.log('info', results);
})
}
})
})
1. Use nGramLength to activate phrase search
2. Set stopwords to []. By default, search-index will remove common english words like 'some' and 'another',
so you need to empty your stopword list to allow them to
be searchable (you could also remove them from the stopword list individually) */