json-object-editor
Version:
JOE the Json Object Editor | Platform Edition
284 lines (264 loc) • 13.2 kB
JavaScript
function Cache(updateNow) {
var modulename = JOE.Utils.color('[cache]','module');
var self = this;
this.settings = {};
this.lookup ={};
this.update = function(callback,collections){//updates the app cache of information that should be quickly accessible.
var uBM = new Benchmarker();
var collections = collections || JOE.Apps.collections || JOE.Apps.get('joe').collections;// || ['trend', 'company', 'eki', 'template', 'doc','capability'];
var callback = callback || function(){logit('cache updated')};
var coll;
var collected = 0;
self.settings = self.settings || {};
self.list = self.list || [];
self.queries = [];
function loadCollection(collection,callb){
//logit('[caching] '+collection);
// JOE.Mongo.get(collection,null,function(err,results){
JOE.Storage.load(collection,null,function(err,results){
collected++;
JOE.Data[collection] = results;
if(collection == "setting" && JOE.Data[collection]){
JOE.Data[collection].map(function(setting){
self.settings[setting.name] = setting.value;
})
}
let coll = (JOE.Data[collection]||[]);
//coll.map(function(item){
for(var i =0;i<coll.length;i++){
let item = coll[i];
self.lookup[item._id] = item;
}
//})
//console.log(Object.keys(self.lookup).length+' keys in lookup');
//TODO: make list update modular
try{
// let prevList = self.list.filter(function(li){return (li.itemtype != collection);});
self.list = self.list.filter(function(li){return (li.itemtype != collection);}).concat(JOE.Data[collection]);
//Object.keys(JOE.Data).map(k=>{
//})
}catch(e){
console.log(JOE.Utils.color('[error] ','red')+e);
}
if(collected == collections.length){
logit(`${modulename} ${collected} of ${collections.length} schemas collected, cached ${self.list.length} items in ${uBM.stop()} secs`);
callb(JOE.Data);
}
})
}
var done = [];
for (var c = 0, tot = collections.length; c < tot; c++) {
coll = collections[c];
if(done.indexOf(coll) == -1){
loadCollection(coll,callback);
done.push(coll);
}
}
};
this.search = function(query){
//gets a search query and finds within all items
//TODO: cache queries
try{
if (!query) { return self.list || []; }
// Fuzzy search path: query contains $fuzzy
if (query.$fuzzy) {
var specs = query.$fuzzy || {};
var exact = {};
for (var k in query) { if (k !== '$fuzzy') { exact[k] = query[k]; } }
var candidates = (self.list || []).where(exact || {});
return self.fuzzySearch(candidates, specs);
}
// Default exact/where
return (self.list || []).where(query);
}catch(e){
console.log(JOE.Utils.color('[cache]','error')+' error in search: '+e);
return [];
}
}
// Compute fuzzy similarity across weighted fields with simple trigram Jaccard
this.fuzzySearch = function(items, specs){
try{
var arr = Array.isArray(items) ? items : (items ? [items] : []);
var q = (specs && specs.q) || '';
var threshold = (typeof specs.threshold === 'number') ? specs.threshold : 0.5;
var limit = (typeof specs.limit === 'number') ? specs.limit : 50;
var offset = (typeof specs.offset === 'number') ? specs.offset : 0;
var highlight = !!specs.highlight;
var minQLen = (typeof specs.minQueryLength === 'number') ? specs.minQueryLength : 2;
if (!q || (q+'').length < minQLen) { return []; }
var resolvedFieldsCache = {};
function norm(s){
try{
if (s == null) { return ''; }
var str = (typeof s === 'string') ? s : (Array.isArray(s) ? s.join(' ') : (typeof s === 'object' ? JSON.stringify(s) : (s+'')));
str = (str+'' ).toLowerCase();
// collapse and trim whitespace so blank/space-only fields don't falsely match
str = str.replace(/\s+/g,' ').trim();
try { str = str.normalize('NFD').replace(/\p{Diacritic}/gu, ''); } catch(_e) {}
return str;
}catch(e){ return (s+'' ).toLowerCase(); }
}
function trigrams(s){
var set = {};
for (var i=0;i<=s.length-3;i++){ set[s.substr(i,3)] = true; }
return set;
}
function jaccard(a,b){
if (!a || !b) { return 0; }
// Direct substring containment is strongest
if (a.indexOf(b) !== -1 || b.indexOf(a) !== -1) { return 1; }
// For very short strings, require exact if not contained
if (a.length < 3 || b.length < 3){ return (a === b) ? 1 : 0; }
var A = trigrams(a), B = trigrams(b);
var i=0, u=0, aSize=0;
var key;
for (key in A){ if (A.hasOwnProperty(key)) { aSize++; if (B[key]) { i++; } u++; } }
for (key in B){ if (B.hasOwnProperty(key) && !A[key]) { u++; } }
var jac = u ? (i/u) : 0;
var containment = aSize ? (i/aSize) : 0; // how much of query's trigrams are in field
return Math.max(containment, jac);
}
function getByPath(obj, path){
try{
if (!path) return undefined;
var parts = (path+'').split('.');
var cur = obj;
for (var i=0;i<parts.length;i++){
if (cur == null) return undefined;
cur = cur[parts[i]];
}
return cur;
}catch(e){ return undefined; }
}
function normalizeFieldDefs(fieldDefs, fromSchema){
// Accept ["name", {path:"info", weight:0.3}] or already-normalized
var defs = [];
var total = 0;
var i;
for (i=0;i<(fieldDefs||[]).length;i++){
var f = fieldDefs[i];
if (typeof f === 'string') { defs.push({ path:f, weight:0 }); }
else if (f && f.path) { defs.push({ path:f.path, weight: typeof f.weight==='number'? f.weight : 0 }); }
}
// If coming from schema and no explicit weights provided, assign equal weights
if (fromSchema === true){
var anyWeighted = defs.some(function(d){ return typeof d.weight === 'number' && d.weight > 0; });
if (!anyWeighted && defs.length){
var eq = 1 / defs.length;
return defs.map(function(d){ return { path:d.path, weight:eq }; });
}
}
// Apply default weights for known fields; distribute remaining equally across those still zero
var DEFAULTS = { name:0.6, info:0.3, description:0.1 };
for (i=0;i<defs.length;i++){
var key = (defs[i].path||'').split('.')[0];
if (defs[i].weight === 0 && DEFAULTS.hasOwnProperty(key)){
defs[i].weight = DEFAULTS[key];
}
}
total = defs.reduce(function(sum,d){ return sum + (d.weight||0); }, 0);
var zeros = defs.filter(function(d){ return !d.weight; });
var remaining = Math.max(0, 1 - total);
if (zeros.length && remaining > 0){
var each = remaining / zeros.length;
zeros.map(function(d){ d.weight = each; });
total = 1;
}
if (total === 0){
// No weights assigned: fall back to defaults trio
return [ {path:'name', weight:0.6}, {path:'info', weight:0.3}, {path:'description', weight:0.1} ];
}
// Normalize to sum=1
defs = defs.map(function(d){ return { path:d.path, weight: d.weight/total }; });
return defs;
}
function resolveFieldsForItem(item, explicitFields){
if (explicitFields && explicitFields.length){ return normalizeFieldDefs(explicitFields, false); }
var it = item && item.itemtype;
if (resolvedFieldsCache[it]) { return resolvedFieldsCache[it]; }
var def = (JOE && JOE.Schemas && JOE.Schemas.schema && JOE.Schemas.schema[it]) || {};
var fields = [];
if (Array.isArray(def.searchables)){
resolvedFieldsCache[it] = normalizeFieldDefs(def.searchables, true);
return resolvedFieldsCache[it];
}
// Default trio
resolvedFieldsCache[it] = [ {path:'name', weight:0.6}, {path:'info', weight:0.3}, {path:'description', weight:0.1} ];
return resolvedFieldsCache[it];
}
var nq = norm(q);
var scored = [];
var useWeighted = Array.isArray(specs && specs.fields) && (specs.fields.length > 0);
for (var idx=0; idx<arr.length; idx++){
var item = arr[idx];
if (!item) { continue; }
var fields = resolveFieldsForItem(item, specs && specs.fields);
var score = 0;
var best = 0;
var matches = [];
for (var f=0; f<fields.length; f++){
var fd = fields[f];
var val = getByPath(item, fd.path);
var s = norm(val);
if (!s) { continue; }
var sim = jaccard(s, nq);
// aggregate
score += (fd.weight||0) * sim;
if (sim > best) { best = sim; }
if (highlight && sim > 0){
var ix = s.indexOf(nq);
if (ix !== -1){ matches.push({ field: fd.path, indices: [[ix, ix + nq.length - 1]] }); }
}
}
var finalScore = useWeighted ? score : best;
if (finalScore >= threshold){
var out = Object.assign({}, item);
out._score = +finalScore.toFixed(6);
if (highlight && matches.length){ out._matches = matches; }
scored.push(out);
}
}
scored.sort(function(a,b){ return (b._score||0) - (a._score||0); });
var totalCount = scored.length;
if (offset && offset > 0) { scored = scored.slice(offset); }
if (limit && limit > 0) { scored = scored.slice(0, limit); }
// Attach count to a non-invasive property if array consumer needs it (not altering API here)
scored.count = totalCount;
return scored;
}catch(e){
console.log(JOE.Utils.color('[cache]','error')+' error in fuzzySearch: '+e);
return [];
}
}
this.findByID = function(collection,id,specs,returnProp){
try{
if($c.isCuid(collection) && !id){
id = collection;
}
if(self.lookup[id]){
return self.lookup[id];
}
var specs = specs || {};
var idprop = specs.idprop||'_id';
var ids = (id||'').split(',');
//return JOE.Data[collection].where(query)||[];
var results = (JOE.Data[collection]||[]).filter(function(item){
return ids.indexOf(item[idprop]+'') != -1;
});
if(id.indexOf(',') == -1){
if(returnProp){
return (results[0]||{})[returnProp];
}
return results[0]||false;
}
return results;
}catch(e){
console.log(JOE.Utils.color('[cache]','error')+' error: in findByID('+(id||'')+') '+e);
}
};
if(updateNow && typeof updateNow == "function"){
this.update(updateNow);
}
return this;
};
module.exports = new Cache();