fz-search
Version:
Fast aproximate string matching library for use in autocomplete, perform both search and highlight.
1,640 lines (1,276 loc) • 89.2 kB
JavaScript
/**
* @license FuzzySearch.js
* Autocomplete suggestion engine using approximate string matching
* https://github.com/jeancroy/FuzzySearch
*
* Copyright (c) 2015, Jean Christophe Roy
* Licensed under The MIT License.
* http://opensource.org/licenses/MIT
*/
(function () { 'use strict';
/**
* @param options
* @constructor
*/
'use strict';
function FuzzySearch(options) {
if (options === undefined) options = {};
if (!(this instanceof FuzzySearch)) return new FuzzySearch(options);
FuzzySearch.setOptions(this, options, FuzzySearch.defaultOptions, _privates, true, this._optionsHook)
}
FuzzySearch.defaultOptions =
/** @lends {FuzzySearchOptions.prototype} */{
//
// Scoring, include in result
//
minimum_match: 1.0, // Minimum score to consider two token are not unrelated
thresh_include: 2.0, // To be a candidate, score of item must be at least this
thresh_relative_to_best: 0.5, // and be at least this fraction of the best score
field_good_enough: 20, // If a field have this score, stop searching other fields. (field score is before item related bonus)
//
// Scoring, bonus
//
bonus_match_start: 0.5, // Additional value per character in common prefix
bonus_token_order: 2.0, // Value of two token properly ordered
bonus_position_decay: 0.7, // Exponential decay for position bonus (smaller : more importance to first item)
score_per_token: true, // if true, split query&field in token, allow to match in different order
// if false, bypass at least half the computation cost, very fast
// also disable different token that score different field, because no more token!!
score_test_fused: false, // Try one extra match where we disregard token separation.
// "oldman" match "old man"
score_acronym: false, // jrrt match against John Ronald Reuel Tolkien
token_sep: " .,-:",
//
// Output sort & transform
//
score_round: 0.1, // Two item that have the same rounded score are sorted alphabetically
output_limit: 0, // Return up to N result, 0 to disable
sorter: compareResults, // Function used to sort. See signature of Array.sort(sorter)
normalize: normalize, // Function used to transform string (lowercase, accents, etc)
filter: null, // Select elements to be searched. (done before each search)
/**@type {string|function({SearchResult})}*/
output_map: "item", // Transform the output, can be a function or a path string.
// output_map="root" return SearchResult object, needed to see the score
// output_map="root.item" return original object.
// output_map="root.item.somefield" output a field of original object.
// (root.) is optional.
//
// output_map=function(root){ return something(root.item) }
// ^this get original object and apply something() on it.
join_str: ", ", //String used to join array fields
//
// Tokens options
//
token_query_min_length: 2, // Avoid processing very small words, include greater or equal, in query
token_field_min_length: 3, // include greater or equal, in item field
token_query_max_length: 64, // Shorten large token to give more even performance.
token_field_max_length: 64, // Shorten large token to give more even performance.
token_fused_max_length: 64, // Shorten large token to give more even performance.
//Do not attempt to match token too different in size: n/m = len(field_tok)/len(query_tok)
token_min_rel_size: 0.6, // Field token should contain query token. Reject field token that are too small.
token_max_rel_size: 10, // Large field token tend to match against everything. Ensure query is long enough to be specific.
//
// Interactive - suggest as you type.
// Avoid doing search that will be discarded without being displayed
// This also help prevent lag/ temp freeze
//
interactive_debounce: 150, // This is initial value. Will try to learn actual time cost. Set to 0 to disable.
interactive_mult: 1.2, // Overhead for variability and to allow other things to happens (like redraw, highlight ).
interactive_burst: 3, // Allow short burst, prevent flicker due to debounce suppression of a callback
//
// Data
//
source: [],
keys: [],
lazy: false, // when true, any refresh happens only when a user make a search, option stay put until changed.
token_re: /\s+/g, //Separator string will be parsed to this re.
identify_item: null, // How to uniquely identify an item when adding to the index. Defaults to null, meaning no duplicate detection. Must be a method that takes a single (source) argument.
use_index_store: false, // Enable a time vs memory trade-off for faster search (but longer initial warm-up).
store_thresh: 0.7, // cutoff point relative to best, to graduate from store phase.
store_max_results: 1500 // Maximum number of result to graduate from store, to the full search quality algorithm
// Note that store only perform a crude search, ignoring some options, so the best result can be only "meh" here.
};
var _privates =
/** @lends {FuzzySearch.prototype} */{
keys: [],
tags: [], // alternative name for each key, support output alias and per key search
index: [], // source is processed using keys, then stored here
index_map: {}, // To manage update of record already in dataset
nb_indexed: 0, // To manage active count of index
store: {}, // Dictionary used for time VS memory trade off. (Optional)
tags_re: null,
acro_re: null,
token_re: null,
/**@type {FuzzySearchOptions}*/
options: null,
dirty: false, // when true, schedule a source refresh using new or existing source & keys, used once then clear itself.
//Information on last search
query: null,
results: [],
start_time: 0,
search_time: 0
};
/**
* Number of bit in a int.
* DEBUG-tip: setting this to zero will force "long string" algorithm for everything!
* @const
*/
var INT_SIZE = 32;
function FuzzySearchOptions(defaults, options) {
for (var key in defaults) {
if (defaults.hasOwnProperty(key)) { //fill self with value from either options or default
this[key] = (options.hasOwnProperty(key) && options[key] !== undefined ) ? options[key] : defaults[key];
}
}
}
FuzzySearchOptions.update = function (self, defaults, options) {
for (var key in options) {
if (options.hasOwnProperty(key) && defaults.hasOwnProperty(key)) {
//explicitly set a options to undefined => reset default, else get value
self[key] = (options[key] === undefined) ? defaults[key] : options[key];
}
}
};
/**
* Set property of object,
* Restrict properties that can be set from a list of available defaults.
*
* @param {FuzzySearch} self
* @param {Object} options
* @param {Object} defaults
* @param {Object} privates
* @param {boolean} reset
* @param {function({Object})} hook
*
*/
FuzzySearch.setOptions = function (self, options, defaults, privates, reset, hook) {
if (reset) {
extend(self, privates);
self.options = new FuzzySearchOptions(defaults, options);
} else {
FuzzySearchOptions.update(self.options, defaults, options);
}
hook.call(self, options)
};
function extend(a, b) {
for (var key in b) if (b.hasOwnProperty(key)) a[key] = b[key];
}
//
// - - - - - - - - - - - -
// SET & PARSE SETTINGS
// - - - - - - - - - - - -
//
extend(FuzzySearch.prototype, /** @lends {FuzzySearch.prototype} */ {
/**
* Allow to change options after the object has been created.
* If source is changed, new source is indexed.
*
* Optional reset allow to change any setting not in options to defaults.
* This is similar to creating new object, but using same pointer.
*
* @param {Object} options
* @param {boolean=} reset
*/
setOptions: function (options, reset) {
if (reset === undefined) reset = options.reset || false;
FuzzySearch.setOptions(this, options, FuzzySearch.defaultOptions, _privates, reset, this._optionsHook);
},
/**
*
* @param {Object} options
* @private
*/
_optionsHook: function (options) {
//Items of options have been copied into this.options
//We still test "option_name in option" to know if we have received something new
//This allow to support "shorthand" options and is used to refresh data.
var self_options = this.options;
//Output stage
if ("output_map" in options && typeof options.output_map === "string") {
if (self_options.output_map === "alias") self_options.output_map = this.aliasResult;
else self_options.output_map = removePrefix(self_options.output_map, ["root", "."]);
}
this.source = self_options.source;
// Input stage, work to allow different syntax for keys definition is done here.
var oKeys;
if (("keys" in options) && ( ( oKeys = options.keys) !== undefined)) {
var key_type = Object.prototype.toString.call(oKeys);
var key_index, nb_keys;
this.tags = null;
if (key_type === "[object String]") {
this.keys = oKeys.length ? [oKeys] : [];
}
else if (key_type === "[object Object]") {
this.keys = [];
this.tags = []; //we don't know the "length" of dictionary
key_index = 0;
for (var tag in oKeys) {
if (oKeys.hasOwnProperty(tag)) {
this.tags[key_index] = tag;
this.keys[key_index] = oKeys[tag];
key_index++;
}
}
}
else {
this.keys = oKeys;
}
oKeys = this.keys;
nb_keys = oKeys.length;
for (key_index = -1; ++key_index < nb_keys;) {
oKeys[key_index] = removePrefix(oKeys[key_index], ["item", "."])
}
if (!this.tags) this.tags = oKeys;
this.tags_re = buildTagsRE(this.tags);
}
if (this.acro_re === null || "acronym_tok" in options) {
this.acro_re = buildAcronymRE(self_options.token_sep);
}
if (this.token_re === null || "token_sep" in options) {
this.token_re = self_options.token_re = new RegExp("[" + re_escape(self_options.token_sep) + "]+", "g");
}
// Determine if we need to rebuild this.index from this.source
if (options.dirty || ("source" in options) || ("keys" in options) || ("use_index_store" in options)) {
if (self_options.lazy) this.dirty = true; // Schedule later.
else {
this._buildIndexFromSource();
this.dirty = false;
}
}
}
});
/**
* Removes optional prefix of paths.
* for example "root.", "."
*
* @param {string} str - input
* @param {Array<string>} prefixes to remove
* @returns {string}
*/
function removePrefix(str, prefixes) {
var n = prefixes.length;
var offset = 0;
for (var i = -1; ++i < n;) {
var p = prefixes[i], l = p.length;
if (str.substr(offset, l) === p) offset += l;
}
return (offset > 0) ? str.substr(offset) : str;
}
function buildTagsRE(tags) {
var n = tags.length;
if (!n) return null;
var tag_str = re_escape(tags[0]);
for (var i = 0; ++i < n;) {
tag_str += "|" + re_escape(tags[i]);
}
return new RegExp("(?:^|\\s)\\s*(" + tag_str + "):\\s*", "g");
}
function buildAcronymRE(sep) {
var n = sep.length;
if (!n) return null;
var acro_str = re_escape(sep);
return new RegExp("(?:^|[" + acro_str + "])+([^" + acro_str + "])[^" + acro_str + "]*", "g");
}
// Build regexp for tagged search
function re_escape(str) {
var re = /[\-\[\]\/\{}\(\)\*\+\?\.\\\^\$\|]/g;
return str.replace(re, "\\$&");
}
//
// - - - - - - - - - - - -
// OUTPUT OR POST PROCESS
// - - - - - - - - - - - -
//
'use strict';
extend(FuzzySearch.prototype, /** @lends {FuzzySearch.prototype} */ {
/**
* Given a SearchResult object, recover the value of the best matching field.
* This is done on demand for display.
*
* @param {SearchResult} result
* @return {string} original field
*/
getMatchingField: function (result) {
var f = FuzzySearch.generateFields(result.item, [this.keys[result.matchIndex]]);
return f[0][result.subIndex];
},
/**
* Given a SearchResult object, generate a new object that follow alias structure
* @param {SearchResult} result
* @return {*} aliased result
*/
aliasResult: function (result) {
var options = this.options;
var f = FuzzySearch.generateFields(result.item, this.keys);
var out = {}, tags = this.tags, join_str = options.join_str;
for (var i = -1, n = f.length; ++i < n;) {
out[tags[i]] = f[i].join(join_str)
}
out._item = result.item;
out._score = result.score;
out._match = f[result.matchIndex][result.subIndex];
return out;
}
});
// - - - - - - - - - - - - - - - - - - - - - -
// Output stage, prepare results for return
//- - - - - - - - - - - - - - - - - - - - - -
/**
* Own version of Array.prototype.map()
*
* @param {Array} source
* @param transform callback
* @param {*=} context (*this* in called function)
* @param {number=} max_out
* @returns {Array}
*/
FuzzySearch.map = function (source, transform, context, max_out) {
var n = source.length;
if (max_out > 0 && max_out < n) n = max_out;
if (typeof transform !== "function") return source.slice(0, n);
var out = new Array(n);
for (var i = -1; ++i < n;) {
out[i] = transform.call(context, source[i], i, source);
}
return out;
};
/**
* Take an array of objects, return an array containing a field of those object.
*
* test = [ {key:"A",value:10}, {key:"B",value:20} ]
* mapField(test,"value") = [10,20]
*
* @param source - array to process
* @param {string} path - key to address on each item OR function to apply
* @param {Number=} [max_out=source.length] - only process first items
* @returns {Array}
*/
FuzzySearch.mapField = function (source, path, max_out) {
var n = source.length;
if (max_out > 0 && max_out < n) n = max_out;
if (path === "") return source.slice(0, n);
var out = new Array(n);
var obj, i;
if (path.indexOf(".") === -1) {
//fast case no inner loop
for (i = -1; ++i < n;) {
obj = source[i];
if (path in obj) out[i] = obj[path];
}
} else {
//general case
var parts = path.split(".");
var nb_level = parts.length;
for (i = -1; ++i < n;) {
obj = source[i];
for (var level = -1; ++level < nb_level;) {
var key = parts[level];
if (!(key in obj)) break;
obj = obj[key];
}
out[i] = obj;
}
}
return out;
};
/**
* Filter array for item where item[field] >= atleast
*
* @param array
* @param field
* @param atleast
* @returns {Array}
*/
FuzzySearch.filterGTE = function (array, field, atleast) {
var i = -1, j = -1;
var n = array.length;
var out = [], obj;
while (++i < n) {
obj = array[i];
if (obj[field] >= atleast) {
out[++j] = obj;
}
}
return out;
};
/**
* SearchResult constructor
* - Internal result list
* - Output of search when output_map=""
*
* @param {*} item
* @param {Array} fields
* @param {number} item_score
* @param {number} matched_field_index
* @param {number} matched_field_sub
* @param {(string|number)} sortkey
* @constructor
*/
function SearchResult(item, fields, item_score, matched_field_index, matched_field_sub, sortkey) {
this.item = item;
this.fields = fields;
this.score = item_score;
this.matchIndex = matched_field_index;
this.subIndex = matched_field_sub;
this.sortKey = sortkey;
}
/**
* Sort function
* first by decreasing order of score, then alphabetical order of sortkey.
*
* @param {SearchResult} a
* @param {SearchResult} b
* @returns {number} - ">0" if b before a, "<0" if b after a.
*/
function compareResults(a, b) {
var d = b.score - a.score;
if (d !== 0) return d;
var ak = a.sortKey, bk = b.sortKey;
return ak > bk ? 1 : ( ak < bk ? -1 : 0);
}
//
// - - - - - - - - - - - -
// Prepare Query
// - - - - - - - - - - - -
//
extend(FuzzySearch.prototype, /** @lends {FuzzySearch.prototype} */ {
/**
* Input: a user search string
* Output a query object
*
* Perform a few transformation to allw faster searching.
* String is set to lowercase, some accents removed, split into tokens.
* Token too small are filtered out, token too large are trimmed.
* Token are packed in group of 32 char, each token is processed to extract an alphabet map.
*
* If score_test_fused is enabled, we do an extra pass disregarding tokens.
* IF score_per_token is disabled this is the only pass we do.
*
* @param query_string
* @returns {Query}
* @private
*/
_prepQuery: function (query_string) {
var options = this.options;
var opt_tok = options.score_per_token;
var opt_fuse = options.score_test_fused;
var opt_fuselen = options.token_fused_max_length;
var opt_qmin = options.token_field_min_length;
var opt_qmax = options.token_field_max_length;
var tags = this.tags;
var tags_re = this.tags_re;
var nb_tags = tags.length;
var token_re = this.token_re;
var norm, fused, fused_map, children, has_tags, group, words;
if (opt_tok && nb_tags && tags_re) {
var start = 0, end;
var q_index = 0;
var q_parts = new Array(nb_tags + 1);
var match = tags_re.exec(query_string);
has_tags = (match !== null);
while (match !== null) {
end = match.index;
q_parts[q_index] = query_string.substring(start, end);
start = end + match[0].length;
q_index = tags.indexOf(match[1]) + 1;
match = tags_re.exec(query_string);
}
q_parts[q_index] = query_string.substring(start);
children = [];
for (var i = -1; ++i < nb_tags;) {
var qp = q_parts[i + 1];
if (!qp || !qp.length) continue;
norm = options.normalize(qp);
fused = norm.substring(0, opt_fuselen);
fused_map = (opt_fuse || !opt_tok) ? FuzzySearch.alphabet(fused) : {};
words = FuzzySearch.filterSize(norm.split(token_re), opt_qmin, opt_qmax);
group = FuzzySearch.pack_tokens(words);
children[i] = new Query(norm, words, group, fused, fused_map, false, []);
}
norm = options.normalize(q_parts[0]);
words = FuzzySearch.filterSize(norm.split(token_re), opt_qmin, opt_qmax);
group = FuzzySearch.pack_tokens(words);
}
else {
norm = options.normalize(query_string);
words = FuzzySearch.filterSize(norm.split(token_re), opt_qmin, opt_qmax);
group = opt_tok ? FuzzySearch.pack_tokens(words) : [];
has_tags = false;
children = new Array(nb_tags);
}
fused = norm.substring(0, opt_fuselen);
fused_map = (opt_fuse || !opt_tok) ? FuzzySearch.alphabet(fused) : {};
return new Query(norm, words, group, fused, fused_map, has_tags, children)
}
});
//
// Query objects
//
/**
* Hold a query
*
* @param {string} normalized
* @param {Array.<string>} words
* @param {Array.<PackInfo>} tokens_groups
* @param {string} fused_str
* @param {Object} fused_map
* @param {boolean} has_children
* @param {Array<Query>} children
*
* @constructor
*/
function Query(normalized, words, tokens_groups, fused_str, fused_map, has_children, children) {
this.normalized = normalized;
this.words = words;
this.tokens_groups = tokens_groups;
this.fused_str = fused_str;
this.fused_map = fused_map;
this.fused_score = 0;
this.has_children = has_children;
this.children = children;
}
//
// Query hold some memory to keep score of it's tokens.
// Used in search methods
/**
* Loop tru each item score and reset to 0, apply to child query
*/
Query.prototype.resetItem = function () {
var groups = this.tokens_groups;
for (var group_index = -1, nb_groups = groups.length; ++group_index < nb_groups;) {
var score_item = groups[group_index].score_item;
for (var i = -1, l = score_item.length; ++i < l;) score_item[i] = 0
}
this.fused_score = 0;
if (this.has_children) {
var children = this.children;
for (var child_index = -1, nb_child = children.length; ++child_index < nb_child;) {
var child = children[child_index];
if (child) child.resetItem();
}
}
};
/**
* Sum each item score and add to child score
*/
Query.prototype.scoreItem = function () {
var query_score = 0;
var groups = this.tokens_groups;
for (var group_index = -1, nb_groups = groups.length; ++group_index < nb_groups;) {
var group_scores = groups[group_index].score_item;
for (var score_index = -1, nb_scores = group_scores.length; ++score_index < nb_scores;) {
query_score += group_scores[score_index]
}
}
if (this.fused_score > query_score) query_score = this.fused_score;
if (this.has_children) {
var children = this.children;
for (var child_index = -1, nb_child = children.length; ++child_index < nb_child;) {
var child = children[child_index];
if (child) query_score += child.scoreItem();
}
}
return query_score;
};
/**
* Hold a group of token for parallel scoring
*
* @param {Array.<string>} group_tokens
* @param {Object} group_map
* @param {number} gate
* @constructor
*/
function PackInfo(group_tokens, group_map, gate) {
this.tokens = group_tokens;
this.map = group_map;
this.gate = gate;
var t = group_tokens.length, i = -1;
var scores = new Array(t);
while (++i < t) scores[i] = 0;
this.score_item = scores.slice();
this.score_field = scores.slice();
this.field_pos = scores;
}
//
// - - - - - - - - - - - - - - - - -
// Prepare Token for search
// - - - - - - - - - - - - - - - - -
// a normal string can be view as an array of char.
// so we map ( position -> char).
//
// we reverse that relation to map
// char -> positions
/**
* Record position of each character in a token.
* If token is small, position is recorded by position of a single bit in an int.
* If token is larger than INT_SIZE, position is recorder as array of number.
*
* @param {string} token
* @returns {Object} key value map char->positions (as array of position or single int (can be seen as an array of bit) )
*/
FuzzySearch.alphabet = function (token) {
var len = token.length;
if (len > INT_SIZE) return FuzzySearch.posVector(token);
else return FuzzySearch.bitVector(token, {}, 0);
};
/**
* Apply FuzzySearch.alphabet on multiple tokens
*
* @param {Array.<string>} tokens
* @returns {Array.<Object>}
*/
FuzzySearch.mapAlphabet = function (tokens) {
var outlen = tokens.length;
var out = new Array(outlen), i = -1;
while (++i < outlen) {
var t = tokens[i];
if (t.length > INT_SIZE) out[i] = FuzzySearch.posVector(t);
else out[i] = FuzzySearch.bitVector(t, {}, 0);
}
return out;
};
/**
* Record position of each char using a single bit
*
* @param {string} token
* @param {Object} map - Existing map to modify, can init with {}
* @param offset - used for packing multiple word in a single map, can init with 0
* @returns {Object} Key value map char -> int
*/
FuzzySearch.bitVector = function (token, map, offset) {
var len = token.length;
var i = -1, c;
var b = offset;
while (++i < len) {
c = token[i];
if (c in map) map[c] |= (1 << b++);
else map[c] = (1 << b++);
}
return map;
};
/**
* Record position of each char in a token using an array
* Append Infinity as a stop marker for llcs_large
*
* map = posVector("position")
* map["p"] -> [0,Inf]
* map["o"] -> [1,6,Inf]
*
* @param {string} pattern
* @returns {Object} - key value map char->array of position (as number)
*/
FuzzySearch.posVector = function (pattern) {
var map = {}, c;
var m = pattern.length, i = -1;
while (++i < m) {
c = pattern[i];
if (c in map) map[c].push(i);
else map[c] = [i];
}
for (c in map) {
if (map.hasOwnProperty(c)) {
map[c].push(Infinity);
}
}
return map;
};
/**
* Given a list of tokens, pack them into group of upto INT_SIZE(32) chars.
* If a single token is bigger than INT_SIZE create a groupe of a single item
* And use posVector instead of bitVector to prepare fallback algorithm.
*
* @param {Array.<string>} tokens
* @returns {Array.<PackInfo>}
*/
FuzzySearch.pack_tokens = function (tokens) {
var token_index = -1;
var nb_tokens = tokens.length;
var large;
var groups = [];
//For each group
while (token_index < nb_tokens) {
var group_tokens = [];
var group_map = {};
var offset = 0;
var gate = 0;
//For each token in the group
while (++token_index < nb_tokens) {
var token = tokens[token_index];
var l = token.length;
if (l >= INT_SIZE) {
large = new PackInfo([token],
FuzzySearch.posVector(token),
0xFFFFFFFF);
break;
}
else if (l + offset >= INT_SIZE) {
token_index--;
break;
}
else {
group_tokens.push(token);
FuzzySearch.bitVector(token, group_map, offset);
gate |= ( (1 << ( token.length - 1) ) - 1 ) << offset;
offset += l
}
}
if (group_tokens.length > 0) {
groups.push(new PackInfo(group_tokens, group_map, gate));
}
if (large) {
groups.push(large);
large = null;
}
}
return groups;
};
//
//-----------------------------
// SCORING FUNCTIONS
// ---------------------------
//
'use strict';
/**
* Score of "search a in b" using self as options.
* @param {string} a
* @param {string} b
*/
FuzzySearch.prototype.score = function (a, b) {
var aMap = FuzzySearch.alphabet(a);
return FuzzySearch.score_map(a, b, aMap, this.options);
};
// Adapted from paper:
// A fast and practical bit-vector algorithm for
// the Longest Common Subsequence problem
// Maxime Crochemore et Al.
//
// With modification from
// Bit-parallel LCS-length computation revisited (H Hyyrö, 2004)
// http://www.sis.uta.fi/~hh56766/pubs/awoca04.pdf
//
/**
* Score of "search a in b" using precomputed alphabet map
* Main algorithm for single query token to score
*
* @param {string} a
* @param {string} b
* @param {Object} aMap - See FuzzySearch.alphabet
* @param {FuzzySearchOptions} options
*/
FuzzySearch.score_map = function (a, b, aMap, options) {
var j, lcs_len;
var m = a.length;
var n = b.length;
var bonus_prefix = options.bonus_match_start;
var k = m < n ? m : n;
if (k === 0) return 0;
//normalize score against length of both inputs
var sz_score = (m + n) / ( 2.0 * m * n);
//common prefix is part of lcs
var prefix = 0;
if (a === b) prefix = k; //speedup equality
else {
while ((a[prefix] === b[prefix]) && (++prefix < k)) {
}
}
//shortest string consumed
if (prefix === k) {
lcs_len = prefix;
return sz_score * lcs_len * lcs_len + bonus_prefix * prefix;
}
//alternative algorithm for large string
//need to keep this condition in sync with bitvector
if (m > INT_SIZE) {
lcs_len = FuzzySearch.llcs_large(a, b, aMap, prefix);
return sz_score * lcs_len * lcs_len + bonus_prefix * prefix;
}
var mask = ( 1 << m ) - 1;
var S = mask, U, c;
j = prefix - 1;
while (++j < n) {
c = b[j];
if (c in aMap) {
// Hyyrö, 2004 S=V'=~V
U = S & aMap[c];
S = (S + U) | (S - U);
}
}
// Remove match already accounted in prefix region.
mask &= ~( ( 1 << prefix ) - 1 );
// lcs_len is number of 0 in S (at position lower than m)
// inverse S, mask it, then do "popcount" operation on 32bit
S = ~S & mask;
S = S - ((S >> 1) & 0x55555555);
S = (S & 0x33333333) + ((S >> 2) & 0x33333333);
lcs_len = (((S + (S >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24;
lcs_len += prefix;
return sz_score * lcs_len * lcs_len + bonus_prefix * prefix;
};
/**
* Call score_map on the first token.
* Filter size
*
* @param {PackInfo} packinfo
* @param {string} token
* @param {FuzzySearchOptions} options
* @return {Array.<number>} score
*/
FuzzySearch.score_single = function (packinfo, token, options) {
var field_tok = packinfo.tokens[0];
var m = field_tok.length;
var n = token.length;
if (n < options.token_min_rel_size * m || n > options.token_max_rel_size * m) return [0];
return [FuzzySearch.score_map(field_tok, token, packinfo.map, options)];
};
/**
* Score multiple query token against a single field token.
* Apply above score function in parallel
* Computation is done as if everything was one big token,
* but ZM bit-vector modify boundary so score are independant
*
* @param {PackInfo} packinfo
* @param {string} field_token
* @param {FuzzySearchOptions} options
* @returns {Array.<number>} scores
*/
FuzzySearch.score_pack = function (packinfo, field_token, options) {
var packed_tokens = packinfo.tokens;
var nb_packed = packed_tokens.length;
//single item token can contain either a single word "overflow" or a large word that need special handling
if (nb_packed == 1)return FuzzySearch.score_single(packinfo, field_token, options);
var S = 0xFFFFFFFF, U, c;
var ZM = packinfo.gate | 0;
var aMap = packinfo.map;
for (var j = -1, n = field_token.length; ++j < n;) {
c = field_token[j];
if (c in aMap) {
U = S & aMap[c];
S = ( (S & ZM) + (U & ZM) ) | (S - U);
}
}
S = ~S;
var bonus_prefix = options.bonus_match_start;
var min_rs = options.token_min_rel_size;
var max_rs = options.token_max_rel_size;
var scores = new Array(nb_packed);
var offset = 0;
for (var k = -1; ++k < nb_packed;) {
var query_tok = packed_tokens[k];
var m = query_tok.length;
var lcs_len, prefix;
if (n < min_rs * m || n > max_rs * m) {
scores[k] = 0;
offset += m;
continue;
}
if (query_tok === field_token)
prefix = lcs_len = m;
else {
var p = (m < n) ? m : n;
prefix = 0;
while ((query_tok[prefix] === field_token[prefix]) && (++prefix < p)) {
}
lcs_len = prefix;
var Sm = ( (S >>> offset) & ( (1 << m) - 1 ) ) >>> prefix;
while (Sm) {
Sm &= Sm - 1;
lcs_len++
}
}
offset += m;
var sz = (m + n) / ( 2.0 * m * n);
scores[k] = sz * lcs_len * lcs_len + bonus_prefix * prefix;
}
return scores;
};
//
// Compute LLCS, using vectors of position.
//
// Based on:
// An input sensitive online algorithm for LCS computation
// Heikki Hyyro 2009
//
// We fill the dynamic programing table line per line
// but instead of storing the whole line we only store position where the line increase
// ( bitvector algorithm store increase yes/no as a bit) this time we will store sequence
//
// s u r g e r y
// g [0,0,0,1,1,1,1] : [3,4] (Add level 1)
// s [1,1,1,1,1,1,1] : [0,1] (Make level 1 happens sooner)
// u [1,2,2,2,2,2,2] : [0,2] (Add level 2, append to block of consecutive increase)
// r [1,2,3,3,3,3,3] : [0,3] (Add level 3, append to block of consecutive increase)
// v [1,2,3,3,3,3,3] : [0,3] (v not in surgery, copy)
// e [1,2,3,3,4,4,4] : [0,3],[4,5] (Add level 4, create new block for it)
// y [1,2,3,3,4,4,5] : [0,3],[4,5],[6,7] (Add level 5, create new block for it)
//
// There is 2 Basic operations:
// - Make a level-up happens sooner
// - Add an extra level up at the end. (this is where llcs increase !)
//
// 12345678901234567890 // Position (for this demo we start at 1)
// ii------iii---i--i-- // Increase point of previous line
// 12222222345555666777 // Score previous line [1,3] [9,12] [15,16] [18,19]
// ---m-m---------m---m // Match of this line
// 12233333345555677778 // Score of this line [1,3] [4,5] [10,12] [15,17] [20,21]
// ii-i-----ii---ii---i // New increase point
// 12345678901234567890 // Position
FuzzySearch.llcs_large = function (a, b, aMap, prefix) {
//var aMap = FuzzySearch.posVector(a);
//Position of next interest point. Interest point are either
// - Increase in previous line
// - Match on this line
var block_start, match_pos;
// We encode increase sequence as [start_pos, end_pos+1]
// So end-start = length
// To avoid dealing with to many edge case we place
// a special token at start & end of list
var last_line, line_index, last_end, block_end;
if (prefix === undefined) prefix = 0;
if (prefix)
last_line = [new Block(0, prefix), new Block(Infinity, Infinity)];
else
last_line = [new Block(Infinity, Infinity)];
var lcs_len = prefix;
var match_list, match_index;
var block, block_index, block_size;
//First line
var nb_blocks = last_line.length;
var n = b.length, j;
for (j = prefix; j < n; j++) {
//Each line we process a single character of b
var c = b[j];
if (!(c in aMap)) continue;
match_list = aMap[c];
//New line
// the number of if block can only increase up to llcs+1+sentinel
// alternatively each block having >1 item can split. (+1 at end accounted by splitting sentinel)
/** @type Array.<Block> */
var current_line = new Array(Math.min(2 * nb_blocks, lcs_len + 2));
line_index = -1;
//First match
match_index = 0;
match_pos = match_list[0];
//Place end of first block before the string
block_end = -1;
block_index = -1;
while (++block_index < nb_blocks) {
//Place cursor just after last block
last_end = block_end;
//Read end block
block = last_line[block_index];
block_start = block.start; //Encode block as [s,e[
block_end = block.end; //End is position of char that follow last.
block_size = block_end - block_start; //Size of block, for sentinel (Inf-Inf=NaN)
//get next match from list of matches
while (match_pos < last_end) {
match_pos = match_list[++match_index];
}
// This cover two case
// a) no match between two block
// b) block happens after last match (so match_pos=Infinity).
// At the last block, this will append closing "sentinel" to line
if (block_start <= match_pos) {
current_line[++line_index] = block;
continue;
}
//
// If we have reached here, we have a dominant match !
// Decide where to register the match ...
//
if (match_pos === last_end) {
//End of last block ? (step a.ii)
current_line[line_index].end++;
}
else {
//Increase need it's own block ( step a.i)
//try to reuse block that will get deleted.
if (block_size === 1) {
//Can we reuse next block ?
block.start = match_pos;
block.end = match_pos + 1;
current_line[++line_index] = block;
} else {
//start a new block
current_line[++line_index] = new Block(match_pos, match_pos + 1);
}
}
// if not empty, append next block to current line (step a.iii)
// (this condition reject "sentinel", it'll get added just after the for loop)
if (block_size > 1) {
block.start++; // Move start by one
current_line[++line_index] = block;
}
}
// If the line finish with a match:
// a) llcs at end of this line is one greater than last line, increase score
// b) we still need to append sentinel
if (block_start > match_pos) {
current_line[++line_index] = block;
lcs_len++
}
//Current become last
last_line = current_line;
//Count actual number of block because we allocate a bit more.
nb_blocks = ++line_index;
}
return lcs_len;
};
/**
* A block with start and end position
* Used to record consecutive increase position in llcs_large
* @param start
* @param end
* @constructor
*/
function Block(start, end) {
this.start = start;
this.end = end;
}
//
// Reference implementation to debug
// Might need to swap input to match internal of a given algorithm
//
/*
function lcs(a, b) {
var m = a.length;
var n = b.length;
var i, j;
//init m by n array with 0
var C = [], row = [], lcs = [];
for (j = 0; j < n; j++) row[j] = 0;
for (i = 0; i < m; i++) C[i] = row.slice();
//fill first row and col
C[0][0] = (a[0] === b[0]) ? 1 : 0;
for (i = 1; i < m; i++) C[i][0] = (a[i] === b[0] || C[i - 1][0]) ? 1 : 0
for (j = 1; j < n; j++) C[0][j] = (a[0] === b[j] || C[0][j - 1]) ? 1 : 0
console.log(JSON.stringify(C[0]));
//bulk
for (i = 1; i < m; i++) {
for (j = 1; j < n; j++) {
C[i][j] = (a[i] === b[j]) ? C[i - 1][j - 1] + 1 : Math.max(C[i][j - 1], C[i - 1][j]);
}
console.log(JSON.stringify(C[i]));
}
//backtrack
i--;
j--;
while (i > -1 && j > -1) {
if (i && C[i][j] == C[i - 1][j]) i--;
else if (j && C[i][j] == C[i][j - 1]) j--;
else {
lcs.push(a[i]);
j--;
i--;
}
}
return lcs.reverse().join('');
}*/
// main entry of the algorithm (once settings are set)
// loop over everything and merge best scores
'use strict';
extend(FuzzySearch.prototype, /** @lends {FuzzySearch.prototype} */ {
/**
* Perform a search on the already indexed source.
*
* @param {string} query_string
* @returns {Array}
*/
search: function (query_string) {
var time_start = Date.now();
this.start_time = time_start;
var options = this.options;
// As long as lazy is set to false, we guarantee that making a search is read only.
if (this.dirty && options.lazy) {
this._buildIndexFromSource();
this.dirty = false;
}
var query = this.query = this._prepQuery(query_string);
var source = this.index;
var results = [];
if (options.use_index_store) {
source = this._storeSearch(query, source);
}
if (options.filter) {
source = options.filter.call(this, source);
}
// ---- MAIN SEARCH LOOP ---- //
var thresh_include = this._searchIndex(query, source, results);
//keep only results that are good enough compared to best
results = FuzzySearch.filterGTE(results, "score", thresh_include);
// sort by decreasing order of score
// equal rounded score: alphabetical order
if (typeof options.sorter === "function")
results = results.sort(options.sorter);
if (options.output_map || options.output_limit > 0) {
if (typeof options.output_map === "function")
results = FuzzySearch.map(results, options.output_map, this, options.output_limit);
else
results = FuzzySearch.mapField(results, options.output_map, options.output_limit);
}
var time_end = Date.now();
this.search_time = time_end - time_start;
this.results = results;
return results
},
/**
* Main search loop for a specified source
* This separation allow to search a different source, or a subset of source
*
* @param {Query} query
* @param {Array.<Indexed>} source
* @param {Array.<SearchResult>} results
* @returns {number} - thresh_include after this run.
*
* @private
*/
_searchIndex: function (query, source, results) {
var options = this.options;
var opt_bpd = options.bonus_position_decay;
var opt_fge = options.field_good_enough;
var opt_trb = options.thresh_relative_to_best;
var opt_score_tok = options.score_per_token;
var opt_round = options.score_round;
var thresh_include = options.thresh_include;
var best_item_score = 0;
var sub_query = query.children;
for (var item_index = -1, nb_items = source.length; ++item_index < nb_items;) {
//get indexed fields
var item = source[item_index];
var item_fields = item.fields;
//reset score
query.resetItem();
var item_score = 0;
var matched_field_index = -1;
var matched_node_index = -1;
var position_bonus = 1.0;
//
//Foreach field
//
for (var field_index = -1, nb_fields = item_fields.length; ++field_index < nb_fields;) {
var field_score = 0;
var field_node = -1;
var field = item_fields[field_index];
var child_query = sub_query[field_index]; //tag search
var tagged = !!child_query;
for (var node_index = -1, nb_nodes = field.length; ++node_index < nb_nodes;) {
var node_score, node = field[node_index];
if (opt_score_tok) {
node_score = this._scoreField(node, query);
if (tagged) node_score += this._scoreField(node, child_query);//tag search
}
else
node_score = FuzzySearch.score_map(query.fused_str, node.join(" "), query.fused_map, options);
if (node_score > field_score) {
field_score = node_score;
field_node = node_index;
}
}
field_score *= (1.0 + position_bonus);
position_bonus *= opt_bpd;
if (field_score > item_score) {
item_score = field_score;
matched_field_index = field_index;
matched_node_index = field_node;
if (field_score > opt_fge) break;
}
}
//
// Different query token match different fields ?
//
if (opt_score_tok) {
var query_score = query.scoreItem();
item_score = 0.5 * item_score + 0.5 * query_score;
}
//
// Keep track of best result, this control inclusion in the list
//
if (item_score > best_item_score) {
best_item_score = item_score;
var tmp = item_score * opt_trb;
if (tmp > thresh_include) thresh_include = tmp;
}
//
//candidate for best result ? push to list
//
if (item_score > thresh_include) {
item_score = Math.round(item_score / opt_round) * opt_round;
results.push(new SearchResult(
item.item,
item_fields,
item_score,
matched_field_index,
matched_node_index,
item_fields[0][0].join(" ")
));
}
}
return thresh_include
},
/**
* Internal loop that is run for each field in an item
*
* @param {Array} field_tokens
* @param {Query} query
* @returns {number}
* @private
*/
_scoreField: function (field_tokens, query) {
var groups = query.tokens_groups;
var nb_groups = groups.length;
var nb_tokens = field_tokens.length;
if (!nb_groups || !nb_tokens) return 0;
var field_score = 0, sc, bf;
var last_index = -1;
var options = this.options;
var bonus_order = options.bonus_token_order;
var minimum_match = options.minimum_match;
var token, scores, i;
for (var group_index = -1; ++group_index < nb_groups;) {
var group_info = groups[group_index];
var nb_scores = group_info.tokens.length;
// Each packinfo have their own reusable scratch pad
// to store best score information, reset them to 0
var best_of_field = group_info.score_field;
for (i = -1; ++i < nb_scores;) best_of_field[i] = 0
var best_index = group_info.field_pos;
for (i = -1; ++i < nb_scores;) best_index[i] = 0
for (var field_tk_index = -1; ++field_tk_index < nb_tokens;) {
token = field_tokens[field_tk_index];
scores = FuzzySearch.score_pack(group_info, token, options);
for (i = -1; ++i < nb_scores;) {
sc = scores[i];
bf = best_of_field[i];
//Score is an improvement OR
//Score is within a token order bonus from being better, but word are swapped
if (sc > bf || ( bf - sc < bonus_order && i > 0 && best_index[i] <= best_index[i - 1] )) {
best_of_field[i] = sc;
best_index[i] = field_tk_index;
}
}
}
var best_match_this_item = group_info.score_item;
for (i = -1; ++i < nb_scores;) {
sc = best_of_field[i];
field_score += sc;
// Give bonus for pair in consecutive order
// Only consider positive match for bonus
if (sc > minimum_match) {
var this_index = best_index[i];
//Bonus is diluted by the distance between words.
//Positive match, but out of order get half the bonus.
var d = this_index - last_index;
var bo = bonus_order * ( 1.0 / (1.0 + Math.abs(d)));
if (d > 0) bo *= 2;
field_score += bo;
sc += bo;
last_index = this_index;
}
if (sc > best_match_this_item[i])
best_match_this_item[i] = sc;
}
}
if (options.score_test_fused) {
// field_tokens.join(" "), remove last one if acronym
// performance of array.join(" ") and str concat look similar on modern browser.
var n = (options.score_acronym) ? nb_tokens - 1 : nb_tokens;
var fused_field = field_tokens[0], fi = 0