itemsjs/lib/helpers.js

Created to perform fast search on small json dataset (up to 1000 elements).

"use strict";

var _ = require('./../vendor/lodash');

var FastBitSet = require('fastbitset');

var clone = function clone(val) {
  try {
    return JSON.parse(JSON.stringify(val));
  } catch (e) {
    return val;
  }
};

var humanize = function humanize(str) {
  return str.replace(/^[\s_]+|[\s_]+$/g, '').replace(/[_\s]+/g, ' ').replace(/^[a-z]/, function (m) {
    return m.toUpperCase();
  });
};

var combination_indexes = function combination_indexes(facets, filters) {
  var indexes = {};

  _.mapValues(filters, function (filter) {
    // filter is still array so disjunctive
    if (Array.isArray(filter[0])) {
      var facet_union = new FastBitSet([]);
      var filter_keys = [];

      _.mapValues(filter, function (disjunctive_filter) {
        var filter_key = disjunctive_filter[0];
        var filter_val = disjunctive_filter[1];
        filter_keys.push(filter_key);
        facet_union = facet_union.new_union(facets['bits_data'][filter_key][filter_val]);
        indexes[filter_key] = facet_union;
      });
    }
  });

  return indexes;
};
/*
 * returns facets and ids
 */


var matrix = function matrix(facets, filters) {
  var temp_facet = _.clone(facets);

  filters = filters || [];

  _.mapValues(temp_facet['bits_data'], function (values, key) {
    _.mapValues(temp_facet['bits_data'][key], function (facet_indexes, key2) {
      temp_facet['bits_data_temp'][key][key2] = temp_facet['bits_data'][key][key2];
    });
  });

  var conjunctive_index;
  var disjunctive_indexes = combination_indexes(facets, filters);
  /**
   * process only conjunctive filters
   */

  _.mapValues(filters, function (filter) {
    if (!Array.isArray(filter[0])) {
      var filter_key = filter[0];
      var filter_val = filter[1];

      if (conjunctive_index && temp_facet['bits_data_temp'][filter_key][filter_val]) {
        conjunctive_index = temp_facet['bits_data_temp'][filter_key][filter_val].new_intersection(conjunctive_index);
      } else if (conjunctive_index && !temp_facet['bits_data_temp'][filter_key][filter_val]) {
        conjunctive_index = new FastBitSet([]);
      } else {
        conjunctive_index = temp_facet['bits_data_temp'][filter_key][filter_val];
      }
    }
  }); // cross all facets with conjunctive index


  if (conjunctive_index) {
    _.mapValues(temp_facet['bits_data_temp'], function (values, key) {
      _.mapValues(temp_facet['bits_data_temp'][key], function (facet_indexes, key2) {
        temp_facet['bits_data_temp'][key][key2] = temp_facet['bits_data_temp'][key][key2].new_intersection(conjunctive_index);
      });
    });
  } // cross all combination indexes with conjunctive index

  /*if (conjunctive_index) {
    _.mapValues(disjunctive_indexes, function(disjunctive_index, disjunctive_key) {
      disjunctive_indexes[disjunctive_key] = conjunctive_index.new_intersection(disjunctive_indexes[disjunctive_key]);
    });
  }*/

  /**
   * process only negative filters
   */


  _.mapValues(filters, function (filter) {
    if (filter.length === 3 && filter[1] === '-') {
      var filter_key = filter[0];
      var filter_val = filter[2];
      var negative_bits = temp_facet['bits_data_temp'][filter_key][filter_val].clone();

      _.mapValues(temp_facet['bits_data_temp'], function (values, key) {
        _.mapValues(temp_facet['bits_data_temp'][key], function (facet_indexes, key2) {
          temp_facet['bits_data_temp'][key][key2] = temp_facet['bits_data_temp'][key][key2].new_difference(negative_bits);
        });
      });
    }
  }); // cross all facets with disjunctive index


  _.mapValues(temp_facet['bits_data_temp'], function (values, key) {
    _.mapValues(temp_facet['bits_data_temp'][key], function (facet_indexes, key2) {
      _.mapValues(disjunctive_indexes, function (disjunctive_index, disjunctive_key) {
        if (disjunctive_key !== key) {
          temp_facet['bits_data_temp'][key][key2] = temp_facet['bits_data_temp'][key][key2].new_intersection(disjunctive_index);
        }
      });
    });
  });

  return temp_facet;
};

var index = function index(items, fields) {
  fields = fields || [];
  var facets = {
    data: {},
    bits_data: {},
    bits_data_temp: {}
  };
  var i = 1;
  items = _.map(items, function (item) {
    if (!item['_id']) {
      item['_id'] = i;
      ++i;
    }

    return item;
  }); // replace chain with forEach

  _.chain(items).map(function (item) {
    fields.forEach(function (field) {
      //if (!item || !item[field]) {
      if (!item) {
        return;
      }

      if (!facets['data'][field]) {
        facets['data'][field] = {};
      }

      if (Array.isArray(item[field])) {
        item[field].forEach(function (v) {
          if (!item[field]) {
            return;
          }

          if (!facets['data'][field][v]) {
            facets['data'][field][v] = [];
          }

          facets['data'][field][v].push(parseInt(item._id));
        });
      } else if (typeof item[field] !== 'undefined') {
        var v = item[field];

        if (!facets['data'][field][v]) {
          facets['data'][field][v] = [];
        }

        facets['data'][field][v].push(parseInt(item._id));
      }
    });
    return item;
  }).value();

  facets['data'] = _.mapValues(facets['data'], function (values, field) {
    if (!facets['bits_data'][field]) {
      facets['bits_data'][field] = {};
      facets['bits_data_temp'][field] = {};
    } //console.log(values);


    return _.mapValues(values, function (indexes, filter) {
      var sorted_indexes = _.sortBy(indexes);

      facets['bits_data'][field][filter] = new FastBitSet(sorted_indexes);
      return sorted_indexes;
    });
  });
  return facets;
};
/**
 * calculates ids for facets
 * if there is no facet input then return null to not save resources for OR calculation
 * null means facets haven't crossed searched items
 */


var facets_ids = function facets_ids(facets_data, filters) {
  var output = new FastBitSet([]);
  var i = 0;

  _.mapValues(filters, function (filters, field) {
    //console.log(facets_data);
    filters.forEach(function (filter) {
      ++i;
      output = output.new_union(facets_data[field][filter]);
    });
  });

  if (i === 0) {
    return null;
  }

  return output;
};

var getBuckets = function getBuckets(data, input, aggregations) {
  var position = 1;
  return _.mapValues(data['bits_data_temp'], function (v, k) {
    var order;
    var sort;
    var size;
    var title;

    if (aggregations[k]) {
      order = aggregations[k].order;
      sort = aggregations[k].sort;
      size = aggregations[k].size;
      title = aggregations[k].title;
    }

    var buckets = _.chain(v).toPairs().map(function (v2) {
      var filters = [];

      if (input && input.filters && input.filters[k]) {
        filters = input.filters[k];
      }

      return {
        key: v2[0],
        doc_count: v2[1].array().length,
        selected: filters.indexOf(v2[0]) !== -1
      };
    }).value();

    if (sort === 'term') {
      buckets = _.orderBy(buckets, ['selected', 'key'], ['desc', order || 'asc']);
    } else {
      buckets = _.orderBy(buckets, ['selected', 'doc_count', 'key'], ['desc', order || 'desc', 'asc']);
    }

    buckets = buckets.slice(0, size || 10);
    return {
      name: k,
      title: title || humanize(k),
      position: position++,
      buckets: buckets
    };
  });
};

var mergeAggregations = function mergeAggregations(aggregations, input) {
  return _.mapValues(clone(aggregations), function (val, key) {
    if (!val.field) {
      val.field = key;
    }

    var filters = [];

    if (input.filters && input.filters[key]) {
      filters = input.filters[key];
    }

    val.filters = filters;
    var not_filters = [];

    if (input.not_filters && input.not_filters[key]) {
      not_filters = input.not_filters[key];
    }

    if (input.exclude_filters && input.exclude_filters[key]) {
      not_filters = input.exclude_filters[key];
    }

    val.not_filters = not_filters;
    return val;
  });
};

var input_to_facet_filters = function input_to_facet_filters(input, config) {
  var filters = [];

  _.mapValues(input.filters, function (values, key) {
    if (values && values.length) {
      if (config[key].conjunction !== false) {
        _.mapValues(values, function (values2) {
          filters.push([key, values2]);
        });
      } else {
        var temp = [];

        _.mapValues(values, function (values2) {
          temp.push([key, values2]);
        });

        filters.push(temp);
      }
    }
  });

  _.mapValues(input.not_filters, function (values, key) {
    if (values && values.length) {
      _.mapValues(values, function (values2) {
        filters.push([key, '-', values2]);
      });
    }
  });

  return filters;
};

module.exports.input_to_facet_filters = input_to_facet_filters;
module.exports.facets_ids = facets_ids;
module.exports.clone = clone;
module.exports.humanize = humanize;
module.exports.index = index;
module.exports.combination_indexes = combination_indexes;
module.exports.matrix = matrix;
module.exports.getBuckets = getBuckets;
module.exports.getFacets = getBuckets;
module.exports.mergeAggregations = mergeAggregations;