vega-crossfilter/src/CrossFilter.js

Indexed cross-filtering for Vega dataflows.

import Bitmaps from './Bitmaps.js';
import Dimension from './Dimension.js';
import SortedIndex from './SortedIndex.js';
import {Transform} from 'vega-dataflow';
import {inherits} from 'vega-util';

/**
 * An indexed multi-dimensional filter.
 * @constructor
 * @param {object} params - The parameters for this operator.
 * @param {Array<function(object): *>} params.fields - An array of dimension accessors to filter.
 * @param {Array} params.query - An array of per-dimension range queries.
 */
export default function CrossFilter(params) {
  Transform.call(this, Bitmaps(), params);
  this._indices = null;
  this._dims = null;
}

CrossFilter.Definition = {
  'type': 'CrossFilter',
  'metadata': {},
  'params': [
    { 'name': 'fields', 'type': 'field', 'array': true, 'required': true },
    { 'name': 'query', 'type': 'array', 'array': true, 'required': true,
      'content': {'type': 'number', 'array': true, 'length': 2} }
  ]
};

inherits(CrossFilter, Transform, {
  transform(_, pulse) {
    if (!this._dims) {
      return this.init(_, pulse);
    } else {
      var init = _.modified('fields')
            || _.fields.some(f => pulse.modified(f.fields));

      return init
        ? this.reinit(_, pulse)
        : this.eval(_, pulse);
    }
  },

  init(_, pulse) {
    const fields = _.fields,
          query = _.query,
          indices = this._indices = {},
          dims = this._dims = [],
          m = query.length;
    let i = 0, key, index;

    // instantiate indices and dimensions
    for (; i<m; ++i) {
      key = fields[i].fname;
      index = indices[key] || (indices[key] = SortedIndex());
      dims.push(Dimension(index, i, query[i]));
    }

    return this.eval(_, pulse);
  },

  reinit(_, pulse) {
    const output = pulse.materialize().fork(),
          fields = _.fields,
          query = _.query,
          indices = this._indices,
          dims = this._dims,
          bits = this.value,
          curr = bits.curr(),
          prev = bits.prev(),
          all = bits.all(),
          out = (output.rem = output.add),
          mod = output.mod,
          m = query.length,
          adds = {};
    let add, index, key, mods, remMap, modMap, i, n, f;

    // set prev to current state
    prev.set(curr);

    // if pulse has remove tuples, process them first
    if (pulse.rem.length) {
      remMap = this.remove(_, pulse, output);
    }

    // if pulse has added tuples, add them to state
    if (pulse.add.length) {
      bits.add(pulse.add);
    }

    // if pulse has modified tuples, create an index map
    if (pulse.mod.length) {
      modMap = {};
      for (mods=pulse.mod, i=0, n=mods.length; i<n; ++i) {
        modMap[mods[i]._index] = 1;
      }
    }

    // re-initialize indices as needed, update curr bitmap
    for (i=0; i<m; ++i) {
      f = fields[i];
      if (!dims[i] || _.modified('fields', i) || pulse.modified(f.fields)) {
        key = f.fname;
        if (!(add = adds[key])) {
          indices[key] = index = SortedIndex();
          adds[key] = add = index.insert(f, pulse.source, 0);
        }
        dims[i] = Dimension(index, i, query[i]).onAdd(add, curr);
      }
    }

    // visit each tuple
    // if filter state changed, push index to add/rem
    // else if in mod and passes a filter, push index to mod
    for (i=0, n=bits.data().length; i<n; ++i) {
      if (remMap[i]) { // skip if removed tuple
        continue;
      } else if (prev[i] !== curr[i]) { // add if state changed
        out.push(i);
      } else if (modMap[i] && curr[i] !== all) { // otherwise, pass mods through
        mod.push(i);
      }
    }

    bits.mask = (1 << m) - 1;
    return output;
  },

  eval(_, pulse) {
    const output = pulse.materialize().fork(),
          m = this._dims.length;
    let mask = 0;

    if (pulse.rem.length) {
      this.remove(_, pulse, output);
      mask |= (1 << m) - 1;
    }

    if (_.modified('query') && !_.modified('fields')) {
      mask |= this.update(_, pulse, output);
    }

    if (pulse.add.length) {
      this.insert(_, pulse, output);
      mask |= (1 << m) - 1;
    }

    if (pulse.mod.length) {
      this.modify(pulse, output);
      mask |= (1 << m) - 1;
    }

    this.value.mask = mask;
    return output;
  },

  insert(_, pulse, output) {
    const tuples = pulse.add,
          bits = this.value,
          dims = this._dims,
          indices = this._indices,
          fields = _.fields,
          adds = {},
          out = output.add,
          n = bits.size() + tuples.length,
          m = dims.length;
    let k = bits.size(), j, key, add;

    // resize bitmaps and add tuples as needed
    bits.resize(n, m);
    bits.add(tuples);

    const curr = bits.curr(),
          prev = bits.prev(),
          all  = bits.all();

    // add to dimensional indices
    for (j=0; j<m; ++j) {
      key = fields[j].fname;
      add = adds[key] || (adds[key] = indices[key].insert(fields[j], tuples, k));
      dims[j].onAdd(add, curr);
    }

    // set previous filters, output if passes at least one filter
    for (; k < n; ++k) {
      prev[k] = all;
      if (curr[k] !== all) out.push(k);
    }
  },

  modify(pulse, output) {
    const out = output.mod,
          bits = this.value,
          curr = bits.curr(),
          all  = bits.all(),
          tuples = pulse.mod;
    let i, n, k;

    for (i=0, n=tuples.length; i<n; ++i) {
      k = tuples[i]._index;
      if (curr[k] !== all) out.push(k);
    }
  },

  remove(_, pulse, output) {
    const indices = this._indices,
          bits = this.value,
          curr = bits.curr(),
          prev = bits.prev(),
          all  = bits.all(),
          map = {},
          out = output.rem,
          tuples = pulse.rem;
    let i, n, k, f;

    // process tuples, output if passes at least one filter
    for (i=0, n=tuples.length; i<n; ++i) {
      k = tuples[i]._index;
      map[k] = 1; // build index map
      prev[k] = (f = curr[k]);
      curr[k] = all;
      if (f !== all) out.push(k);
    }

    // remove from dimensional indices
    for (k in indices) {
      indices[k].remove(n, map);
    }

    this.reindex(pulse, n, map);
    return map;
  },

  // reindex filters and indices after propagation completes
  reindex(pulse, num, map) {
    const indices = this._indices,
          bits = this.value;

    pulse.runAfter(() => {
      const indexMap = bits.remove(num, map);
      for (const key in indices) indices[key].reindex(indexMap);
    });
  },

  update(_, pulse, output) {
    const dims = this._dims,
          query = _.query,
          stamp = pulse.stamp,
          m = dims.length;
    let mask = 0, i, q;

    // survey how many queries have changed
    output.filters = 0;
    for (q=0; q<m; ++q) {
      if (_.modified('query', q)) { i = q; ++mask; }
    }

    if (mask === 1) {
      // only one query changed, use more efficient update
      mask = dims[i].one;
      this.incrementOne(dims[i], query[i], output.add, output.rem);
    } else {
      // multiple queries changed, perform full record keeping
      for (q=0, mask=0; q<m; ++q) {
        if (!_.modified('query', q)) continue;
        mask |= dims[q].one;
        this.incrementAll(dims[q], query[q], stamp, output.add);
        output.rem = output.add; // duplicate add/rem for downstream resolve
      }
    }

    return mask;
  },

  incrementAll(dim, query, stamp, out) {
    const bits = this.value,
          seen = bits.seen(),
          curr = bits.curr(),
          prev = bits.prev(),
          index = dim.index(),
          old = dim.bisect(dim.range),
          range = dim.bisect(query),
          lo1 = range[0],
          hi1 = range[1],
          lo0 = old[0],
          hi0 = old[1],
          one = dim.one;
    let i, j, k;

    // Fast incremental update based on previous lo index.
    if (lo1 < lo0) {
      for (i = lo1, j = Math.min(lo0, hi1); i < j; ++i) {
        k = index[i];
        if (seen[k] !== stamp) {
          prev[k] = curr[k];
          seen[k] = stamp;
          out.push(k);
        }
        curr[k] ^= one;
      }
    } else if (lo1 > lo0) {
      for (i = lo0, j = Math.min(lo1, hi0); i < j; ++i) {
        k = index[i];
        if (seen[k] !== stamp) {
          prev[k] = curr[k];
          seen[k] = stamp;
          out.push(k);
        }
        curr[k] ^= one;
      }
    }

    // Fast incremental update based on previous hi index.
    if (hi1 > hi0) {
      for (i = Math.max(lo1, hi0), j = hi1; i < j; ++i) {
        k = index[i];
        if (seen[k] !== stamp) {
          prev[k] = curr[k];
          seen[k] = stamp;
          out.push(k);
        }
        curr[k] ^= one;
      }
    } else if (hi1 < hi0) {
      for (i = Math.max(lo0, hi1), j = hi0; i < j; ++i) {
        k = index[i];
        if (seen[k] !== stamp) {
          prev[k] = curr[k];
          seen[k] = stamp;
          out.push(k);
        }
        curr[k] ^= one;
      }
    }

    dim.range = query.slice();
  },

  incrementOne(dim, query, add, rem) {
    const bits = this.value,
          curr = bits.curr(),
          index = dim.index(),
          old = dim.bisect(dim.range),
          range = dim.bisect(query),
          lo1 = range[0],
          hi1 = range[1],
          lo0 = old[0],
          hi0 = old[1],
          one = dim.one;
    let i, j, k;

    // Fast incremental update based on previous lo index.
    if (lo1 < lo0) {
      for (i = lo1, j = Math.min(lo0, hi1); i < j; ++i) {
        k = index[i];
        curr[k] ^= one;
        add.push(k);
      }
    } else if (lo1 > lo0) {
      for (i = lo0, j = Math.min(lo1, hi0); i < j; ++i) {
        k = index[i];
        curr[k] ^= one;
        rem.push(k);
      }
    }

    // Fast incremental update based on previous hi index.
    if (hi1 > hi0) {
      for (i = Math.max(lo1, hi0), j = hi1; i < j; ++i) {
        k = index[i];
        curr[k] ^= one;
        add.push(k);
      }
    } else if (hi1 < hi0) {
      for (i = Math.max(lo0, hi1), j = hi0; i < j; ++i) {
        k = index[i];
        curr[k] ^= one;
        rem.push(k);
      }
    }

    dim.range = query.slice();
  }
});