react-mapfilter/es5/selectors/field_analysis.js

A React Component for viewing and filtering GeoJSON

'use strict';

Object.defineProperty(exports, "__esModule", {
  value: true
});

var _typeof2 = require('babel-runtime/helpers/typeof');

var _typeof3 = _interopRequireDefault(_typeof2);

var _stringify = require('babel-runtime/core-js/json/stringify');

var _stringify2 = _interopRequireDefault(_stringify);

var _defineProperty2 = require('babel-runtime/helpers/defineProperty');

var _defineProperty3 = _interopRequireDefault(_defineProperty2);

var _keys = require('babel-runtime/core-js/object/keys');

var _keys2 = _interopRequireDefault(_keys);

var _MAX_DISCRETE_VALUES, _isFilterable, _isMediaField, _isStringOrArray, _isNumberOrArray;

var _reselect = require('reselect');

var _urlRegex = require('url-regex');

var _urlRegex2 = _interopRequireDefault(_urlRegex);

var _url = require('url');

var _url2 = _interopRequireDefault(_url);

var _path = require('path');

var _path2 = _interopRequireDefault(_path);

var _constants = require('../constants');

var _filter_helpers = require('../util/filter_helpers');

var _flattened_features = require('./flattened_features');

var _flattened_features2 = _interopRequireDefault(_flattened_features);

function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }

/**
 * Analyzes the fields of features in a featureCollection and guesses the
 *   field type and what type of filter to use: `discrete`, `number`,
 *   `date` (subtype of continuous), or `text` (and field that has more than
 *   `maxTextValues` discrete values). Number fields with <= `maxNumberCount`
 *   different values are considered discrete.
 * @param {object} featureCollection GeoJson FeatureCollection
 * @return {object} An object with a key for each unique field name in the
 *   FeatureCollection with properties `type` of filter to use, a count for
 *   each discrete option, or a min/max for continuous fields
 */
var getFieldAnalysis = (0, _reselect.createSelector)(_flattened_features2.default, function (state) {
  return state.fieldTypes;
}, function analyzeFields(features, fieldTypes) {
  var props;
  var feature;
  var keys;
  var i;
  var j;
  var value;
  var fieldname;
  var field;
  var geometryType;

  var analysis = {
    properties: {},
    $id: {},
    $type: {}

    // Iterate over every feature in the FeatureCollection
    // This is performance critical, so we use for loops instead of Array.reduce
  };for (i = 0; i < features.length; i++) {
    feature = features[i];
    props = feature.properties || {};
    keys = (0, _keys2.default)(props);
    for (j = 0; j < keys.length; j++) {
      value = props[keys[j]];
      fieldname = keys[j];
      field = analysis.properties[fieldname] = analysis.properties[fieldname] || { fieldname: fieldname };
      analyzeField(field, value, i);
    }
    analyzeField(analysis.$id, feature.id, i);
    geometryType = feature.geometry && feature.geometry.type;
    analyzeField(analysis.$type, geometryType, i);
  }

  for (fieldname in analysis.properties) {
    field = analysis.properties[fieldname];
    field.isUnique = isUnique(field, features.length);
    if (isUUIDField(field)) field.type = _constants.FIELD_TYPE_UUID;
    field.filterType = getFilterType(field);
    if (field.filterType === _constants.FILTER_TYPE_DISCRETE && field.count < features.length) {
      // Add count of undefined values
      field.values[_constants.UNDEFINED_KEY] = (field.values[_constants.UNDEFINED_KEY] || 0) + (features.length - field.count);
    }
    if (field.filterType !== _constants.FILTER_TYPE_DISCRETE) {
      // Free up memory if we're not going to use field.values
      field.values = null;
    } else {
      field.values = parseMapValues(field.values);
    }
    field.type = fieldTypes[fieldname] || field.type;
  }

  analysis.$id.isUnique = isUnique(analysis.$id, features.length);
  analysis.$id.values = null;
  analysis.$type.filterType = getFilterType(analysis.$type);
  analysis.$type.values = parseMapValues(analysis.$type.values);

  return analysis;
});

exports.default = getFieldAnalysis;


var urlRegex = (0, _urlRegex2.default)({ exact: true });

// Max number of unique text values for a field to still be a filterable discrete field
var MAX_DISCRETE_VALUES = (_MAX_DISCRETE_VALUES = {}, (0, _defineProperty3.default)(_MAX_DISCRETE_VALUES, _constants.FIELD_TYPE_STRING, 20), (0, _defineProperty3.default)(_MAX_DISCRETE_VALUES, _constants.FIELD_TYPE_NUMBER, 5), _MAX_DISCRETE_VALUES);

var imageExts = ['jpg', 'tif', 'jpeg', 'png', 'tiff', 'webp'];
var videoExts = ['mov', 'mp4', 'avi', 'webm'];
var audioExts = ['3gpp', 'wav', 'wma', 'mp3', 'm4a', 'aiff', 'ogg'];
var mediaExts = imageExts.concat(videoExts, audioExts);

var types = {
  'string': _constants.FIELD_TYPE_STRING,
  'boolean': _constants.FIELD_TYPE_BOOLEAN,
  'number': _constants.FIELD_TYPE_NUMBER,
  'undefined': _constants.FIELD_TYPE_UNDEFINED
};

var isFilterable = (_isFilterable = {}, (0, _defineProperty3.default)(_isFilterable, _constants.FIELD_TYPE_DATE, true), (0, _defineProperty3.default)(_isFilterable, _constants.FIELD_TYPE_STRING, true), (0, _defineProperty3.default)(_isFilterable, _constants.FIELD_TYPE_NUMBER, true), (0, _defineProperty3.default)(_isFilterable, _constants.FIELD_TYPE_BOOLEAN, true), (0, _defineProperty3.default)(_isFilterable, _constants.FIELD_TYPE_ARRAY, true), (0, _defineProperty3.default)(_isFilterable, _constants.FIELD_TYPE_STRING_OR_ARRAY, true), (0, _defineProperty3.default)(_isFilterable, _constants.FIELD_TYPE_NUMBER_OR_ARRAY, true), _isFilterable);

var isMediaField = (_isMediaField = {}, (0, _defineProperty3.default)(_isMediaField, _constants.FIELD_TYPE_VIDEO, true), (0, _defineProperty3.default)(_isMediaField, _constants.FIELD_TYPE_IMAGE, true), (0, _defineProperty3.default)(_isMediaField, _constants.FIELD_TYPE_MEDIA, true), _isMediaField);

var isStringOrArray = (_isStringOrArray = {}, (0, _defineProperty3.default)(_isStringOrArray, _constants.FIELD_TYPE_STRING, true), (0, _defineProperty3.default)(_isStringOrArray, _constants.FIELD_TYPE_ARRAY, true), (0, _defineProperty3.default)(_isStringOrArray, _constants.FIELD_TYPE_STRING_OR_ARRAY, true), _isStringOrArray);

var isNumberOrArray = (_isNumberOrArray = {}, (0, _defineProperty3.default)(_isNumberOrArray, _constants.FIELD_TYPE_NUMBER, true), (0, _defineProperty3.default)(_isNumberOrArray, _constants.FIELD_TYPE_ARRAY, true), (0, _defineProperty3.default)(_isNumberOrArray, _constants.FIELD_TYPE_NUMBER_OR_ARRAY, true), _isNumberOrArray);

function analyzeField(field, value, i) {
  var type = getType(value);
  field.type = typeReduce(field.type, type);
  field.count = (field.count || 0) + 1;
  if (!isFilterable[field.type]) return;
  if (type === _constants.FIELD_TYPE_STRING) {
    field.wordStats = statReduce(field.wordStats, wc(value), i);
    field.lengthStats = statReduce(field.lengthStats, value.length, i);
  } else if (type === _constants.FIELD_TYPE_ARRAY) {
    field.maxArrayLength = Math.max(field.maxArrayLength || 1, value.length);
  } else if (type === _constants.FIELD_TYPE_NUMBER) {
    field.valueStats = statReduce(field.valueStats, value, i);
  } else if (type === _constants.FIELD_TYPE_DATE) {
    field.valueStats = statReduce(field.valueStats, (0, _filter_helpers.parseDate)(value), i);
  }
  field.values = valuesReduce(field.values, value);
}

function wc(s) {
  return s.split(' ').length;
}

/**
 * Reducer that computes running mean, variance, min and max
 * Adapted from http://www.johndcook.com/blog/standard_deviation/
 * @param {Object} p The previous value for the analysis
 * @param {Number} x New value to be included in analysis
 * @param {Number} i Index of the current element being processed
 * @return {Object} New analysis including `x`
 */
function statReduce() {
  var p = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : { mean: NaN, vari: NaN, min: +Infinity, max: -Infinity };
  var x = arguments[1];
  var i = arguments[2];

  p.mean = isNaN(p.mean) ? 0 : p.mean;
  var mean = p.mean + (x - p.mean) / (i + 1);
  x = x instanceof Date ? +x : x;
  return {
    mean: mean,
    min: x < p.min ? x : p.min,
    max: x > p.max ? x : p.max,
    vari: i < 1 ? 0 : (p.vari * i + (x - p.mean) * (x - mean)) / (i + 1)
  };
}

/**
 * Reducer that returns 'mixed' if values are not all the same,
 * or 'media' if field is a mixture of image and video files
 * @param {any} p Previous value
 * @param {any} v Current value
 * @return {any} `v` or `mixed`
 */
function typeReduce(p, v) {
  if (!p || v === p) return v;
  if (isMediaField[p] && isMediaField[v]) {
    return _constants.FIELD_TYPE_MEDIA;
  } else if (isMediaField[p] && (v === _constants.FIELD_TYPE_LINK || v === _constants.FIELD_TYPE_UNDEFINED || v === _constants.FIELD_TYPE_NULL || v === _constants.FIELD_TYPE_FILENAME)) {
    // If this contains media + links, assume the links are to the same type of media
    // media field might also have a string filename if image is lost
    return p;
  } else if (isMediaField[v] && (p === _constants.FIELD_TYPE_LINK || p === _constants.FIELD_TYPE_UNDEFINED || p === _constants.FIELD_TYPE_NULL || p === _constants.FIELD_TYPE_FILENAME)) {
    // If this contains media + links, assume the links are to the same type of media
    return v;
  } else if (p === _constants.FIELD_TYPE_LINK && (isMediaField[v] || p === _constants.FIELD_TYPE_UNDEFINED || p === _constants.FIELD_TYPE_NULL)) {
    return v;
  } else if (isStringOrArray[p] && isStringOrArray[v]) {
    return _constants.FIELD_TYPE_STRING_OR_ARRAY;
  } else if (isNumberOrArray[p] && isNumberOrArray[v]) {
    return _constants.FIELD_TYPE_NUMBER_OR_ARRAY;
  } else {
    return _constants.FIELD_TYPE_MIXED;
  }
}

/**
 * Reducer that returns the count of each value for a field
 * @param {Object} p Accumulator
 * @param {Any} v Any value
 * @return {Object} An object with a key for each value and the count of that value
 */
function valuesReduce() {
  var p = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};
  var v = arguments[1];

  v = Array.isArray(v) ? v : [v];
  v.forEach(function (w) {
    w = (0, _stringify2.default)(w);
    p[w] = typeof p[w] === 'undefined' ? 1 : p[w] + 1;
  });
  return p;
}

/**
 * Return the filter type for a field `f`.
 * @param {Object} f A field object with analysis props
 * @return {String} Filter type: date, range, discrete or text
 */
function getFilterType(f) {
  var keyCount = f.values && (0, _keys2.default)(f.values).length;
  if (!isFilterable[f.type]) return;
  switch (f.type) {
    case _constants.FIELD_TYPE_DATE:
      return _constants.FILTER_TYPE_DATE;
    case _constants.FIELD_TYPE_NUMBER:
      if (keyCount <= MAX_DISCRETE_VALUES[_constants.FIELD_TYPE_NUMBER]) {
        return _constants.FILTER_TYPE_DISCRETE;
      } else {
        return _constants.FILTER_TYPE_RANGE;
      }
    case _constants.FIELD_TYPE_BOOLEAN:
      return _constants.FILTER_TYPE_DISCRETE;
    case _constants.FIELD_TYPE_STRING:
      // Strings with lots of words we count as text fields, not discrete fields
      if (f.wordStats.mean > 3) {
        return _constants.FILTER_TYPE_TEXT;
      }
    // eslint-disable-next-line no-fallthrough
    case _constants.FIELD_TYPE_ARRAY:
    case _constants.FIELD_TYPE_STRING_OR_ARRAY:
    case _constants.FIELD_TYPE_NUMBER_OR_ARRAY:
      if (keyCount <= MAX_DISCRETE_VALUES[_constants.FIELD_TYPE_STRING]) {
        return _constants.FILTER_TYPE_DISCRETE;
      } else {
        return _constants.FILTER_TYPE_TEXT;
      }
  }
}

/**
 * Returns the type of a value, guessing types `date`, `link`, `image`, `video`
 * @param {any} v Value to be evaluated
 * @return {string} Field type
 */
function getType(v) {
  if (Array.isArray(v)) return _constants.FIELD_TYPE_ARRAY;
  if (v === null) return _constants.FIELD_TYPE_NULL;
  if (typeof v !== 'string') return types[typeof v === 'undefined' ? 'undefined' : (0, _typeof3.default)(v)];
  // isDate() is the most expensive test, so we do it as little as possible
  if ((0, _filter_helpers.isDate)(v)) return _constants.FIELD_TYPE_DATE;
  if (urlRegex.test(v)) {
    var pathname = _url2.default.parse(v).pathname;
    var _ext = _path2.default.extname(pathname).slice(1);
    if (imageExts.indexOf(_ext) > -1) return _constants.FIELD_TYPE_IMAGE;
    if (videoExts.indexOf(_ext) > -1) return _constants.FIELD_TYPE_VIDEO;
    if (audioExts.indexOf(_ext) > -1) return _constants.FIELD_TYPE_AUDIO;
    return _constants.FIELD_TYPE_LINK;
  }
  var ext = _path2.default.extname(v).slice(1);
  if (v.split('/').length === 1 && mediaExts.indexOf(ext) > -1) return _constants.FIELD_TYPE_FILENAME;
  return _constants.FIELD_TYPE_STRING;
}

/**
 * We initially process values as a map value -> count,
 * with the value encoded with JSON.stringify() so that
 * it is not coerced to a string when set as an object property.
 * This function parses those values back to their original
 * type and turns the values map into a sorted array
 */
function parseMapValues(values) {
  if (!values) return [];
  return (0, _keys2.default)(values).sort().map(function (v) {
    var parsed;
    if (v === _constants.UNDEFINED_KEY) {
      parsed = v;
    } else {
      try {
        parsed = JSON.parse(v);
      } catch (e) {
        console.error('problem parsing', v);
      }
    }
    return { value: parsed, count: values[v] };
  });
}

/**
 * Is a field unique?
 * @param {object} field        A field object with analysis props
 * @param {number} featureCount Total number of features
 * @return {Boolean}
 */
function isUnique(field, featureCount) {
  var valueCount = field.values && (0, _keys2.default)(field.values).length;
  return featureCount === valueCount;
}

/**
 * Guess if a field is a UUID: if it is unique, has a length greater than
 * 30, no variance in length, and is only one word.
 */
function isUUIDField(f) {
  if (!f.isUnique) return;
  if (f.type !== _constants.FIELD_TYPE_STRING) return;
  return f.lengthStats.mean > 30 && f.lengthStats.vari === 0 && f.wordStats.max === 1;
}
//# sourceMappingURL=field_analysis.js.map