alinex-fs/lib/helper/filter.js

Extension of nodes filesystem tools.

/*
Filter Rules
=================================================
The filter is used to select some of the files based on specific settings.
You can't call the filter directly but it is used from most methods for file selection.

#4 Options

The filter definition is given as options array which may have some of the following
specification settings. But some methods may have special additional options not mentioned here.

The filter is given as `filter` element in the options object and be specified as subobject
or list of objects with these settings:
- `include` - `Array<String|RegExp>|String|RegExp` to specify a inclusion pattern
- `exclude` - `Array<String|RegExp>|String|RegExp` to specify an exclusion pattern
- `mindepth` - `Integer` minimal depth to match
- `maxdepth` - `Integer` maximal depth to match
- `dereference` - `Boolean` set to true to follow symbolic links
- `type` - `String` the inode type it should be one of:
  - `file`, `f`
  - `directory`, `dir`, `d`
  - `link`, `l`
  - `fifo`, `pipe`, `p`
  - `socket`, `s`
- `minsize` - `Integer|String` minimal filesize
- `maxsize` - `Integer|String` maximal filesize
- `user` - `Integer|String` owner name or id
- `group` - `Integer|String` owner group name or id
- `accessedAfter` - `Integer|String` last access time should be after that value
- `accessedBefore` - `Integer|String` last access time should be before that value
- `modifiedAfter` - `Integer|String` last modified time should be after that value
- `modifiedBefore` - `Integer|String` last modified time should be before that value
- `createdAfter` - `Integer|String` creation time should be after that value
- `createdBefore` - `Integer|String` creation time should be before that value
- `test` - `Function` own function to use

If you use multiple options all of them have to match the file to be valid.
See the details below.

#4 Multiple Option Sets

Multiple sets of the above rules can also be given as list of option arrays. If so
all files are allowed, which match any of the given option sets.
 */

(function() {
  var async, chrono, debug, filestat, filestatSync, fs, path, posix, sizeHumanToInt, skipDepth, skipDepthSync, skipFunction, skipFunctionSync, skipOwner, skipOwnerSync, skipPath, skipPathSync, skipSize, skipSizeSync, skipTime, skipTimeSync, skipType, skipTypeSync, timeCheck, util;

  debug = require('debug')('fs:filter');

  fs = require('fs');

  path = require('path');

  async = require('async');

  chrono = require('chrono-node');

  util = require('util');

  posix = require('posix');

  module.exports.filter = function(file, depth, options, cb) {
    var list, subpath;
    if (depth == null) {
      depth = 0;
    }
    if (options == null) {
      options = {};
    }
    if (cb == null) {
      cb = function() {};
    }
    if ((options != null ? options.filter : void 0) == null) {
      return cb(true);
    }
    list = Array.isArray(options.filter) ? options.filter : [options.filter];
    subpath = file.split(/\//);
    subpath = subpath.slice(subpath.length - depth).join('/');
    if (!subpath.length) {
      subpath = null;
    }
    return async.map(list, function(options, cb) {
      if (!Object.keys(options).length) {
        return cb(true);
      }
      return skipPath(subpath != null ? subpath : file, options, function(skip) {
        if (skip) {
          if (skip === 'SKIPPATH') {
            return cb(skip);
          }
          return cb(false);
        }
        return async.parallel([
          function(cb) {
            return skipDepth(file, depth, options, cb);
          }, function(cb) {
            return skipType(file, options, cb);
          }, function(cb) {
            return skipSize(file, options, cb);
          }, function(cb) {
            return skipTime(file, options, cb);
          }, function(cb) {
            return skipOwner(file, options, cb);
          }, function(cb) {
            return skipFunction(file, options, cb);
          }
        ], function(skip) {
          return cb(!skip);
        });
      });
    }, function(err) {
      if (err === 'SKIPPATH') {
        debug(file + " SKIPPATH");
        return cb();
      }
      debug(file + " " + (err ? 'OK' : 'SKIP'));
      return cb(err ? true : false);
    });
  };

  module.exports.filterSync = function(file, depth, options) {
    var i, len, list, res, subpath;
    if (depth == null) {
      depth = 0;
    }
    if ((options != null ? options.filter : void 0) == null) {
      return true;
    }
    list = Array.isArray(options.filter) ? options.filter : [options.filter];
    subpath = file.split(/\//);
    subpath = subpath.slice(subpath.length - depth).join('/');
    if (!subpath.length) {
      subpath = null;
    }
    for (i = 0, len = list.length; i < len; i++) {
      options = list[i];
      if (!Object.keys(options).length) {
        return true;
      }
      debug(("check " + file + " for ") + util.inspect(options));
      if (res = skipPathSync(subpath != null ? subpath : file, options)) {
        if (res === 'SKIPPATH') {
          debug(file + " SKIP");
          return void 0;
        }
        continue;
      }
      if (skipTypeSync(file, options)) {
        continue;
      }
      if (skipDepthSync(file, depth, options)) {
        continue;
      }
      if (skipSizeSync(file, options)) {
        continue;
      }
      if (skipTimeSync(file, options)) {
        continue;
      }
      if (skipOwnerSync(file, options)) {
        continue;
      }
      if (skipFunctionSync(file, options)) {
        continue;
      }
      debug(file + " OK");
      return true;
    }
    debug(file + " SKIP");
    return false;
  };


  /*
  #3 File/Path Matching
  
  This is based on glob expressions like used in unix systems. You may use these
  as the `include` or `exclude` pattern while the `exclude` pattern has the higher
  priority. All files are matched which are in the include pattern and not in the
  exclude pattern.
  
  Both patterns may also be given as `Array` to match multiple. If multiple are given
  they are combined logically using OR meaning that at least one include have to match
  and no exclude should match.
  
  __Regular expressions__
  
  The pattern may be a regular expression given as String. See {@link RegExp()}
  for the format description.
  
  __Pattern Matching__
  
  Alternatively you may use glob pattern string with the following specification:
  
  To use one of the special characters `*`, `?` or `[` you have to preceed it with an
  backslash.
  
  The patter may contain:
  
  - `?` (not between brackets) matches any single character.
  - `*` (not between brackets) matches any string, including the empty string.
  - `**` (not between brackets) matches any string and also includes the path separator.
  
  Character groups:
  
  - `[ade]` or `[a-z]` Matches any one of the enclosed characters ranges can be given using a hyphen.
  - `[!ade]` or `[!a-z]` negates the search and matches any character not enclosed.
  - `[^ade]` or `[^a-z]` negates the search and matches any character not enclosed.
  
  Brace Expansion:
  
  - `{a,b}` will be expanded to `a` or `b`
  - `{a,b{c,d}}` stacked to match `a`, `bc` or `bd`
  - `{1..3}` will be expanded to `1` or `2` or `3`
  
  Extended globbing is also possible:
  
  - ?(list): Matches zero or one occurrence of the given patterns.
  - *(list): Matches zero or more occurrences of the given patterns.
  - +(list): Matches one or more occurrences of the given patterns.
  - @(list): Matches one of the given patterns.
  
  See more information about pattern matching in {@link minimatch}.
  
  __Example__
  
  ``` coffee
  fs = require 'alinex-fs'
  fs.find '/tmp/some/directory',
    filter:
      include: 'a*'
      exclude: '*c'
  , (err, list) ->
     * list may include 'a', 'abd', 'abe'
     * but not 'abc'
  ```
   */

  skipPath = function(file, options, cb) {
    return cb(skipPathSync(file, options));
  };

  skipPathSync = function(file, options) {
    var exclude, i, include, j, len, len1, list, minimatch, ok;
    if (!(options.include || options.exclude)) {
      return false;
    }
    if (options.exclude) {
      list = Array.isArray(options.exclude) ? options.exclude : [options.exclude];
      for (i = 0, len = list.length; i < len; i++) {
        exclude = list[i];
        if (exclude instanceof RegExp) {
          if (file.match(exclude)) {
            debug("skip " + file + " because path excluded (regexp)");
            return 'SKIPPATH';
          }
        } else if (exclude === path.basename(file)) {
          return true;
        } else {
          minimatch = require('minimatch');
          if (minimatch(file, exclude, {
            matchBase: true
          })) {
            debug("skip " + file + " because path excluded (glob)");
            return 'SKIPPATH';
          }
        }
      }
    }
    if (options.include) {
      list = Array.isArray(options.include) ? options.include : [options.include];
      ok = false;
      for (j = 0, len1 = list.length; j < len1; j++) {
        include = list[j];
        if (include instanceof RegExp) {
          if (file.match(include)) {
            ok = true;
            break;
          }
        } else if (include !== path.basename(file)) {
          minimatch = require('minimatch');
          if (minimatch(file, include, {
            matchBase: true
          })) {
            ok = true;
            break;
          }
        } else {
          ok = true;
          break;
        }
      }
      if (!ok) {
        debug("skip " + file + " because path not included");
        return true;
      }
    }
    return false;
  };


  /*
  #3 Search depth
  
  The search depth specifies in which level of subdirectories the filter will match.
  1 means everything in the given directory, 2 one level deeper.
  - `mindepth` - `Integer` minimal depth to match
  - `maxdepth` - `Integer` maximal depth to match
  
  __Example__
  
  ``` coffee
  fs = require 'alinex-fs'
  fs.find '/tmp/some/directory',
    filter:
      mindepth: 1
      maxdepth: 1
  , (err, list) ->
     * only the first sublevele:
     * list may include 'dir1/abc', 'dir1/abd', 'dir1/abe', 'dir1/bb', 'dir1/bcd', 'dir1/dir2'
  ```
   */

  skipDepth = function(file, depth, options, cb) {
    return cb(skipDepthSync(file, depth, options));
  };

  skipDepthSync = function(file, depth, options) {
    if ((options.maxdepth != null) && options.maxdepth < depth) {
      debug("skip " + file + " because deeper than specified depth");
      return 'SKIPPATH';
    }
    if ((options.mindepth != null) && options.mindepth > depth) {
      debug("skip " + file + " because not in specified depth");
      return true;
    }
    return false;
  };

  filestat = function(file, options, cb) {
    var stat;
    stat = options.dereference != null ? fs.stat : fs.lstat;
    return stat(file, function(err, stats) {
      if (err && (options.dereference != null)) {
        debug("error resolving " + file + " link");
        return filestat(file, {}, cb);
      }
      return cb(err, stats);
    });
  };

  filestatSync = function(file, options) {
    var error, stat;
    stat = options.dereference != null ? fs.statSync : fs.lstatSync;
    try {
      return stat(file);
    } catch (error1) {
      error = error1;
      debug("error resolving " + file + " link " + error.message);
      if (options.dereference != null) {
        return filestatSync(file, {});
      }
      throw error;
    }
  };


  /*
  #3 File type
  
  Use `type` to specify which type of file you want to use.
  
  Possible values:
  
  - `file`, `f`
  - `directory`, `dir`, `d`
  - `link`, `l`
  - `fifo`, `pipe`, `p`
  - `socket`, `s`
  
  Also you may set `dereference` to `true` to follow symbolic links and analyze their
  target.
  
  __Example__
  
  ``` coffee
  fs = require 'alinex-fs'
  fs.find '/tmp/some/directory',
    filter:
      type: 'f'
  , (err, list) ->
     * list may include 'test/temp/file1', 'test/temp/file2', 'test/temp/dir1/file11'
  ```
   */

  skipType = function(file, options, cb) {
    if (!options.type) {
      return cb();
    }
    return filestat(file, options, function(err, stats) {
      if (err) {
        debug("skip because error " + err + " in stat for " + file);
        return cb();
      }
      switch (options.type) {
        case 'file':
        case 'f':
          if (stats.isFile()) {
            return cb();
          }
          debug("skip " + file + " because not a file entry");
          break;
        case 'directory':
        case 'dir':
        case 'd':
          if (stats.isDirectory()) {
            return cb();
          }
          debug("skip " + file + " because not a directory entry");
          break;
        case 'link':
        case 'l':
          if (stats.isSymbolicLink()) {
            return cb();
          }
          debug("skip " + file + " because not a link entry");
          break;
        case 'fifo':
        case 'pipe':
        case 'p':
          if (stats.isFIFO()) {
            return cb();
          }
          debug("skip " + file + " because not a FIFO entry");
          break;
        case 'socket':
        case 's':
          if (stats.isSocket()) {
            return cb();
          }
          debug("skip " + file + " because not a socket entry");
      }
      return cb(true);
    });
  };

  skipTypeSync = function(file, options) {
    var err, stats;
    if (!options.type) {
      return false;
    }
    try {
      stats = filestatSync(file, options);
    } catch (error1) {
      err = error1;
      debug("skip because error " + err + " in stat for " + file);
      return;
    }
    switch (options.type) {
      case 'file':
      case 'f':
        if (stats.isFile()) {
          return;
        }
        debug("skip " + file + " because not a file entry");
        break;
      case 'directory':
      case 'dir':
      case 'd':
        if (stats.isDirectory()) {
          return;
        }
        debug("skip " + file + " because not a directory entry");
        break;
      case 'link':
      case 'l':
        if (stats.isSymbolicLink()) {
          return;
        }
        debug("skip " + file + " because not a link entry");
        break;
      case 'fifo':
      case 'pipe':
      case 'p':
        if (stats.isFIFO()) {
          return;
        }
        debug("skip " + file + " because not a FIFO entry");
        break;
      case 'socket':
      case 's':
        if (stats.isSocket()) {
          return;
        }
        debug("skip " + file + " because not a socket entry");
    }
    return true;
  };

  sizeHumanToInt = function(text) {
    var match;
    if (typeof text === 'string' && (match = text.match(/^(\d*\.?\d*)\s*([kKmMgGtTpP])$/))) {
      switch (match[2]) {
        case 'k':
        case 'K':
          return match[1] * 1024;
        case 'm':
        case 'M':
          return match[1] * Math.pow(1024, 2);
        case 'g':
        case 'G':
          return match[1] * Math.pow(1024, 3);
        case 'T':
        case 'T':
          return match[1] * Math.pow(1024, 4);
        case 'P':
        case 'P':
          return match[1] * Math.pow(1024, 5);
      }
    }
    return text;
  };


  /*
  #3 File size
  
  With the `minsize` and  `maxsize` options it is possible to specify the exact
  size of the matching files in bytes:
  
  - use an `Integer` value as number of bytes
  - use a `String` like `1M` or `100k`
  
  __Example__
  
  ``` coffee
  fs = require 'alinex-fs'
  fs.find '/tmp/some/directory',
    filter:
      maxsize: 1024 * 1024
  , (err, list) ->
     * list contains only files larger than 1MB
  ```
   */

  skipSize = function(file, options, cb) {
    if (!(options.minsize || options.maxsize)) {
      return cb();
    }
    if (options.minsize) {
      options.minsize = sizeHumanToInt(options.minsize);
    }
    if (options.maxsize) {
      options.maxsize = sizeHumanToInt(options.maxsize);
    }
    return filestat(file, options, function(err, stats) {
      var skip;
      if (err) {
        debug("skip because error " + err + " in stat for " + file);
        return cb();
      }
      skip = ((options.minsize != null) && options.minsize > stats.size) || ((options.maxsize != null) && options.maxsize < stats.size);
      if (skip) {
        debug("skip " + file + " because size mismatch");
      }
      return cb(skip);
    });
  };

  skipSizeSync = function(file, options) {
    var err, skip, stats;
    if (!(options.minsize || options.maxsize)) {
      return false;
    }
    if (options.minsize) {
      options.minsize = sizeHumanToInt(options.minsize);
    }
    if (options.maxsize) {
      options.maxsize = sizeHumanToInt(options.maxsize);
    }
    try {
      stats = filestatSync(file, options);
    } catch (error1) {
      err = error1;
      debug("skip because error " + err + " in stat for " + file);
      return;
    }
    skip = ((options.minsize != null) && options.minsize > stats.size) || ((options.maxsize != null) && options.maxsize < stats.size);
    if (skip) {
      debug("skip " + file + " because size mismatch");
    }
    return skip;
  };


  /*
  #3 Owner and Group
  
  You may also specify files based on the user which owns the files or the group
  of the files.
  
  Both may be specified as id (uid or gid) or using the alias name.
  - `user` - `Integer|String` owner name or id
  - `group` - `Integer|String` owner group name or id
  
  __Example__
  
  ``` coffee
  fs = require 'alinex-fs'
  fs.find '/tmp/some/directory',
    filter:
      user: process.uid
  , (err, list) ->
     * list contains only files belonging to the current user
  ```
   */

  skipOwner = function(file, options, cb) {
    var error, gid, uid;
    if (!(options.user || options.group)) {
      return cb();
    }
    if (options.user && !isNaN(options.user)) {
      try {
        uid = posix.getpwnam(options.user).uid;
      } catch (error1) {
        error = error1;
        return cb(error);
      }
    }
    if (options.group && !isNaN(options.group)) {
      try {
        gid = posix.getgrnam(options.group).gid;
      } catch (error1) {
        error = error1;
        return cb(error);
      }
    }
    return filestat(file, options, function(err, stats) {
      var skip;
      if (err) {
        if (err) {
          return cb(err);
        }
        debug("skip because error " + err + " in stat for " + file);
        return cb();
      }
      skip = (uid && uid === !stats.uid) || (gid && gid === !stats.gid);
      if (skip) {
        debug("skip " + file + " because owner mismatch");
      }
      return cb(skip);
    });
  };

  skipOwnerSync = function(file, options) {
    var err, gid, skip, stats, uid;
    if (!(options.user || options.group)) {
      return false;
    }
    if (options.user && !isNaN(options.user)) {
      uid = posix.getpwnam(options.user).uid;
    }
    if (options.group && !isNaN(options.group)) {
      gid = posix.getgrnam(options.group).gid;
    }
    try {
      stats = filestatSync(file, options);
    } catch (error1) {
      err = error1;
      debug("skip because error " + err + " in stat for " + file);
      throw err;
    }
    skip = (uid && uid === !stats.uid) || (gid && gid === !stats.gid);
    if (skip) {
      debug("skip " + file + " because owner mismatch");
    }
    return skip;
  };

  timeCheck = function(stats, options) {
    var dir, i, j, len, len1, ref, ref1, ref2, ref3, type, value;
    ref1 = ['accessed', 'modified', 'created'];
    for (i = 0, len = ref1.length; i < len; i++) {
      type = ref1[i];
      ref2 = ['After', 'Before'];
      for (j = 0, len1 = ref2.length; j < len1; j++) {
        dir = ref2[j];
        if (!options[type + dir]) {
          continue;
        }
        ref = options[type + dir];
        if (typeof ref === 'string') {
          ref = ((ref3 = chrono.parseDate(ref)) != null ? ref3.getTime() : void 0) / 1000;
        }
        if (!ref) {
          throw new Error("Given value '" + options[type + dir] + "' in option " + (type + dir) + " is invalid.");
        }
        value = stats[type.charAt(0) + 'time'].getTime() / 1000;
        if (dir === 'Before' && value >= ref) {
          return false;
        }
        if (dir === 'After' && value <= ref) {
          return false;
        }
      }
    }
    return true;
  };


  /*
  #3 Time specification
  
  It is also possible to select files based on their `creation`, last `modified`
  or last `accessed` time.
  
  Specify the `Before` and `After` time appended to one of the above as:
  
  - Unix timestamp
  - ISO-8601 date formats
  - some local formats (based on platform support for Date.parse())
  - time difference from now (human readable)
  
  The following time definitions are an example what you may use:
  
  - `yesterday`, `2 days ago`, `last Monday` to specify a day from now
  - `yesterday 15:00`, `yesterday at 15:00` to also specify the time
  - `1 March`, `1st March` specifies a date in this year
  - `1 March 2014`, `1st March 2014`, '03/01/13`, `01.03.2014` all specifiying the 1st of march
  - `9:00`, `9:00 GMT+0900` to specify a time today or in combination with a date
  - `last night`, `00:00`
  
  If only a day is given it will use 12:00 as the time.
  
  __Example__
  
  ``` coffee
  fs = require 'alinex-fs'
  fs.find '/tmp/some/directory',
    filter:
      modifiedBefore: 'yesterday 12:00'
  , (err, list) ->
     * list contains only files older than yesterday 12 o'clock
  ```
   */

  skipTime = function(file, options, cb) {
    var dir, i, j, len, len1, ref1, ref2, type, used;
    used = false;
    ref1 = ['accessed', 'modified', 'created'];
    for (i = 0, len = ref1.length; i < len; i++) {
      type = ref1[i];
      ref2 = ['After', 'Before'];
      for (j = 0, len1 = ref2.length; j < len1; j++) {
        dir = ref2[j];
        if (options[type + dir]) {
          used = true;
        }
      }
    }
    if (!used) {
      return cb(false);
    }
    return filestat(file, options, function(err, stats) {
      var skip;
      if (err) {
        debug("skip because error " + err + " in stat for " + file);
        return cb();
      }
      skip = !timeCheck(stats, options);
      if (skip) {
        debug("skip " + file + " because out of time range");
      }
      return cb(skip);
    });
  };

  skipTimeSync = function(file, options) {
    var dir, err, i, j, len, len1, ref1, ref2, skip, stats, type, used;
    used = false;
    ref1 = ['accessed', 'modified', 'created'];
    for (i = 0, len = ref1.length; i < len; i++) {
      type = ref1[i];
      ref2 = ['After', 'Before'];
      for (j = 0, len1 = ref2.length; j < len1; j++) {
        dir = ref2[j];
        if (options[type + dir]) {
          used = true;
        }
      }
    }
    if (!used) {
      return false;
    }
    try {
      stats = filestatSync(file, options);
    } catch (error1) {
      err = error1;
      debug("skip because error " + err + " in stat for " + file);
      return;
    }
    skip = !timeCheck(stats, options);
    if (skip) {
      debug("skip " + file + " because out of time range");
    }
    return skip;
  };


  /*
  #3 User defined function
  
  With the `test` parameter you may add an user defined function which will be
  called to check each file. It will get the file path and options array so you
  may also add some configuration therefore in additional option values.
  
  Asynchronous call:
  
  ``` coffee
  fs = require 'alinex-fs'
  fs.find '.',
    filter:
      test: (file, options, cb) ->
        cb ~file.indexOf 'ab'
  , (err, list) ->
    console.log "Found #{list.length} matches."
  ```
  
  Or use synchronous calls:
  
  ``` coffee
  fs = require 'alinex-fs'
  list = fs.findSync 'test/temp',
    filter:
      test: (file, options) ->
        return ~file.indexOf 'ab'
  console.log "Found #{list.length} matches."
  ```
   */

  skipFunction = function(file, options, cb) {
    if (!(options.test || typeof options.test === !'function')) {
      return cb();
    }
    return options.test(file, options, function(ok) {
      if (!ok) {
        debug("skip " + file + " by user function");
      }
      return cb(!ok);
    });
  };

  skipFunctionSync = function(file, options) {
    var ok;
    if (!(options.test || typeof options.test === !'function')) {
      return false;
    }
    ok = options.test(file, options);
    if (!ok) {
      debug("skip " + file + " by user function");
    }
    return !ok;
  };

}).call(this);

//# sourceMappingURL=filter.map