seaweedfs/lib/seaweedfs.js

A simple HTTP REST client for the seaweedFS file database

// Generated by CoffeeScript 1.9.2
var Bunyan, Promise, SeaweedFS, _, bformat, fs, qs, request;

qs = require('querystring');

fs = require('fs');

_ = require('lodash');

Promise = require('bluebird');

request = require('request');

Bunyan = require('bunyan');

bformat = require('bunyan-format');

SeaweedFS = (function() {
  function SeaweedFS(config, logger) {
    this.config = config != null ? config : {};
    this.logger = logger;
    _.defaults(this.config, {
      masters: [
        {
          host: 'localhost',
          port: 9333
        }
      ],
      scheme: 'http',
      retry_count: 60,
      retry_timeout: 2000,
      log_name: 'SeaweedFS',
      log_level: 'info'
    });
    if (this.logger == null) {
      this.logger = new Bunyan({
        name: this.config.log_name,
        streams: [
          {
            level: this.config.log_level,
            stream: bformat({
              outputMode: 'short'
            })
          }
        ]
      });
    }
    this.active_masters = _.cloneDeep(_.sortByAll(this.config.masters, ['host', 'port']));
    this.Promise = Promise;
    return;
  }

  SeaweedFS.prototype.makeMasterRequest = function(options) {
    if (options == null) {
      options = {};
    }
    options.uri = "" + this.address + options.path;
    this.logger.debug(options, "Making request to master");
    return this._makeRequest(options)["catch"]((function(_this) {
      return function(err) {

        /**
          Normally the server is supposed to send json responses but when it doesn't send a json response then
          that means that the server is warming up and has just now become the leader and is waiting to connect
          to the volumes this means that requests like /dir/assign will fail but /cluster/status will pass
          so we need to wait it out till the server warms up so we add a delay time out for that particlular
          request so that it doesn't fail
         */
        if (_.contains(err.message, 'Unexpected content-type')) {
          return _this._makeMasterRequest(options);
        } else if (_.contains(err.message, 'ECONNREFUSED')) {
          return _this.connect().then(function() {
            return _this.makeMasterRequest(options);
          });
        } else {
          return Promise.reject(err);
        }
      };
    })(this));
  };

  SeaweedFS.prototype._makeMasterRequest = function(options) {
    if (options == null) {
      options = {};
    }
    options = _.defaults(options, {
      retry_count: this.config.retry_count,
      retry_timeout: this.config.retry_timeout
    });
    return Promise.delay(options.retry_timeout).then((function(_this) {
      return function() {
        if (options.retry_count === 0) {
          return Promise.reject(new Error("Failed request to " + options.uri));
        } else {
          _this.logger.debug(options, 'Retrying request to master');
          options.retry_count -= 1;
          return _this.makeMasterRequest(options);
        }
      };
    })(this));
  };

  SeaweedFS.prototype.makeVolumeRequest = function(options) {
    var file_id, preferred_location, request_options;
    if (options == null) {
      options = {};
    }
    file_id = options.file_id, preferred_location = options.preferred_location, request_options = options.request_options;
    return this.find(file_id).then((function(_this) {
      return function(locations) {
        if (locations.length > 0) {
          if (preferred_location != null) {
            locations.unshift(_this.config.scheme + "://" + preferred_location + "/" + file_id);
          }
          return Promise.reduce(locations, function(result, location) {
            if (result.success) {
              return Promise.resolve(result);
            } else {
              request_options.uri = location;
              _this.logger.debug(_.pick(request_options, 'method', 'uri'), "Making request to volume");
              return _this._makeRequest(request_options).then(function(response) {
                result.success = true;
                result.value = response;
                return Promise.resolve(result);
              })["catch"](function(err) {
                if (result.errors == null) {
                  result.errors = {};
                }
                result.errors[location] = {
                  is_error: true,
                  message: err.message
                };
                return Promise.resolve(result);
              });
            }
          }, {
            success: false
          }).then(function(arg) {
            var errors, value;
            value = arg.value, errors = arg.errors;
            if ((value != null) || (request_options.stream != null)) {
              return Promise.resolve(value);
            } else {
              return Promise.reject(new Error("Unable to perform file operations on '" + file_id + "': " + (JSON.stringify(errors))));
            }
          });
        } else {
          return Promise.reject(new Error("File location for '" + file_id + "' not found"));
        }
      };
    })(this));
  };

  SeaweedFS.prototype._makeRequest = function(options) {
    if (options == null) {
      options = {};
    }
    if (options.full_response == null) {
      options.full_response = false;
    }
    return new Promise((function(_this) {
      return function(resolve, reject) {
        var file, ref;
        if (options.stream != null) {
          _this.logger.debug(_.pick(options, 'uri'), "Streaming file");
          return request(options.uri).on('response', function(response) {
            return resolve();
          }).on('error', function(err) {
            return reject(err);
          }).pipe(options.stream);
        } else {
          if ((file = (ref = options.formData) != null ? ref.file : void 0) != null) {
            if (!(file instanceof Buffer)) {
              file = fs.createReadStream(file);
              file.on('error', function(err) {
                return reject(err);
              });
              options.formData.file = file;
            }
          }
          return request(options, function(err, response) {
            if (err) {
              return reject(err);
            } else if (options.full_response) {
              return resolve(response);
            } else {
              if (/application\/json/.test(response.headers['content-type'])) {
                return resolve(JSON.parse(response.body));
              } else {
                try {
                  return resolve(JSON.parse(response.body));
                } catch (_error) {
                  err = _error;
                  return reject(new Error("Unexpected content-type '" + response.headers['content-type'] + "' in response from " + options.uri));
                }
              }
            }
          });
        }
      };
    })(this));
  };

  SeaweedFS.prototype.connect = function() {
    return this._connect(0, this.config.retry_count);
  };

  SeaweedFS.prototype._connect = function(master_index, retry_count) {
    var master;
    this.logger.debug({
      master_index: master_index,
      active_masters: this.active_masters
    }, "Connecting to master");
    if ((master = this.active_masters[master_index]) != null) {
      this.address = this.config.scheme + "://" + master.host + ":" + master.port;
      return this._makeRequest({
        uri: this.address + "/cluster/status"
      }).then((function(_this) {
        return function(status) {
          _this.logger.debug("Connected to master " + _this.address);
          return Promise.resolve();
        };
      })(this))["catch"]((function(_this) {
        return function(err) {
          _this.logger.debug("Error connecting to master " + _this.address);
          return _this._connect(master_index + 1, retry_count);
        };
      })(this));
    } else {
      return Promise.delay(this.config.retry_timeout).then((function(_this) {
        return function() {
          if (retry_count === 0) {
            return Promise.reject(new Error('Could not connect to any nodes'));
          } else {
            _this.logger.debug(retry_count, 'Retrying connection to master');
            return _this._connect(0, retry_count - 1);
          }
        };
      })(this));
    }
  };

  SeaweedFS.prototype.clusterStatus = function() {
    this.logger.trace('Getting cluster status');
    return this.makeMasterRequest({
      path: "/cluster/status"
    });
  };

  SeaweedFS.prototype.find = function(file_id) {
    var volume;
    this.logger.trace(file_id, 'Finding');
    if (!/^\d+,[a-zA-Z0-9_]+$/.test(file_id)) {
      return Promise.reject(new Error("File '" + file_id + "' is not a valid file_id"));
    }
    volume = file_id.split(',')[0];
    return this.makeMasterRequest({
      path: "/dir/lookup?volumeId=" + volume
    }).then((function(_this) {
      return function(result) {
        var i, len, location, locations, ref;
        locations = [];
        if (result.locations != null) {
          ref = result.locations;
          for (i = 0, len = ref.length; i < len; i++) {
            location = ref[i];
            locations.push(_this.config.scheme + "://" + location.publicUrl + "/" + file_id);
          }
        }
        return Promise.resolve(locations);
      };
    })(this));
  };

  SeaweedFS.prototype.read = function(file_id, stream, options) {
    var preferred_location;
    if (options == null) {
      options = {};
    }
    this.logger.trace(file_id, 'Reading');
    preferred_location = options.preferred_location;
    return this.makeVolumeRequest({
      file_id: file_id,
      preferred_location: preferred_location,
      request_options: {
        stream: stream,
        method: 'GET',
        encoding: null,
        full_response: true
      }
    }).then((function(_this) {
      return function(response) {
        if (stream != null) {
          return Promise.resolve();
        } else {
          if (response.statusCode === 404) {
            return Promise.reject(new Error("file '" + file_id + "' not found"));
          } else {
            return Promise.resolve(response.body);
          }
        }
      };
    })(this));
  };

  SeaweedFS.prototype.assign = function(files, options) {
    if (options == null) {
      options = {};
    }
    this.logger.trace(options, 'Assigning');
    options = _.defaults(options, {
      retry_count: this.config.retry_count,
      retry_timeout: this.config.retry_timeout
    });
    return this.makeMasterRequest({
      path: "/dir/assign?" + (qs.stringify({
        count: files.length
      }))
    }).then((function(_this) {
      return function(response) {
        if ((response.error != null) && options.retry_count > 0) {
          options.retry_count -= 1;
          return Promise.delay(options.retry_timeout).then(function() {
            return _this.assign(files, options);
          });
        } else {
          return Promise.resolve(response);
        }
      };
    })(this))["catch"]((function(_this) {
      return function(err) {
        if (_.contains(err.message, 'Unexpected content-type') && options.retry_count > 0) {
          options.retry_count -= 1;
          return Promise.delay(options.retry_timeout).then(function() {
            return _this.assign(files, options);
          });
        } else {
          return Promise.reject(err);
        }
      };
    })(this));
  };

  SeaweedFS.prototype.write = function(files) {
    var is_error, results;
    this.logger.trace(files, 'Writing');
    if (!_.isArray(files)) {
      files = [files];
    }
    is_error = false;
    results = [];
    return this.assign(files).then((function(_this) {
      return function(file_info) {
        var current;
        current = Promise.cast();
        _.forEach(files, function(file, index) {
          return current = current.then(function() {
            var file_id;
            if (parseInt(index) === 0) {
              file_id = file_info.fid;
            } else {
              file_id = file_info.fid + "_" + index;
            }
            return _this.makeVolumeRequest({
              file_id: file_id,
              request_options: {
                method: 'POST',
                formData: {
                  file: file
                }
              }
            }).then(function(result) {
              return results.push(result);
            })["catch"](function(err) {
              is_error = true;
              return results.push({
                is_error: true,
                message: err.message
              });
            });
          });
        });
        return current.then(function() {
          if (is_error || results.length !== files.length) {
            return Promise.reject(new Error("An error occured while upload files: " + (JSON.stringify(results))));
          } else {
            return Promise.resolve(file_info, results);
          }
        });
      };
    })(this));
  };

  SeaweedFS.prototype.remove = function(file_id) {
    this.logger.trace(file_id, 'Removing');
    return this.makeVolumeRequest({
      file_id: file_id,
      request_options: {
        method: 'DELETE',
        full_response: true
      }
    }).then((function(_this) {
      return function(response) {
        if (response.statusCode === 404) {
          return Promise.reject(new Error("file '" + file_id + "' not found"));
        } else {
          return Promise.resolve(JSON.parse(response.body));
        }
      };
    })(this));
  };

  return SeaweedFS;

})();

module.exports = SeaweedFS;