modern-valhalla/lib/storage/s3/index.js

Private npm repository server

/* eslint prefer-rest-params: "off" */
/* eslint prefer-spread: "off" */
'use strict';

const assert = require('assert');
const fs = require('fs');
const Path = require('path');
const Stream = require('stream');
const url = require('url');
const async = require('async');
const createError = require('http-errors');
const _ = require('lodash');
const semver = require('semver');

const DBManager = require('./db-manager');
const Logger = require('../../logger');
const customStream = require('../../storage/streams');
const Utils = require('../../utils');

let FileManager;
if (process.env.NODE_ENV === 'production') {
  FileManager = require('./s3-client');
} else {
  FileManager = require('./s3-client-mock');
}

const pkgFileName = 'package.json';
const noSuchFile = 'ENOENT';
// const resourceNotAvailable = 'EAGAIN';

const generatePackageTemplate = function(name) {
  return {
    // standard things
    'name': name,
    'versions': {},
    'dist-tags': {},

    // our own object
    '_distfiles': {},
    '_attachments': {},
    '_uplinks': {},
  };
};

/**
 * Implements Storage interface (same for storage.js, local-storage.js, up-storage.js).
 */
class LocalStorage {
  /**
   * Constructor
   * @param {Object} config config list of properties
   */
  constructor(config) {
    this.config = config;
    this.db = new DBManager(config);
    this.fileManager = new FileManager(config);
    this.logger = Logger.logger.child({sub: 'sql'});
  }

  /**
   * Add a package.
   * @param {string} name
   * @param {Object} info
   * @param {Function} callback
   */
  addPackage(name, info, callback) {
    this.updateVersions(name, info, (err) => { // _writePackage
      callback(err, info);
    });
  }

  /**
   * Remove package.
   * @param {*} name
   * @param {*} callback
   * @return {Function}
   */
  removePackage(name, callback) {
    this.logger.info({name: name}, 'unpublishing @{name} (all)');

    let storage = this.storage;
    if (!storage) {
      return callback(createError(404, 'no such package available'));
    }

    storage.readJSON(pkgFileName, (err, data) => {
      if (err) {
        if (err.code === noSuchFile) {
          return callback(createError(404, 'no such package available'));
        } else {
          return callback(err);
        }
      }
      this._normalizePackage(data);

      storage.unlink(pkgFileName, function(err) {
        if (err) {
          return callback(err);
        }

        const files = Object.keys(data._attachments);

        const unlinkNext = function(cb) {
          if (files.length === 0) {
            return cb();
          }

          let file = files.shift();
          storage.unlink(file, function() {
            unlinkNext(cb);
          });
        };

        unlinkNext(function() {
          // try to unlink the directory, but ignore errors because it can fail
          storage.rmdir('.', function(err) {
            callback(err);
          });
        });
      });
    });
  }

  /**
   * Synchronize remote package info with the local one
   * @param {*} name
   * @param {*} newdata
   * @param {*} callback
   */
  updateVersions(name, newdata, callback) {
    this._readCreatePackage(name, (err, data) => {
      if (err) {
        return callback(err);
      }

      const refreshUplinksOnly = typeof newdata._uplinks === 'object' &&
        Object.keys(newdata._uplinks).map((up) => {
          return Utils.is_object(data._uplinks[up])
            && newdata._uplinks[up].etag === data._uplinks[up].etag
            && newdata._uplinks[up].fetched > data._uplinks[up].fetched;
        })
        .every((x) => x === true);

      if (refreshUplinksOnly) {
        this.logger.info({name: name}, 'Data from remote still fresh, refreshing uplinks for @{name}');
        return this._writeUplinks(name, data, function(err) {
          callback(err, data);
        });
      }

      let change = false;
      for (let ver in newdata.versions) {
        if (_.isNil(data.versions[ver])) {
          let verdata = newdata.versions[ver];

          // we don't keep readmes for package versions,
          // only one readme per package
          delete verdata.readme;

          change = true;
          data.versions[ver] = verdata;

          if (verdata.dist && verdata.dist.tarball) {
            let filename = url.parse(verdata.dist.tarball).pathname.replace(/^.*\//, '');
            // we do NOT overwrite any existing local package
            if (_.isNil(data._attachments[ver])) {
              let hash = data._distfiles[filename] = {
                url: verdata.dist.tarball,
                sha: verdata.dist.shasum,
              };

              if (verdata._valhalla_uplink) {
                // if we got this information from a known registry,
                // use the same protocol for the tarball
                //
                // see https://github.com/rlidwka/sinopia/issues/166
                const tarball_url = url.parse(hash.url);
                const uplink_url = url.parse(this.config.uplinks[verdata._valhalla_uplink].url);
                if (uplink_url.host === tarball_url.host) {
                  tarball_url.protocol = uplink_url.protocol;
                  hash.registry = verdata._valhalla_uplink;
                  hash.url = url.format(tarball_url);
                }
              }
            }
          }
        }
      }

      for (let tag in newdata['dist-tags']) {
        if (!data['dist-tags'][tag] || semver.lt(data['dist-tags'][tag], newdata['dist-tags'][tag])) {
          change = true;
          data['dist-tags'][tag] = newdata['dist-tags'][tag];
        }
      }

      for (let up in newdata._uplinks) {
        if (Object.prototype.hasOwnProperty.call(newdata._uplinks, up)) {
          const need_change = !Utils.is_object(data._uplinks[up])
            || newdata._uplinks[up].etag !== data._uplinks[up].etag
            || newdata._uplinks[up].fetched !== data._uplinks[up].fetched;

          if (need_change) {
            change = true;
            data._uplinks[up] = newdata._uplinks[up];
          }
        }
      }

      if (newdata.readme !== data.readme) {
        data.readme = newdata.readme;
        change = true;
      }

      if (newdata['dist-tags'].latest && newdata.versions[newdata['dist-tags'].latest]) {
        const latest = newdata.versions[newdata['dist-tags'].latest];

        if (latest.description && latest.description !== data.description) {
          data.description = latest.description;
          change = true;
        }

        if (latest.author && latest.author !== data.author) {
          data.author = typeof latest.author === 'object' ? latest.author.name : latest.author;
          change = true;
        }
      }

      if (change) {
        this.logger.debug('updating package info');
        this._writePackage(name, data)
          .then(() => callback(null, data))
          .catch((err) => callback(err, data));
      } else {
        callback(null, data);
      }
    });
  }

  /**
   * Add a new version to a previous local package.
   * @param {*} name
   * @param {*} version
   * @param {*} metadata
   * @param {*} tag
   * @param {*} callback
   */
  addVersion(name, version, metadata, tag, callback) {
    this._updatePackage(name, (data, cb) => {
      // keep only one readme per package
      data.readme = metadata.readme;
      delete metadata.readme;

      if (data.versions[version] != null) {
        return cb(createError[409]('this version already present'));
      }

      // if uploaded tarball has a different shasum, it's very likely that we have some kind of error
      if (Utils.is_object(metadata.dist) && typeof(metadata.dist.tarball) === 'string') {
        let tarball = metadata.dist.tarball.replace(/.*\//, '');
        if (Utils.is_object(data._attachments[tarball])) {
          if (data._attachments[tarball].shasum != null && metadata.dist.shasum != null) {
            if (data._attachments[tarball].shasum != metadata.dist.shasum) {
              return cb(createError[400]('shasum error, '
                + data._attachments[tarball].shasum
                + ' != ' + metadata.dist.shasum));
            }
          }
          data._attachments[tarball].version = version;
        }
      }

      data.versions[version] = metadata;
      Utils.tag_version(data, version, tag);
      // this.localList.add(name);
      cb();
    })
      .then(() => callback())
      .catch((err) => callback(err));
  }

  /**
   * Merge a new list of tags for a local packages with the existing one.
   * @param {*} name
   * @param {*} tags
   * @param {*} callback
   */
  mergeTags(name, tags, callback) {
    this._updatePackage(name, function updater(data, cb) {
      for (let t in tags) {
        if (tags[t] === null) {
          delete data['dist-tags'][t];
          continue;
        }
        // be careful here with == (cast)
        if (_.isNil(data.versions[tags[t]])) {
          return cb(createError[404]('this version doesn\'t exist'));
        }

        Utils.tag_version(data, tags[t], t);
      }
      cb();
    })
      .then(() => callback())
      .catch((err) => callback(err));
  }

  /**
   * Replace the complete list of tags for a local package.
   * @param {*} name
   * @param {*} tags
   * @param {*} callback
   */
  replaceTags(name, tags, callback) {
    this._updatePackage(name, function updater(data, cb) {
      data['dist-tags'] = {};

      for (let t in tags) {
        if (_.isNull(tags[t])) {
          delete data['dist-tags'][t];
          continue;
        }

        if (_.isNil(data.versions[tags[t]])) {
          return cb(createError[404]('this version doesn\'t exist'));
        }

        Utils.tag_version(data, tags[t], t);
      }
      cb();
    })
      .then(() => callback())
      .catch((err) => callback(err));
  }

  /**
   * Update the package metadata, tags and attachments (tarballs).
   * Note: Currently supports unpublishing only.
   * @param {*} name
   * @param {*} metadata
   * @param {*} revision
   * @param {*} callback
   * @return {Function}
   */
  changePackage(name, metadata, revision, callback) {
    if (!Utils.is_object(metadata.versions) || !Utils.is_object(metadata['dist-tags'])) {
      return callback(createError[422]('bad data'));
    }

    this._updatePackage(name, (data, cb) => {
      for (let ver in data.versions) {
        if (_.isNil(metadata.versions[ver])) {
          this.logger.info({name: name, version: ver},
            'unpublishing @{name}@@{version}');
          delete data.versions[ver];
          for (let file in data._attachments) {
            if (data._attachments[file].version === ver) {
              delete data._attachments[file].version;
            }
          }
        }
      }
      data['dist-tags'] = metadata['dist-tags'];
      cb();
    })
      .then(() => callback())
      .catch((err) => callback(err));
  }

  /**
   * Remove a tarball.
   * @param {*} name
   * @param {*} filename
   * @param {*} revision
   * @param {*} callback
   * @return {Promise}
   */
  removeTarball(name, filename, revision) {
    assert(Utils.validate_name(filename));

    return new Promise((resolve, reject) => {
      const filePath = this._getFilePath(name, filename);

      this.fileManager.deleteFile(filePath, (err, result) => {
        if (err) {
          this.logger.error({file: filePath, error: err}, 'Failed to delete @{file}: @{error.message}');
          return reject(createError[404]('no such file available'));
        }
        this.logger.info({file: filePath}, '@{file} succesfully deleted from S3 storage');
        return resolve(result);
      });
    });
  }

  /**
   * Add a tarball.
   * @param {String} name A name of the package
   * @param {String} filename Complete name of the file to be saved including path and extension
   * @param {String|Buffer} data The data to be saved into
   * @return {Promise}
   */
  addTarball(name, filename, data) {
    return new Promise((resolve, reject) => {
      if (!Utils.validate_name(filename)) {
        return reject(createError[400]('Package name has a wrong format'));
      }

      const filePath = this._getFilePath(name, filename);

      this.fileManager.putFileContent(data, filePath, false, (err, result) => {
        if (err) {
          this.logger.error({file: filePath, error: err}, 'Failed to upload @{file}: @{error.message}');
          return reject(err);
        }

        this.logger.info({file: filePath}, '@{file} successfully uploaded to S3 storage');
        return resolve(result);
      });
    });
  }

  /**
   * Get a tarball
   * Currently works only for local packages since all remote packages pointing directly to upstreams
   * @param {String} name Package name
   * @param {String} filename Attachment filename
   * @return {Promise}
   */
  getTarball(name, filename) {
    return this.db.getPackageAttachment(name, filename)
      .catch((err) => {
        throw this._internalError(err, name, 'DB error');
      })
      .then((pkg) => {
        if (!pkg) {
          throw createError(404, 'Package not found');
        }

        const stream = new customStream.ReadTarball();
        stream.abort = function() {
          if (rstream) {
            rstream.abort();
          }
        };

        const rstream = this.fileManager.getFileStream(this._getFilePath(name, filename));

        rstream.on('error', (err) => {
          if (err && err.code === noSuchFile) {
            stream.emit('error', createError(404, 'no such file available'));
          } else {
            stream.emit('error', err);
          }
        });
        rstream.on('content-length', (l) => {
          stream.emit('content-length', l);
        });
        rstream.pipe(stream);

        return stream;
      });
  }

  /**
   * Retrieve a package by name.
   * @param {*} name
   * @return {Promise}
   */
  getPackage(name) {
    return this.db.getPackage(name)
      .catch((err) => {
        throw this._internalError(err, name, 'DB error');
      })
      .then((pkg) => {
        if (!pkg) {
          throw createError[404]('no such package available');
        }

        this._normalizePackage(pkg);
        return pkg;
      });
  }

  getPackageReadme(name) {
    return this.db.getPackageReadme(name)
      .catch((err) => {
        throw this._internalError(err, name, 'DB error');
      })
      .then((readme) => {
        if (!readme) {
          throw createError[404]('no such package available');
        }

        return readme;
      });
  }

  /**
   * Retrieve a list of local packages.
   * @return {Promise}
   */
  getLocalPackages() {
    return this.db.getLocalPackages()
      .then((pkgs) => {
        pkgs.forEach((pkg) => this._normalizePackage(pkg));
        return pkgs;
      });
  }

  /**
   * Retrieve a list of local packages that matches a query `q`.
   * @param {String} query Search query
   * @param {Number} limit Results limit
   * @return {Promise}
   */
  searchLocalPackages(query, limit) {
    return this.db.searchLocalPackages(query, limit)
      .then((pkgs) => {
        pkgs.forEach((pkg) => this._normalizePackage(pkg));
        return pkgs;
      });
  }

  /**
   * Search a local package.
   * @param {*} startKey
   * @param {*} options
   * @return {Function}
   */
  search(startKey, options) {
    const stream = new Stream.PassThrough({objectMode: true});

    this._eachPackage((item, cb) => {
      fs.stat(item.path, (err, stats) => {
        if (err) {
          return cb(err);
        }

        if (stats.mtime > startKey) {
          this.getPackage(item.name, options, function(err, data) {
            if (err) {
              return cb(err);
            }

            const versions = Utils.semver_sort(Object.keys(data.versions));
            const latest = data['dist-tags'] && data['dist-tags'].latest ? data['dist-tags'].latest : versions.pop();

            if (data.versions[latest]) {
              const version = data.versions[latest];
              stream.push({
                'name': version.name,
                'description': version.description,
                'dist-tags': {latest: latest},
                'maintainers': version.maintainers || [version.author].filter(Boolean),
                'author': version.author,
                'repository': version.repository,
                'readmeFilename': version.readmeFilename || '',
                'homepage': version.homepage,
                'keywords': version.keywords,
                'bugs': version.bugs,
                'license': version.license,
                'time': {
                  modified: item.time ? new Date(item.time).toISOString() : undefined,
                },
                'versions': {},
              });
            }

            cb();
          });
        } else {
          cb();
        }
      });
    }, function on_end(err) {
      if (err) return stream.emit('error', err);
      stream.end();
    });

    return stream;
  }

  hasLocalPackage(name, version) {
    return this.db.hasLocalPackage(name, version);
  }

  /**
   * Walks through each package and calls `on_package` on them.
   * @param {*} onPackage
   * @param {*} on_end
   */
  _eachPackage(onPackage, on_end) {
    let storages = {};

    storages[this.config.storage] = true;
    if (this.config.packages) {
      Object.keys(this.packages || {}).map((pkg) => {
        if (this.config.packages[pkg].storage) {
          storages[this.config.packages[pkg].storage] = true;
        }
      });
    }
    const base = Path.dirname(this.config.self_path);

    async.eachSeries(Object.keys(storages), function(storage, cb) {
      fs.readdir(Path.resolve(base, storage), function(err, files) {
        if (err) {
          return cb(err);
        }

        async.eachSeries(files, function(file, cb) {
          if (file.match(/^@/)) {
            // scoped
            fs.readdir(Path.resolve(base, storage, file), function(err, files) {
              if (err) {
                return cb(err);
              }

              async.eachSeries(files, function(file2, cb) {
                if (Utils.validate_name(file2)) {
                  onPackage({
                    name: `${file}/${file2}`,
                    path: Path.resolve(base, storage, file, file2),
                  }, cb);
                } else {
                  cb();
                }
              }, cb);
            });
          } else if (Utils.validate_name(file)) {
            onPackage({
              name: file,
              path: Path.resolve(base, storage, file),
            }, cb);
          } else {
            cb();
          }
        }, cb);
      });
    }, on_end);
  }

  /**
   * Normalise package properties, tags, revision id.
   * @param {Object} pkg package reference.
   */
  _normalizePackage(pkg) {
    this.logger.debug({name: pkg.name}, 'Normalizing package @{name}');

    if (pkg.DistTags) {
      if (!Array.isArray(pkg.DistTags)) {
        pkg['dist-tags'] = {};
      } else {
        pkg['dist-tags'] = pkg.DistTags.reduce((acc, tag) => {
          acc[tag.name] = tag.version;
          return acc;
        }, {});
      }
      delete pkg.DistTags;
    }

    if (pkg.Uplinks) {
      if (!Array.isArray(pkg.Uplinks)) {
        pkg._uplinks = {};
      } else {
        pkg._uplinks = pkg.Uplinks.reduce((acc, ul) => {
          acc[ul.name] = {
            etag: ul.etag,
            fetched: ul.fetched,
          };
          return acc;
        }, {});
      }
      delete pkg.Uplinks;
    }

    if (!pkg._attachments) {
      pkg._attachments = {};
    }

    if (!pkg._distfiles) {
      pkg._distfiles = {};
    }

    if (pkg.Versions) {
      if (!Array.isArray(pkg.Versions)) {
        pkg.versions = {};
      } else {
        pkg.versions = pkg.Versions.reduce((acc, v) => {
          const version = v.version;
          const versionData = JSON.parse(v._raw);

          versionData.dist = {};

          if (v.sha) {
            versionData.dist.shasum = v.sha;
          }
          if (v.registry) {
            Object.defineProperty(versionData, '_valhalla_uplink', {
              value: v.registry,
              enumerable: false,
              configurable: false,
              writable: true,
            });
            versionData.dist.tarball = v.url;
          } else {
            pkg._attachments[version] = v.attachment;
            versionData.dist.tarball = this._getPackageUrl(this.config.s3.path, pkg.name, v.attachment);
          }

          acc[version] = versionData;

          return acc;
        }, {});
      }
      delete pkg.Versions;
    }

    if (!pkg._rev && typeof pkg._rev !== 'string') {
      pkg._rev = '0-0000000000000000';
    }

    if (!pkg.readme) {
      pkg.readme = null;
    }

    // normalize dist-tags
    Utils.normalize_dist_tags(pkg);
  }

  /**
   * Retrieve either a previous created local package or a boilerplate.
   * @param {*} name
   * @param {*} callback
   */
  _readCreatePackage(name, callback) {
    this.db.getPackage(name)
      .catch((err) => {
        throw this._internalError(err, name, 'DB Error');
      })
      .then((pkg) => {
        if (pkg) {
          this._normalizePackage(pkg);
          callback(null, pkg);
        } else {
          // if package doesn't exist, we create it here
          const pkg = generatePackageTemplate(name);
          this._normalizePackage(pkg);
          return callback(null, pkg);
        }
      });
  }

  /**
   * Handle internal error
   * @param {*} err
   * @param {String} name Package name
   * @param {String} message Custom message
   * @return {Object} Error instance
   */
  _internalError(err, name, message) {
    this.logger.error({err: err, name: name}, message + ' @{name}: @{err.message}');
    return createError[500]();
  }

  /**
   * Update the package by passing the data through updater function
   * @param {*} name package name
   * @param {*} updateFn function(package, cb) - update function
   * @return {Promise}
   */
  _updatePackage(name, updateFn) {
    return this.db.getPackage(name)
      .catch((err) => {
        throw this._internalError(err, name, 'DB Error');
      })
      .then((pkg) => {
        if (!pkg) {
          throw createError[404]('no such package available');
        }

        this._normalizePackage(pkg);

        updateFn(pkg, (err) => {
          if (err) {
            // throw this._internalError(new Error('asdf'), name, 'DB error');
            throw err;
          }

          return this._writePackage(name, pkg);
        });
      });
  }

  /**
   * Update the revision (_rev) string for a package.
   * @param {*} name
   * @param {*} json
   * @param {*} callback
   * @return {Promise}
   */
  _writePackage(name, json) {
    return this.db.updatePackage(name, json);
  }

  /**
   * Update the revision (_rev) string for a package.
   * @param {*} name
   * @param {*} json
   * @param {*} callback
   * @return {Promise}
   */
  _writeUplinks(name, json, callback) {
    return this.db.updateUplinks(name, json)
      .then(() => callback(null))
      .catch((err) => callback(err));
  }

  _getFilePath(name, filename) {
    const dir = this.config.s3.packagesDirectory;
    const scheme = (dir ? `${dir}/` : '') + (this.config.s3.filePathScheme || '{{package}}/{{file}}');
    return scheme.replace('{{package}}', name).replace('{{file}}', filename);
  }

  _getPackageUrl(path, name, filename) {
    const scheme = this.config.s3.packageUrlScheme || '{{path}}{{package}}/-/{{file}}';
    return scheme.replace('{{path}}', path).replace('{{package}}', name).replace('{{file}}', filename);
  }
}

module.exports = LocalStorage;