modern-valhalla/lib/storage/s3/db-manager.js

Private npm repository server

const path = require('path');
const createError = require('http-errors');
const Sequelize = require('sequelize');
const Crypto = require('crypto');
const Logger = require('../../logger');
const Op = Sequelize.Op;

/**
 *
 */
class DBManager {
  /**
   *
   * @param {object} config
   */
  constructor(config = {}) {
    this.config = config;
    this.logger = Logger.logger.child({sub: 'sql'});
    this.logger.level('debug');

    if (!this.config.sql) {
      this.config.sql = {};
    }

    const dbOptions = {
      host: this.config.sql.host || 'localhost',
      port: this.config.sql.port || null,
      dialect: this.config.sql.dialect || 'sqlite',
      logging: false,
      retry: {
        match: /deadlock/i,
        max: 3,
      },
    };

    // storage options is for SQLite only
    if (dbOptions.dialect === 'sqlite' && !this.config.sql.storage) {
      dbOptions.storage = path.join(path.resolve(path.dirname(config.self_path || ''),
        config.storage,
        'database.sqlite'
      ));
    }

    dbOptions.pool = Object.assign({
      max: 5,
      min: 0,
      idle: 10000,
    }, config.sql.pool);

    const {database, username, password} = this.config.sql;

    let sequelize;
    if (process.env.DATABASE_URL) {
      sequelize = new Sequelize(process.env.DATABASE_URL, dbOptions);
    } else {
      sequelize = new Sequelize(database, username, password, dbOptions);
    }

    // SQL Models
    this.Package = sequelize.import(path.join(__dirname, 'models', 'package.js'));
    this.DistTag = sequelize.import(path.join(__dirname, 'models', 'dist-tag.js'));
    this.Uplink = sequelize.import(path.join(__dirname, 'models', 'uplink.js'));
    this.Version = sequelize.import(path.join(__dirname, 'models', 'version.js'));

    ['Package', 'DistTag', 'Uplink', 'Version'].forEach((modelName) => {
      if ('associate' in this[modelName]) {
        this[modelName].associate(this);
      }
    });

    // Bootstrap the storage, create the DB and its schema when needed
    // TODO: Ensure that storage is ready, call it in a master process only
    if (this.config.sql.sync) {
      sequelize.sync({force: true}) // {force: true} - drop and create
        .then(() => {
          this.logger.info('DB has been synced successfully.');
        })
        .catch((err) => {
          this.logger.error({err: err}, 'Unable to sync the database:: @{err.message}');
        });
    }

    // sequelize instance reference
    this.sequelize = sequelize;

    // Sequelize object reference
    this.Sequelize = Sequelize;
  }

  getPackage(name, plain = true) {
    return this.Package.find({
      where: {name: {[Op.eq]: name}},
      include: [
        {model: this.DistTag},
        {model: this.Uplink},
        {model: this.Version},
      ],
    }).then((pkg) => {
      return pkg ? (plain ? pkg.get({plain: true}) : pkg) : null;
      // const err = HTTPError[404]('Package not found');
      // err.code = 'ENOENT';
      // return cb(err);
    });
  }

  getPackageAttachment(name, filename) {
    return this.Package.find({
      attributes: ['name'],
      where: {name: {[Op.eq]: name}},
      include: [
        {
          model: this.Version,
          where: {attachment: {[Op.eq]: filename}},
        },
      ],
    });
  }

  getPackageReadme(name) {
    return this.Package.find({
      attributes: ['readme'],
      where: {name: {[Op.eq]: name}},
    }).then((pkg) => {
      return pkg ? pkg.get('readme') : null;
    });
  }

  hasLocalPackage(name, version) {
    // Looking for the specific version of a package
    if (version) {
      return this.Package.find({
        where: {name: {[Op.eq]: name}, local: {[Op.eq]: true}},
        attributes: ['name'],
        include: [{
          model: this.Version,
          where: {version: {[Op.eq]: version}},
        }],
      }).then((pkg) => pkg !== null);
    }

    // Checking for the existence of a package in general
    return this.Package.find({
      where: {name: {[Op.eq]: name}, local: {[Op.eq]: true}},
      attributes: ['name'],
    }).then((pkg) => pkg !== null);
  }

  getLocalPackages() {
    return this.Package.findAll({
      where: {local: {[Op.eq]: true}},
      exclude: ['readme'],
      include: [
        {model: this.DistTag},
      ],
    }).then((pkgs) => pkgs.map((pkg) => pkg.get({plain: true})));
  }

  searchLocalPackages(query, limit = 100) {
    return this.Package.findAll({
      where: {
        [Op.or]: [
          {name: {[Op.like]: `%${query}%`}},
          {description: {[Op.like]: `%${query}%`}}, // Doubt
        ],
        local: {[Op.eq]: true},
      },
      exclude: ['readme'],
      include: [
        {model: this.DistTag},
      ],
      limit: limit,
    }).then((pkgs) => pkgs.map((pkg) => pkg.get({plain: true})));
  }

  updatePackage(name, json) {
    // Package.findOrCreate is not work well here since we need to be sure that package creation will perform in
    // transaction to make the rollback on any possible error
    return this.getPackage(name, false)
      .then((_pkg) => {
        return this.sequelize.transaction((t) => {
          return Promise.resolve(_pkg ? _pkg : this.Package.create(json, {transaction: t}))
            .then((pkg) => {
              const newAttrs = {
                local: Boolean(json.versions &&
                  Object.keys(json.versions).find((v) => !json.versions[v]._valhalla_uplink)),
              };

              ['description', 'author', 'readme', '_rev'].forEach((attrName) => {
                if (pkg[attrName] !== json[attrName] && json[attrName]) {
                  newAttrs[attrName] = json[attrName];
                }
              });

              // calculate revision a la couchdb
              if (typeof(newAttrs._rev) !== 'string' || !newAttrs._rev) {
                newAttrs._rev = pkg.get('_rev') || '0-0000000000000000';
              }

              const rev = newAttrs._rev.split('-');
              newAttrs._rev = ((+rev[0] || 0) + 1) + '-' + Crypto.pseudoRandomBytes(8).toString('hex');

              return pkg.updateAttributes(newAttrs, {transaction: t})
                .then((pkg) => {
                  const changes = [];

                  if (json['dist-tags']) {
                    const disttags = Object.keys(json['dist-tags']).map((key) => this.DistTag.create({
                      name: key,
                      version: json['dist-tags'][key],
                    }, {transaction: t}));

                    changes.push(Promise.all(disttags).then((x) => pkg.setDistTags(x, {transaction: t})));
                  }

                  if (json._uplinks) {
                    const uplinks = Object.keys(json._uplinks).map((key) => this.Uplink.create(Object.assign({name: key},
                      json._uplinks[key]), {transaction: t}));

                    changes.push(Promise.all(uplinks).then((x) => pkg.setUplinks(x, {transaction: t})));
                  }

                  if (json.versions) {
                    const versions = Object.keys(json.versions).map((key) => {
                      // json object is referenced above and may be used later in the flow, clone to not delete the keys
                      const version = Object.assign({}, json.versions[key]);

                      const versionData = {
                        version: key,
                        sha: version.dist.shasum,
                      };

                      if (json.versions[key]._valhalla_uplink) {
                        versionData.url = version.dist.tarball;
                        versionData.registry = json.versions[key]._valhalla_uplink;
                      } else {
                        const parts = name.split('/');
                        versionData.attachment = `${parts.length === 2 ? parts[1] : name}-${key}.tgz`;
                      }

                      delete version._resolved;
                      delete version._shasum;
                      delete version._from;
                      delete version.dist;
                      versionData._raw = JSON.stringify(version);

                      return this.Version.create(versionData, {transaction: t});
                    });

                    changes.push(Promise.all(versions).then((x) => pkg.setVersions(x, {transaction: t})));
                  }

                  return Promise.all(changes);
                });
            });
        })
          .then((result) => {
            // Destroy the previous relationship rows which currently has a null reference
            // TODO: find a better form (sequelize config?) to make the corresponding cleaning
            this.sequelize.transaction((t) => {
              const params = {where: {PackageId: {[Op.eq]: null}}, transaction: t};
              return Promise.all([
                this.DistTag.destroy(params),
                this.Version.destroy(params),
                this.Uplink.destroy(params),
              ]);
            })
              .catch((err) => {
                this.logger.error({err: err}, 'Cleaning was not successful: @{err.message}');
              });

            return result;
          })
          .catch((err) => {
            this.logger.error({err: err}, 'Transaction has been rolled back: @{err.message}');
            throw err;
          });
      });
  }

  updateUplinks(name, json) {
    return this.Package.find({
      where: {name: {[Op.eq]: name}},
      include: [
        {model: this.Uplink},
      ],
    }).then((pkg) => {
      if (!pkg) {
        throw createError(404, `Package ${name} not found`);
      }

      return this.sequelize.transaction((t) => {
        const changes = [];

        if (json._uplinks) {
          const uplinks = Object.keys(json._uplinks).map((key) => this.Uplink.create(Object.assign({name: key},
            json._uplinks[key]), {transaction: t}));

          changes.push(Promise.all(uplinks).then((x) => pkg.setUplinks(x, {transaction: t})));
        }

        return Promise.all(changes);
      })
        .then((result) => {
          // TODO: find a better way to make the corresponding cleaning
          this.sequelize.transaction((t) => {
            return this.Uplink.destroy({where: {PackageId: {[Op.eq]: null}}, transaction: t});
          })
            .catch((err) => {
              this.logger.error({err: err}, 'Cleaning was not successful: @{err.message}');
            });

          return result;
        })
        .catch((err) => {
          throw err;
        });
    });
  }
}

module.exports = DBManager;