webseeded-torrent-generator/lib/http-hasher.js

Generate webseeded torrent files from urls to files.

'use strict';

var crypto = require('crypto');
var q = require('q');
var qsequence = require('../util/qsequence');
var _ = require('lodash');
var assert = require('assert');
var BunyanPromiseLogger = require('bunyan-promise');

var promiseLog = new BunyanPromiseLogger({
    name: 'http-hasher'
});

var REQUEST_SIZE = 0x1000000;

// Resolves when entire file has been read
// Sends progress notification for each buffer
var requestUrl = function (file) {
    var defer = q.defer();

    var fileSizeRequest = file.size();
    fileSizeRequest.then(function (fileSize) {
        var numPieces = Math.ceil(fileSize / REQUEST_SIZE);

        var queue = q.resolve();
        _.times(numPieces, function (pieceIndex) {
            var requestStart = pieceIndex * REQUEST_SIZE;
            var requestLength = Math.min(REQUEST_SIZE, fileSize - requestStart);
            queue = queue.then(function () {
                var ret = file.read(requestStart, requestLength).then(function (buffer) {
                    defer.notify(buffer);
                    return q.resolve();
                });
                return ret;
            });
        });
        return queue;
    }).then(function () {
        defer.resolve();
    }, function (err) {
        defer.reject(err);
    });
    promiseLog.trace(defer.promise, 'request url - ' + file.url);
    return defer.promise;
};

// Resolves when all files have been read
// Sends progress notifications for each buffer in sequence
var requestUrls = function (files) {
    var queue = q.resolve();
    _.each(files, function (file) {
        queue = queue.then(function () {
            return requestUrl(file);
        });
    }, this);
    promiseLog.trace(queue, 'request urls');
    return queue;
};

var Hasher = function (files, pieceSize) {
    this.files = files;
    this.pieceSize = pieceSize;
};

Hasher.prototype.hash = function () {
    var defer = q.defer();
    var unhashedData = new Buffer(0);
    var pieceHashes = [];

    var sizesRequest = qsequence(_.map(this.files, function (file) {
        return function () {
            return file.size();
        };
    }));

    sizesRequest.then(function (sizes) {
        var totalSize = _.reduce(sizes, function (memo, size) {
            return memo + size;
        }, 0);
        var hashedDataSize = 0;

        var hashData = function (size) {
            var unhashedDataChunk = unhashedData.slice(0, size);
            unhashedData = unhashedData.slice(size);

            var shasum = crypto.createHash('sha1');
            shasum.update(unhashedDataChunk);
            var pieceHash = shasum.digest('hex');
            pieceHashes.push(pieceHash);

            hashedDataSize += unhashedDataChunk.length;
            defer.notify(100 * hashedDataSize / totalSize);
        };

        var requestUrlsRequest = requestUrls(this.files);
        requestUrlsRequest.then(function () {
            if (unhashedData.length > 0) {
                hashData(unhashedData.length);
            }

            assert(hashedDataSize === totalSize, 'expected to hash all the data before resolving');
            assert(unhashedData.length === 0, 'expect to not have any unhashed data remaining before resolving');

            defer.resolve(pieceHashes);
        });

        requestUrlsRequest.progress(function (data) {
            unhashedData = Buffer.concat([unhashedData, data]);

            while (unhashedData.length > this.pieceSize) {
                hashData(this.pieceSize);
            }
        }.bind(this));
    }.bind(this));


    promiseLog.trace(defer.promise, 'hash');

    return defer.promise;
};

module.exports = Hasher;