firmament-vita
Version:
Firmament module for automating VITA tasks
248 lines • 11.3 kB
JavaScript
;
var __extends = (this && this.__extends) || (function () {
var extendStatics = Object.setPrototypeOf ||
({ __proto__: [] } instanceof Array && function (d, b) { d.__proto__ = b; }) ||
function (d, b) { for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p]; };
return function (d, b) {
extendStatics(d, b);
function __() { this.constructor = d; }
d.prototype = b === null ? Object.create(b) : (__.prototype = b.prototype, new __());
};
})();
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
return c > 3 && r && Object.defineProperty(target, key, r), r;
};
var __metadata = (this && this.__metadata) || function (k, v) {
if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
};
var __param = (this && this.__param) || function (paramIndex, decorator) {
return function (target, key) { decorator(target, key, paramIndex); }
};
Object.defineProperty(exports, "__esModule", { value: true });
var inversify_1 = require("inversify");
var firmament_yargs_1 = require("firmament-yargs");
var fs = require('fs');
var async = require('async');
var path = require('path');
var mkdirp = require('mkdirp');
var VitaPcapMergeImpl = (function (_super) {
__extends(VitaPcapMergeImpl, _super);
function VitaPcapMergeImpl(commandUtil, vitaFileUtil, vitaSpawn) {
var _this = _super.call(this) || this;
_this.commandUtil = commandUtil;
_this.vitaFileUtil = vitaFileUtil;
_this.vitaSpawn = vitaSpawn;
_this.useDummyMerge = false;
_this.counter = 0;
return _this;
}
VitaPcapMergeImpl_1 = VitaPcapMergeImpl;
VitaPcapMergeImpl.prototype.mergePcapFiles = function (mergePcapFileOptions, cbStatus, cbFinal) {
var _this = this;
cbStatus = this.checkCallback(cbStatus);
cbFinal = this.checkCallback(cbFinal);
var mergedPcapFile = mergePcapFileOptions.mergedPcapFile;
var pcapFiles = mergePcapFileOptions.pcapFiles;
var totalBytes = VitaPcapMergeImpl_1.sumOfFileSizes(pcapFiles);
var bytesWritten = 0;
var args = ['-w', '-'];
mkdirp(path.dirname(mergedPcapFile));
var writeFileStream = fs.createWriteStream(mergedPcapFile);
args = args.concat(pcapFiles);
this.vitaSpawn.spawnIt('mergecap', args, function (err, result) {
bytesWritten += result.length;
var taskName = "Creating " + mergedPcapFile;
var current = bytesWritten;
var total = totalBytes;
cbStatus(null, { taskName: taskName, current: current, total: total });
writeFileStream.write(result);
}, function (err, result) {
writeFileStream.close();
if (mergePcapFileOptions.deleteOriginalPcapFiles) {
_this.vitaFileUtil.deleteFiles(mergePcapFileOptions.pcapFiles);
}
cbFinal(null, { error: err, mergedPcapFile: mergedPcapFile });
});
};
VitaPcapMergeImpl.sumOfFileSizes = function (files) {
var retVal = 0;
files.forEach(function (file) {
retVal += VitaPcapMergeImpl_1.getFilesizeInBytes(file);
});
return retVal;
};
VitaPcapMergeImpl.getFilesizeInBytes = function (filename) {
var stats = fs.statSync(filename);
var fileSizeInBytes = stats["size"];
return fileSizeInBytes;
};
VitaPcapMergeImpl.prototype.mergePcapFilesParallel = function (files, mergedFilename, cb) {
var _this = this;
var me = this;
cb = this.checkCallback(cb);
files = files || [];
var columns = [];
var mask = 0x80000000;
while (files.length) {
while (mask && !(mask & files.length)) {
mask >>>= 1;
}
columns.push(files.slice(0, mask));
files = files.slice(mask);
}
var fnArray = [];
for (var i = 0; i < columns.length; ++i) {
var column = columns[i];
fnArray.push(async.apply(this.reducePowerOf2FileListToOneFile.bind(this), column, me.useDummyMerge));
}
async.series(fnArray, function (err, results) {
var me = _this;
files = results.map(function (result) { return result.outFile; });
if (files.length > 1) {
_this.mergePcapFilesParallel(files, mergedFilename, function (err, res) {
cb(null, res);
});
}
else {
if (results.length !== 1) {
cb(new Error('Bad results length'));
return;
}
if (!me.useDummyMerge) {
fs.rename(results[0].outFile, mergedFilename, function (err) {
if (me.commandUtil.callbackIfError(cb, err)) {
return;
}
fs.stat(mergedFilename, function (err, stats) {
cb(null, [{
error: err,
inFile0: '',
inFile1: '',
outFile: mergedFilename,
outFileStats: JSON.stringify(stats)
}]);
});
});
}
else {
cb(null, [{ error: err, inFile0: '', inFile1: '', outFile: mergedFilename, outFileStats: 'Using DummyMerge' }]);
}
}
});
};
VitaPcapMergeImpl.prototype.reducePowerOf2FileListToOneFile = function (files, useDummyMerge, cb) {
var _this = this;
var me = this;
var triangleReduceRows = [];
me.buildMergeTupleArray(files, 0, triangleReduceRows);
var fnRowReduceSeriesArray = [];
var _loop_1 = function (j) {
var triangleReduceRow = triangleReduceRows[j];
var fnRowReduceParallelArray = [];
triangleReduceRow.forEach(function (twoToOneReduce) {
var inFile0 = twoToOneReduce.inFile0;
var inFile1 = twoToOneReduce.inFile1;
var outFile = twoToOneReduce.outFile;
if (!useDummyMerge) {
if (!inFile1) {
fnRowReduceParallelArray.push(async.apply(me.vitaFileUtil.copyFile.bind(me.vitaFileUtil), inFile0, outFile));
}
else {
fnRowReduceParallelArray.push(async.apply(me.spawnMergeTwoPcapsOperation.bind(me), inFile0, inFile1, outFile));
}
}
else {
fnRowReduceParallelArray.push(async.apply(VitaPcapMergeImpl_1.dummyMergeOperation.bind(me), inFile0, inFile1, outFile));
}
});
fnRowReduceSeriesArray.push(async.apply(async.parallelLimit, fnRowReduceParallelArray, 3));
};
for (var j = 0; j < triangleReduceRows.length; ++j) {
_loop_1(j);
}
async.series(fnRowReduceSeriesArray, function (err, resultAryAry) {
var resultAry = resultAryAry[resultAryAry.length - 1];
var retVal = resultAry[0];
var filesToDelete = [];
while (resultAry = resultAryAry.pop()) {
resultAry.forEach(function (result) {
if (VitaPcapMergeImpl_1.isIntermediateFile(result.inFile0)) {
filesToDelete.push(result.inFile0);
}
if (VitaPcapMergeImpl_1.isIntermediateFile(result.inFile1)) {
filesToDelete.push(result.inFile1);
}
});
}
_this.vitaFileUtil.deleteFiles(filesToDelete);
cb(err, retVal);
});
};
VitaPcapMergeImpl.prototype.buildMergeTupleArray = function (files, depth, parallelTuples) {
var tuples = [];
for (var i = 0; i < files.length; i += 2) {
var inFile0 = files[i];
var inFile1 = files[i + 1];
var workingDir = path.dirname(inFile0);
var baseFile = path.basename(inFile0);
var outFile = this.getIntermediateFileName(baseFile);
outFile = path.resolve(workingDir, outFile);
tuples.push({ inFile0: inFile0, inFile1: inFile1, outFile: outFile });
}
parallelTuples.push(tuples);
files = tuples.map(function (tuple) { return tuple.outFile; });
if (files.length > 1) {
this.buildMergeTupleArray(files, depth + 1, parallelTuples);
}
};
VitaPcapMergeImpl.isIntermediateFile = function (fileName) {
return (/\d{6}/.test(fileName) && /\.tmp$/.test(fileName));
};
VitaPcapMergeImpl.prototype.getIntermediateFileName = function (fileName) {
var retVal;
var prefix = VitaPcapMergeImpl_1.zeroFill(this.counter++, 6);
if (VitaPcapMergeImpl_1.isIntermediateFile(fileName)) {
retVal = "" + prefix + fileName.substr(6);
}
else {
retVal = prefix + "_" + fileName + ".tmp";
}
this.commandUtil.log(retVal);
return retVal;
};
VitaPcapMergeImpl.dummyMergeOperation = function (inFile0, inFile1, outFile, cb) {
cb(null, { error: null, inFile0: inFile0, inFile1: inFile1, outFile: outFile, outFileStats: 'Using DummyMerge' });
};
VitaPcapMergeImpl.prototype.spawnMergeTwoPcapsOperation = function (inFile0, inFile1, outFile, cb) {
this.vitaSpawn.spawnIt('mergecap', [
'-w',
outFile,
inFile0,
inFile1
], function (err, result) {
}, function (err, result) {
cb(null, { error: err, inFile0: inFile0, inFile1: inFile1, outFile: outFile, outFileStats: 'Nobody Cares (intermediate file)' });
});
};
VitaPcapMergeImpl.zeroFill = function (number, width) {
width -= number.toString().length;
if (width > 0) {
return new Array(width + (/\./.test(number) ? 2 : 1)).join('0') + number;
}
return number + "";
};
VitaPcapMergeImpl = VitaPcapMergeImpl_1 = __decorate([
inversify_1.injectable(),
__param(0, inversify_1.inject('CommandUtil')),
__param(1, inversify_1.inject('VitaFileUtil')),
__param(2, inversify_1.inject('VitaSpawn')),
__metadata("design:paramtypes", [Object, Object, Object])
], VitaPcapMergeImpl);
return VitaPcapMergeImpl;
var VitaPcapMergeImpl_1;
}(firmament_yargs_1.ForceErrorImpl));
exports.VitaPcapMergeImpl = VitaPcapMergeImpl;
//# sourceMappingURL=vita-pcap-merge-impl.js.map