bulk-whois-parser
Version:
A lib to parse bulk whois data
254 lines (252 loc) • 11.8 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", {
value: true
});
exports["default"] = void 0;
var _md = _interopRequireDefault(require("md5"));
var _fs = _interopRequireDefault(require("fs"));
var _readline = _interopRequireDefault(require("readline"));
var _zlib = _interopRequireDefault(require("zlib"));
var _moment = _interopRequireDefault(require("moment"));
function _interopRequireDefault(e) { return e && e.__esModule ? e : { "default": e }; }
function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
function _slicedToArray(r, e) { return _arrayWithHoles(r) || _iterableToArrayLimit(r, e) || _unsupportedIterableToArray(r, e) || _nonIterableRest(); }
function _nonIterableRest() { throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
function _unsupportedIterableToArray(r, a) { if (r) { if ("string" == typeof r) return _arrayLikeToArray(r, a); var t = {}.toString.call(r).slice(8, -1); return "Object" === t && r.constructor && (t = r.constructor.name), "Map" === t || "Set" === t ? Array.from(r) : "Arguments" === t || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(t) ? _arrayLikeToArray(r, a) : void 0; } }
function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length); for (var e = 0, n = Array(a); e < a; e++) n[e] = r[e]; return n; }
function _iterableToArrayLimit(r, l) { var t = null == r ? null : "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"]; if (null != t) { var e, n, i, u, a = [], f = !0, o = !1; try { if (i = (t = t.call(r)).next, 0 === l) { if (Object(t) !== t) return; f = !1; } else for (; !(f = (e = i.call(t)).done) && (a.push(e.value), a.length !== l); f = !0); } catch (r) { o = !0, n = r; } finally { try { if (!f && null != t["return"] && (u = t["return"](), Object(u) !== u)) return; } finally { if (o) throw n; } } return a; } }
function _arrayWithHoles(r) { if (Array.isArray(r)) return r; }
function _defineProperties(e, r) { for (var t = 0; t < r.length; t++) { var o = r[t]; o.enumerable = o.enumerable || !1, o.configurable = !0, "value" in o && (o.writable = !0), Object.defineProperty(e, _toPropertyKey(o.key), o); } }
function _createClass(e, r, t) { return r && _defineProperties(e.prototype, r), t && _defineProperties(e, t), Object.defineProperty(e, "prototype", { writable: !1 }), e; }
function _classCallCheck(a, n) { if (!(a instanceof n)) throw new TypeError("Cannot call a class as a function"); }
function _defineProperty(e, r, t) { return (r = _toPropertyKey(r)) in e ? Object.defineProperty(e, r, { value: t, enumerable: !0, configurable: !0, writable: !0 }) : e[r] = t, e; }
function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == _typeof(i) ? i : i + ""; }
function _toPrimitive(t, r) { if ("object" != _typeof(t) || !t) return t; var e = t[Symbol.toPrimitive]; if (void 0 !== e) { var i = e.call(t, r || "default"); if ("object" != _typeof(i)) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === r ? String : Number)(t); }
var urlParser = require('url');
var https = require('https');
var http = require('http');
var agentOptions = {
family: 4,
keepAlive: true,
maxSockets: 240,
keepAliveMsecs: 50000000,
maxFreeSockets: 256,
rejectUnauthorized: false
};
var proto = {
https: {
fetch: https,
agent: new https.Agent(agentOptions)
},
http: {
fetch: http,
agent: new http.Agent(agentOptions)
}
};
var Connector = exports["default"] = /*#__PURE__*/_createClass(function Connector(params) {
var _this = this;
_classCallCheck(this, Connector);
_defineProperty(this, "_readLines", function (compressedFile, type, filterFunction) {
var fields = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : [];
var forEachFunction = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : null;
return new Promise(function (resolve, reject) {
if (!filterFunction) {
reject(new Error("You MUST specify a filter function"));
}
var lastObject = null;
var objects = [];
var lineReader = _readline["default"].createInterface({
input: _fs["default"].createReadStream(compressedFile).pipe(_zlib["default"].createGunzip()).on("error", function (error) {
console.log(error);
console.log("ERROR: Delete the cache file ".concat(compressedFile));
})
});
lineReader.on('line', function (line) {
if (!lastObject && line.startsWith(type + ":")) {
// new object
var _this$getStandardForm = _this.getStandardFormat(_this.getKeyValue(line)),
_this$getStandardForm2 = _slicedToArray(_this$getStandardForm, 2),
key = _this$getStandardForm2[0],
value = _this$getStandardForm2[1];
lastObject = _defineProperty({}, key, value);
} else if (lastObject && line.length === 0) {
// end of object
var objTmp = _this.getStandardObject(lastObject);
if (filterFunction(objTmp)) {
if (!!forEachFunction) {
forEachFunction(objTmp);
} else {
objects.push(objTmp);
}
}
lastObject = null;
} else if (lastObject) {
var _this$getStandardForm3 = _this.getStandardFormat(_this.getKeyValue(line)),
_this$getStandardForm4 = _slicedToArray(_this$getStandardForm3, 2),
_key = _this$getStandardForm4[0],
_value = _this$getStandardForm4[1];
if (_key && (!fields.length || fields.includes(_key))) {
if (lastObject[_key]) {
if (!Array.isArray(lastObject[_key])) {
lastObject[_key] = [lastObject[_key]];
}
lastObject[_key].push(_value);
} else {
lastObject[_key] = _value;
}
}
}
}).on("error", function (error) {
if (_this.params.deleteCorruptedCacheFile) {
try {
_fs["default"].unlinkSync(compressedFile);
} catch (error) {
console.log("Corrupted file ".concat(compressedFile, " already deleted"));
}
}
return reject(error, "Delete the cache file ".concat(compressedFile));
}).on("close", function () {
resolve(objects);
});
});
});
_defineProperty(this, "getKeyValue", function (line) {
return line.split(/:(.+)/).filter(function (i) {
return i.length;
}).map(function (i) {
return i.trim();
});
});
_defineProperty(this, "getStandardFormat", function (_ref) {
var _ref2 = _slicedToArray(_ref, 2),
key = _ref2[0],
value = _ref2[1];
if (key && value && !key.startsWith("#") && !key.startsWith("%")) {
return [key, value];
}
return [null, null];
});
_defineProperty(this, "getStandardObject", function (object) {
if (object.remarks) {
if (!Array.isArray(object.remarks)) {
object.remarks = [object.remarks];
}
}
if (object.members) {
if (!Array.isArray(object.members)) {
object.members = [object.members];
}
}
return object;
});
_defineProperty(this, "getCacheFileName", function (originalName) {
return [_this.cacheDir, (0, _md["default"])(originalName)].join("/").replace("//", "/");
});
_defineProperty(this, "_isCacheValid", function (file, days) {
var _file, _days;
file = (_file = file) !== null && _file !== void 0 ? _file : _this.cacheFile;
days = (_days = days) !== null && _days !== void 0 ? _days : _this.daysWhoisCache;
if (_fs["default"].existsSync(file)) {
var stats = _fs["default"].statSync(file);
var lastDownloaded = (0, _moment["default"])(stats.ctime);
if ((0, _moment["default"])((0, _moment["default"])()).diff(lastDownloaded, 'days') <= days) {
return true;
}
}
return false;
});
_defineProperty(this, "_writeFile", function (file, data) {
if (_typeof(data) === "object") {
_fs["default"].writeFileSync(file, JSON.stringify(data));
} else {
_fs["default"].writeFileSync(file, data);
}
return Promise.resolve(data);
});
_defineProperty(this, "_readFile", function (file, json) {
return new Promise(function (resolve, reject) {
try {
var content = _fs["default"].readFileSync(file, 'utf-8');
if (json) {
content = JSON.parse(content);
}
resolve(content);
} catch (error) {
reject(error);
}
});
});
_defineProperty(this, "_downloadAndReadFile", function (url, file) {
var days = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 1;
var json = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : true;
if (!_this._isCacheValid(file, days)) {
return _this._downloadFile(url, file).then(function () {
return _this._readFile(file, json);
});
} else {
return _this._readFile(file, json)["catch"](function () {
return _this._downloadFile(url, file).then(function () {
return _this._readFile(file, json);
});
});
}
});
_defineProperty(this, "_downloadFile", function (url, file) {
return new Promise(function (resolve, reject) {
var fileStream = _fs["default"].createWriteStream(file);
var protocol = url.toLowerCase().split(":")[0];
var options = urlParser.parse(url);
options.agent = proto[protocol].agent;
options.method = 'GET';
options.gzip = true;
options.timeout = 20000;
options.keepAliveTimeout = 600000000;
proto[protocol].fetch.get(options, function (response) {
response.pipe(fileStream);
fileStream.on('finish', function (_) {
resolve(file);
});
fileStream.on('error', function (error) {
reject(error);
});
}).on('error', function (error) {
reject(error);
});
});
});
_defineProperty(this, "_getDump", function () {
if (_this._isCacheValid()) {
console.log("[".concat(_this.connectorName, "] Using cached whois data"));
return Promise.resolve(_this.cacheFile);
} else {
console.log("[".concat(_this.connectorName, "] Downloading whois data"));
return _this._downloadFile(_this.dumpUrl, _this.cacheFile)["catch"](function (error) {
console.log("Delete the cache file ".concat(_this.cacheFile));
return Promise.reject(error);
});
}
});
_defineProperty(this, "getObjects", function (types, filterFunction, fields, forEachFunction) {
fields = fields || [];
return _this._getDump().then(function (file) {
console.log("[".concat(_this.connectorName, "] Parsing whois data: ").concat(types));
return Promise.all(types.map(function (type) {
return _this._readLines(file, type, filterFunction, fields, forEachFunction);
})).then(function (objects) {
return objects.flat();
});
});
});
this.params = params || {};
this.userAgent = this.params.userAgent || "bulk-whois-parser";
this.connectorName = "connector";
this.cacheDir = this.params.cacheDir || ".cache/";
this.cacheFile = null;
this.dumpUrl = null;
this.daysWhoisCache = this.params.defaultCacheDays || 1;
if (!_fs["default"].existsSync(this.cacheDir)) {
_fs["default"].mkdirSync(this.cacheDir, {
recursive: true
});
}
});