bulk-whois-parser
Version:
A lib to parse bulk whois data
386 lines (385 loc) • 21.5 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", {
value: true
});
exports["default"] = void 0;
var _connector = _interopRequireDefault(require("./connector"));
var _fs = _interopRequireDefault(require("fs"));
var _ipSub = _interopRequireDefault(require("ip-sub"));
var _cliProgress = _interopRequireDefault(require("cli-progress"));
var _batchPromises = _interopRequireDefault(require("batch-promises"));
var _moment = _interopRequireDefault(require("moment/moment"));
function _interopRequireDefault(e) { return e && e.__esModule ? e : { "default": e }; }
function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
function _createForOfIteratorHelper(r, e) { var t = "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"]; if (!t) { if (Array.isArray(r) || (t = _unsupportedIterableToArray(r)) || e && r && "number" == typeof r.length) { t && (r = t); var _n = 0, F = function F() {}; return { s: F, n: function n() { return _n >= r.length ? { done: !0 } : { done: !1, value: r[_n++] }; }, e: function e(r) { throw r; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var o, a = !0, u = !1; return { s: function s() { t = t.call(r); }, n: function n() { var r = t.next(); return a = r.done, r; }, e: function e(r) { u = !0, o = r; }, f: function f() { try { a || null == t["return"] || t["return"](); } finally { if (u) throw o; } } }; }
function _slicedToArray(r, e) { return _arrayWithHoles(r) || _iterableToArrayLimit(r, e) || _unsupportedIterableToArray(r, e) || _nonIterableRest(); }
function _nonIterableRest() { throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
function _iterableToArrayLimit(r, l) { var t = null == r ? null : "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"]; if (null != t) { var e, n, i, u, a = [], f = !0, o = !1; try { if (i = (t = t.call(r)).next, 0 === l) { if (Object(t) !== t) return; f = !1; } else for (; !(f = (e = i.call(t)).done) && (a.push(e.value), a.length !== l); f = !0); } catch (r) { o = !0, n = r; } finally { try { if (!f && null != t["return"] && (u = t["return"](), Object(u) !== u)) return; } finally { if (o) throw n; } } return a; } }
function _arrayWithHoles(r) { if (Array.isArray(r)) return r; }
function _toConsumableArray(r) { return _arrayWithoutHoles(r) || _iterableToArray(r) || _unsupportedIterableToArray(r) || _nonIterableSpread(); }
function _nonIterableSpread() { throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
function _unsupportedIterableToArray(r, a) { if (r) { if ("string" == typeof r) return _arrayLikeToArray(r, a); var t = {}.toString.call(r).slice(8, -1); return "Object" === t && r.constructor && (t = r.constructor.name), "Map" === t || "Set" === t ? Array.from(r) : "Arguments" === t || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(t) ? _arrayLikeToArray(r, a) : void 0; } }
function _iterableToArray(r) { if ("undefined" != typeof Symbol && null != r[Symbol.iterator] || null != r["@@iterator"]) return Array.from(r); }
function _arrayWithoutHoles(r) { if (Array.isArray(r)) return _arrayLikeToArray(r); }
function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length); for (var e = 0, n = Array(a); e < a; e++) n[e] = r[e]; return n; }
function _defineProperties(e, r) { for (var t = 0; t < r.length; t++) { var o = r[t]; o.enumerable = o.enumerable || !1, o.configurable = !0, "value" in o && (o.writable = !0), Object.defineProperty(e, _toPropertyKey(o.key), o); } }
function _createClass(e, r, t) { return r && _defineProperties(e.prototype, r), t && _defineProperties(e, t), Object.defineProperty(e, "prototype", { writable: !1 }), e; }
function _classCallCheck(a, n) { if (!(a instanceof n)) throw new TypeError("Cannot call a class as a function"); }
function _callSuper(t, o, e) { return o = _getPrototypeOf(o), _possibleConstructorReturn(t, _isNativeReflectConstruct() ? Reflect.construct(o, e || [], _getPrototypeOf(t).constructor) : o.apply(t, e)); }
function _possibleConstructorReturn(t, e) { if (e && ("object" == _typeof(e) || "function" == typeof e)) return e; if (void 0 !== e) throw new TypeError("Derived constructors may only return object or undefined"); return _assertThisInitialized(t); }
function _assertThisInitialized(e) { if (void 0 === e) throw new ReferenceError("this hasn't been initialised - super() hasn't been called"); return e; }
function _isNativeReflectConstruct() { try { var t = !Boolean.prototype.valueOf.call(Reflect.construct(Boolean, [], function () {})); } catch (t) {} return (_isNativeReflectConstruct = function _isNativeReflectConstruct() { return !!t; })(); }
function _getPrototypeOf(t) { return _getPrototypeOf = Object.setPrototypeOf ? Object.getPrototypeOf.bind() : function (t) { return t.__proto__ || Object.getPrototypeOf(t); }, _getPrototypeOf(t); }
function _inherits(t, e) { if ("function" != typeof e && null !== e) throw new TypeError("Super expression must either be null or a function"); t.prototype = Object.create(e && e.prototype, { constructor: { value: t, writable: !0, configurable: !0 } }), Object.defineProperty(t, "prototype", { writable: !1 }), e && _setPrototypeOf(t, e); }
function _setPrototypeOf(t, e) { return _setPrototypeOf = Object.setPrototypeOf ? Object.setPrototypeOf.bind() : function (t, e) { return t.__proto__ = e, t; }, _setPrototypeOf(t, e); }
function _defineProperty(e, r, t) { return (r = _toPropertyKey(r)) in e ? Object.defineProperty(e, r, { value: t, enumerable: !0, configurable: !0, writable: !0 }) : e[r] = t, e; }
function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == _typeof(i) ? i : i + ""; }
function _toPrimitive(t, r) { if ("object" != _typeof(t) || !t) return t; var e = t[Symbol.toPrimitive]; if (void 0 !== e) { var i = e.call(t, r || "default"); if ("object" != _typeof(i)) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === r ? String : Number)(t); }
var execSync = require('child_process').execSync;
var ConnectorARIN = exports["default"] = /*#__PURE__*/function (_Connector) {
function ConnectorARIN(params) {
var _this;
_classCallCheck(this, ConnectorARIN);
_this = _callSuper(this, ConnectorARIN, [params]);
_defineProperty(_this, "_getStatFile", function () {
console.log("[arin] Downloading stat file");
var cacheFile = "".concat(_this.cacheDir, "arin-stat-file");
return _this._downloadAndReadFile(_this.statFile, cacheFile, _this.daysWhoisCache, false);
});
_defineProperty(_this, "_toPrefix", function (firstIp, hosts) {
var af = _ipSub["default"].getAddressFamily(firstIp);
var bits = af === 4 ? 32 - Math.log2(hosts) : hosts;
return "".concat(firstIp, "/").concat(bits);
});
_defineProperty(_this, "_addSubAllocations", function (stats) {
if (_this.skipSuballocations) {
console.log("[arin] Skipping sub allocations");
return stats;
} else {
var v4File = [_this.cacheDir, "arin-stat-file-v4.json"].join("/").replace("//", "/");
var v6File = [_this.cacheDir, "arin-stat-file-v6.json"].join("/").replace("//", "/");
return _this._addSubAllocationsByType(stats, "ipv4").then(function (v4) {
return _this._writeFile(v4File, v4);
}).then(function (v4) {
return _this._addSubAllocationsByType(stats, "ipv6").then(function (v6) {
return _this._writeFile(v6File, v6);
}).then(function (v6) {
return [].concat(_toConsumableArray(v4), _toConsumableArray(v6));
});
});
}
});
_defineProperty(_this, "_getRemoteSuballocationStatFile", function (type) {
var file = "https://geofeeds.packetvis.com/geolocatemuch/arin-stat-file-".concat(type, ".json");
var cacheFile = _this.getCacheFileName(file);
return _this._downloadAndReadFile(file, cacheFile, _this.daysWhoisCache, true).then(function (response) {
if (response && response.length > 8000) {
return response;
} else {
return Promise.reject("Empty remote sub allocation file");
}
});
});
_defineProperty(_this, "_addSubAllocationByTypeLocally", function (stats, type) {
stats = stats.filter(function (i) {
return i.type === type && i.status === "allocated";
});
var out = stats;
var progressBar = new _cliProgress["default"].SingleBar({}, _cliProgress["default"].Presets.shades_classic);
progressBar.start(stats.length, 0);
return (0, _batchPromises["default"])(1, stats, function (item) {
return _this._whois(item.prefix).then(function (data) {
progressBar.increment();
try {
var ips = data.map(function (i) {
return i.split(" ");
}).filter(function (i) {
return i.length >= 5;
}).map(function (i) {
return i.filter(function (n) {
return _ipSub["default"].isValidIP(n);
});
}).filter(function (i) {
return i.length > 0;
});
for (var _i = 0, _arr = _toConsumableArray(new Set(ips)); _i < _arr.length; _i++) {
var _arr$_i = _slicedToArray(_arr[_i], 2),
firstIp = _arr$_i[0],
lastIp = _arr$_i[1];
var prefixes = _ipSub["default"].ipRangeToCidr(firstIp, lastIp);
var _iterator = _createForOfIteratorHelper(prefixes),
_step;
try {
for (_iterator.s(); !(_step = _iterator.n()).done;) {
var prefix = _step.value;
out.push({
rir: "arin",
type: type,
prefix: prefix,
firstIp: firstIp,
status: "allocated"
});
}
} catch (err) {
_iterator.e(err);
} finally {
_iterator.f();
}
}
} catch (error) {}
})["catch"](console.log);
})["catch"](console.log).then(function () {
progressBar.stop();
var index = {};
var _iterator2 = _createForOfIteratorHelper(out),
_step2;
try {
for (_iterator2.s(); !(_step2 = _iterator2.n()).done;) {
var i = _step2.value;
index[i.firstIp] = i;
}
} catch (err) {
_iterator2.e(err);
} finally {
_iterator2.f();
}
return Object.values(index);
});
});
_defineProperty(_this, "_addSubAllocationsByType", function (stats, type) {
console.log("[arin] Fetching sub allocations ".concat(type));
if (_this.compileSuballocationLocally) {
return _this._addSubAllocationByTypeLocally(stats, type);
} else {
return _this._getRemoteSuballocationStatFile(type)["catch"](function () {
console.log("[arin] It was not possible to download precompiled sub allocations ".concat(type, ", I will try to compile them from whois instead"));
return _this._addSubAllocationByTypeLocally(stats, type);
});
}
});
_defineProperty(_this, "_whois", function (prefix) {
var file = _this.getCacheFileName("whois-prefix-".concat(prefix));
if (_this._isCacheValid(file, _this._getDistributedCacheTime())) {
return _this._readFile(file, true);
} else {
return new Promise(function (resolve, reject) {
var flag = "h";
var output = execSync("whois -".concat(flag, " whois.arin.net \"r > ").concat(prefix, "\""), {
encoding: 'utf-8'
});
_this._writeFile(file, output.split("\n")).then(resolve);
});
}
});
_defineProperty(_this, "_compileNetRangesListLocally", function () {
return _this._getStatFile().then(function (data) {
var structuredData = data.split("\n").filter(function (line) {
return line.includes("ipv4") || line.includes("ipv6");
}).map(function (line) {
return line.split("|");
}).map(function (_ref) {
var _ref2 = _slicedToArray(_ref, 8),
rir = _ref2[0],
cc = _ref2[1],
type = _ref2[2],
firstIpUp = _ref2[3],
hosts = _ref2[4],
date = _ref2[5],
status = _ref2[6],
hash = _ref2[7];
var firstIp = firstIpUp.toLowerCase();
return {
rir: rir,
type: type,
prefix: _this._toPrefix(firstIp, hosts),
firstIp: firstIp,
hosts: hosts,
date: date,
status: status
};
}).filter(function (i) {
return i.rir === "arin" && ["ipv4", "ipv6"].includes(i.type) && ["allocated", "assigned"].includes(i.status);
});
return structuredData.reverse();
}).then(_this._addSubAllocations);
});
_defineProperty(_this, "_createWhoisDump", function (types) {
if (_this._isCacheValid(_this.cacheFile, 1)) {
console.log("[arin] Using cached whois data: ".concat(types));
return Promise.resolve(JSON.parse(_fs["default"].readFileSync(_this.cacheFile, 'utf-8')));
} else {
return _this.getRemotePreFilteredNetRanges()["catch"](_this._compileNetRangesListLocally).then(_this._toStandardFormat).then(function (inetnums) {
return inetnums.filter(function (i) {
return !!i;
});
}).then(function (inetnums) {
return _this._writeFile(_this.cacheFile, inetnums);
});
}
});
_defineProperty(_this, "_getDistributedCacheTime", function () {
var rndInt = Math.floor(Math.random() * parseInt(_this.daysWhoisSuballocationsCache / 2)) + 1;
return Math.max(_this.daysWhoisCache, _this.daysWhoisSuballocationsCache - rndInt);
});
_defineProperty(_this, "_getRdapQuery", function (prefix) {
var url = "https://rdap.arin.net/registry/ip/".concat(prefix);
var file = _this.getCacheFileName(url);
return _this._downloadAndReadFile(url, file, _this._getDistributedCacheTime(), true)["catch"](function (error) {
console.log("Cannot retrieve ".concat(prefix, ": ").concat(error));
return null;
});
});
_defineProperty(_this, "_toStandardFormat", function (items) {
console.log("[arin] Fetching NetRanges");
var progressBar = new _cliProgress["default"].SingleBar({}, _cliProgress["default"].Presets.shades_classic);
progressBar.start(items.length, 0);
var singleBatch = function singleBatch(items) {
return (0, _batchPromises["default"])(4, items, function (item) {
return _this._getRdapQuery(item.firstIp).then(function (data) {
progressBar.increment();
if (data) {
var startAddress = data.startAddress,
endAddress = data.endAddress,
remarks = data.remarks,
events = data.events;
var inetnum = {};
if (remarks) {
var remarksArray = remarks.map(function (remark) {
return remark.description || [];
});
var cleanRemarks = remarksArray.flat().filter(function (i) {
return i.toLowerCase().includes("geofeed");
});
if (cleanRemarks !== null && cleanRemarks !== void 0 && cleanRemarks.length) {
var af = _ipSub["default"].getAddressFamily(startAddress);
if (af === 4) {
inetnum.inetnum = "".concat(startAddress, " - ").concat(endAddress);
inetnum.type = "inetnum";
} else {
inetnum.inet6num = item.prefix;
inetnum.type = "inet6num";
}
var lastChanges = (events || []).filter(function (i) {
return i.eventAction === "last changed";
}).pop();
inetnum["last-modified"] = lastChanges ? lastChanges.eventDate : null;
inetnum.remarks = cleanRemarks;
for (var prop in data) {
if (typeof data[prop] === "string" && !inetnum[prop]) {
inetnum[prop] = data[prop];
}
}
return inetnum;
}
}
}
return null;
});
});
};
var halfList = Math.ceil(items.length / 2);
return Promise.all([singleBatch(items.slice(0, halfList)), singleBatch(items.slice(halfList))]).then(function (inetnums) {
progressBar.stop();
return inetnums.flat();
});
});
_defineProperty(_this, "getRemotePreFilteredNetRanges", function () {
if (_this.compileSuballocationLocally) {
return Promise.reject("Compile locally");
}
var url = "https://geofeeds.packetvis.com/geolocatemuch/arin.inetnums";
var metadataUrl = "https://geofeeds.packetvis.com/geolocatemuch/metadata.json";
var file = _this.getCacheFileName(url);
var metadataFile = _this.getCacheFileName(metadataUrl);
return _this._downloadAndReadFile(metadataUrl, metadataFile, 1, true).then(function (metadata) {
var lastUpdate = _moment["default"].unix(metadata.lastUpdate);
if ((0, _moment["default"])((0, _moment["default"])()).diff(lastUpdate, 'days') <= 10) {
return _this._downloadAndReadFile(url, file, 1, true);
} else {
return Promise.reject("Remote file is too old");
}
}).then(function (data) {
console.log("[arin] Using pre filtered inetnums");
return data.map(function (_ref3) {
var startAddress = _ref3.startAddress,
ipVersion = _ref3.ipVersion,
inet6num = _ref3.inet6num;
return {
firstIp: startAddress,
prefix: ipVersion === "v6" ? inet6num : null
};
});
});
});
_defineProperty(_this, "getObjects", function (types, filterFunction, fields, forEachFunction) {
if (_this.params.arinBulk) {
console.log("[arin] bulk whois data not yet supported");
return Promise.resolve([]);
} else {
return _this._createWhoisDump(types).then(function (data) {
var filtered = data.filter(function (i) {
return types.includes(i.type) && filterFunction(i);
});
if (fields && fields.length) {
return filtered.map(function (item) {
var out = {};
for (var k in item) {
if (fields.includes(k)) {
out[k] = item[k];
}
}
return out;
});
} else {
return filtered;
}
}).then(function (data) {
if (!!forEachFunction) {
var _iterator3 = _createForOfIteratorHelper(data),
_step3;
try {
for (_iterator3.s(); !(_step3 = _iterator3.n()).done;) {
var i = _step3.value;
forEachFunction(i);
}
} catch (err) {
_iterator3.e(err);
} finally {
_iterator3.f();
}
return [];
}
return data;
});
}
});
_this.connectorName = "arin-rir";
_this.cacheDir += _this.connectorName + "/";
_this.statFile = "http://ftp.arin.net/pub/stats/arin/delegated-arin-extended-latest";
_this.cacheFile = [_this.cacheDir, "arin.inetnums"].join("/").replace("//", "/");
_this.daysWhoisCache = _this.params.defaultCacheDays || 7;
_this.daysWhoisSuballocationsCache = _this.params.daysWhoisSuballocationsCache || 30;
_this.skipSuballocations = !!_this.params.skipSuballocations;
_this.compileSuballocationLocally = !!_this.params.compileSuballocationLocally;
if (_this.daysWhoisSuballocationsCache < 7) {
console.log("Sub allocations in ARIN cannot be fetched more than once every 7 days. Using 7 days.");
_this.daysWhoisSuballocationsCache = 7;
}
if (_this.daysWhoisSuballocationsCache < _this.daysWhoisCache) {
console.log("Sub allocations in ARIN cannot be fetched more than once every ".concat(_this.daysWhoisCache, " days. Using ").concat(_this.daysWhoisCache, " days."));
_this.daysWhoisSuballocationsCache = _this.daysWhoisCache;
}
if (_this.daysWhoisCache < 3) {
console.log("NetRanges in ARIN cannot be fetched more than once every 3 days. Using 3 days.");
_this.daysWhoisCache = 3;
}
if (!_fs["default"].existsSync(_this.cacheDir)) {
_fs["default"].mkdirSync(_this.cacheDir, {
recursive: true
});
}
return _this;
}
_inherits(ConnectorARIN, _Connector);
return _createClass(ConnectorARIN);
}(_connector["default"]);