whois-link-finder
Version:
A tool to find files in whois
356 lines (353 loc) • 17.4 kB
JavaScript
;
Object.defineProperty(exports, "__esModule", {
value: true
});
exports["default"] = void 0;
var _batchPromises = _interopRequireDefault(require("batch-promises"));
var _redaxios = _interopRequireDefault(require("redaxios"));
var _bulkWhoisParser = _interopRequireDefault(require("bulk-whois-parser"));
var _longestPrefixMatch = _interopRequireDefault(require("longest-prefix-match"));
var _md = _interopRequireDefault(require("md5"));
var _fs = _interopRequireDefault(require("fs"));
var _moment = _interopRequireDefault(require("moment"));
var _ipSub = _interopRequireDefault(require("ip-sub"));
function _interopRequireDefault(e) { return e && e.__esModule ? e : { "default": e }; }
function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
function ownKeys(e, r) { var t = Object.keys(e); if (Object.getOwnPropertySymbols) { var o = Object.getOwnPropertySymbols(e); r && (o = o.filter(function (r) { return Object.getOwnPropertyDescriptor(e, r).enumerable; })), t.push.apply(t, o); } return t; }
function _objectSpread(e) { for (var r = 1; r < arguments.length; r++) { var t = null != arguments[r] ? arguments[r] : {}; r % 2 ? ownKeys(Object(t), !0).forEach(function (r) { _defineProperty(e, r, t[r]); }) : Object.getOwnPropertyDescriptors ? Object.defineProperties(e, Object.getOwnPropertyDescriptors(t)) : ownKeys(Object(t)).forEach(function (r) { Object.defineProperty(e, r, Object.getOwnPropertyDescriptor(t, r)); }); } return e; }
function _toConsumableArray(r) { return _arrayWithoutHoles(r) || _iterableToArray(r) || _unsupportedIterableToArray(r) || _nonIterableSpread(); }
function _nonIterableSpread() { throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
function _iterableToArray(r) { if ("undefined" != typeof Symbol && null != r[Symbol.iterator] || null != r["@@iterator"]) return Array.from(r); }
function _arrayWithoutHoles(r) { if (Array.isArray(r)) return _arrayLikeToArray(r); }
function _createForOfIteratorHelper(r, e) { var t = "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"]; if (!t) { if (Array.isArray(r) || (t = _unsupportedIterableToArray(r)) || e && r && "number" == typeof r.length) { t && (r = t); var _n = 0, F = function F() {}; return { s: F, n: function n() { return _n >= r.length ? { done: !0 } : { done: !1, value: r[_n++] }; }, e: function e(r) { throw r; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var o, a = !0, u = !1; return { s: function s() { t = t.call(r); }, n: function n() { var r = t.next(); return a = r.done, r; }, e: function e(r) { u = !0, o = r; }, f: function f() { try { a || null == t["return"] || t["return"](); } finally { if (u) throw o; } } }; }
function _unsupportedIterableToArray(r, a) { if (r) { if ("string" == typeof r) return _arrayLikeToArray(r, a); var t = {}.toString.call(r).slice(8, -1); return "Object" === t && r.constructor && (t = r.constructor.name), "Map" === t || "Set" === t ? Array.from(r) : "Arguments" === t || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(t) ? _arrayLikeToArray(r, a) : void 0; } }
function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length); for (var e = 0, n = Array(a); e < a; e++) n[e] = r[e]; return n; }
function _defineProperties(e, r) { for (var t = 0; t < r.length; t++) { var o = r[t]; o.enumerable = o.enumerable || !1, o.configurable = !0, "value" in o && (o.writable = !0), Object.defineProperty(e, _toPropertyKey(o.key), o); } }
function _createClass(e, r, t) { return r && _defineProperties(e.prototype, r), t && _defineProperties(e, t), Object.defineProperty(e, "prototype", { writable: !1 }), e; }
function _classCallCheck(a, n) { if (!(a instanceof n)) throw new TypeError("Cannot call a class as a function"); }
function _defineProperty(e, r, t) { return (r = _toPropertyKey(r)) in e ? Object.defineProperty(e, r, { value: t, enumerable: !0, configurable: !0, writable: !0 }) : e[r] = t, e; }
function _toPropertyKey(t) { var i = _toPrimitive(t, "string"); return "symbol" == _typeof(i) ? i : i + ""; }
function _toPrimitive(t, r) { if ("object" != _typeof(t) || !t) return t; var e = t[Symbol.toPrimitive]; if (void 0 !== e) { var i = e.call(t, r || "default"); if ("object" != _typeof(i)) return i; throw new TypeError("@@toPrimitive must return a primitive value."); } return ("string" === r ? String : Number)(t); }
require("events").EventEmitter.defaultMaxListeners = 200;
var Finder = exports["default"] = /*#__PURE__*/_createClass(function Finder(params) {
var _this = this;
_classCallCheck(this, Finder);
_defineProperty(this, "filterFunction", function (inetnum) {
var _iterator = _createForOfIteratorHelper(_this.specialKeys),
_step;
try {
var _loop = function _loop() {
var _inetnum$remarks;
var key = _step.value;
if (inetnum[key] && _this.matchUrl(inetnum[key]).length) {
return {
v: true
};
}
if ((inetnum === null || inetnum === void 0 || (_inetnum$remarks = inetnum.remarks) === null || _inetnum$remarks === void 0 ? void 0 : _inetnum$remarks.length) > 0) {
return {
v: inetnum.remarks.some(function (i) {
return _this.testKeys(i, key);
})
};
}
},
_ret;
for (_iterator.s(); !(_step = _iterator.n()).done;) {
_ret = _loop();
if (_ret) return _ret.v;
}
} catch (err) {
_iterator.e(err);
} finally {
_iterator.f();
}
return false;
});
_defineProperty(this, "getBlocks", function () {
var selector = _this.params.af.map(function (i) {
return i === 4 ? "inetnum" : "inet6num";
});
return _this.whois.getObjects(selector, _this.filterFunction, ["inetnum", "inet6num", "remarks", "last-updated"].concat(_toConsumableArray(_this.specialKeys))).then(function (blocks) {
return blocks.flat().filter(function (i) {
return !!i.inetnum || !!i.inet6num;
});
})["catch"](_this.logger.log);
});
_defineProperty(this, "_getFileName", function (file) {
return _this.cacheDir + (0, _md["default"])(file);
});
_defineProperty(this, "_setCacheHeaders", function (response, cachedFile) {
var _this$cacheHeadersInd;
var setAge = 3600 * 24 * (_this.params.fileCacheDays || 7); // default 1 week (see draft)
if (response.headers["cache-control"]) {
var maxAge = response.headers["cache-control"].split(",").filter(function (h) {
return h.includes("max-age");
}).map(function (h) {
return h.trim();
}).pop();
if (maxAge) {
var age = maxAge.split("=").pop();
if (age && !isNaN(age)) {
setAge = Math.min(Math.max(parseInt(age), 3600), 3600 * 24 * 7); // Min 1 hour, max 1 week of cache (to avoid random max-age settings)
}
}
}
_this.cacheHeadersIndex[cachedFile] = (_this$cacheHeadersInd = _this.cacheHeadersIndex[cachedFile]) !== null && _this$cacheHeadersInd !== void 0 ? _this$cacheHeadersInd : (0, _moment["default"])(_this.startTime).add(setAge, "seconds");
});
_defineProperty(this, "_isCachedValid", function (cachedFile) {
if (_this.params.test) {
return false;
} else {
return _fs["default"].existsSync(cachedFile) && _this.cacheHeadersIndex[cachedFile] && (0, _moment["default"])(_this.cacheHeadersIndex[cachedFile]).isSameOrAfter(_this.startTime);
}
});
_defineProperty(this, "_importCacheHeaderIndex", function () {
var tmp;
if (_fs["default"].existsSync(_this.cacheHeadersIndexFileName)) {
tmp = JSON.parse(_fs["default"].readFileSync(_this.cacheHeadersIndexFileName, "utf-8"));
for (var key in tmp) {
tmp[key] = (0, _moment["default"])(tmp[key]);
}
}
_this.cacheHeadersIndex = tmp || {};
});
_defineProperty(this, "_persistCacheIndex", function () {
_fs["default"].writeFileSync(_this.cacheHeadersIndexFileName, JSON.stringify(_this.cacheHeadersIndex));
});
_defineProperty(this, "logEntry", function (file, cache) {
console.log("".concat(file, " ").concat(cache ? "[cache]" : "[download]"));
});
_defineProperty(this, "_getFile", function (file) {
var abortTimeout = parseInt(_this.params.downloadTimeout) * 1000;
return new Promise(function (resolve, reject) {
var timeout = setTimeout(function () {
_this.logger.log("Error: ".concat(file, " timeout"));
resolve(null);
}, abortTimeout);
var resolveAndClear = function resolveAndClear(data) {
resolve(data);
clearTimeout(timeout);
};
var cachedFile = _this._getFileName(file);
if (_this._isCachedValid(cachedFile)) {
try {
_this.logEntry(file, true);
resolveAndClear(_fs["default"].readFileSync(cachedFile, "utf8"));
} catch (error) {
_this.logger.log("Error: ".concat(file, " ").concat(error));
resolveAndClear(null);
}
} else {
_this.logEntry(file, false);
(0, _redaxios["default"])({
url: file,
method: "GET",
timeout: abortTimeout
}).then(function (response) {
var data = response.data;
if (/<a|<div|<span|<style|<link/gi.test(data)) {
var message = "Error: ".concat(file, " is not a valid file format");
_this.logger.log(message);
console.log(message);
resolveAndClear(null);
} else {
_fs["default"].writeFileSync(cachedFile, data);
_this._setCacheHeaders(response, cachedFile);
resolveAndClear(data);
}
})["catch"](function (error) {
_this.logger.log("Error: ".concat(file, " ").concat(error.message));
resolveAndClear(null);
});
}
});
});
_defineProperty(this, "getFiles", function (blocks) {
var out = [];
var uniqueBlocks = _toConsumableArray(new Set(blocks.map(function (i) {
return i.value;
})));
var half = Math.floor(uniqueBlocks.length / 2);
// pre load all files
return Promise.all([(0, _batchPromises["default"])(40, uniqueBlocks.slice(0, half), _this._getFile), (0, _batchPromises["default"])(40, uniqueBlocks.slice(half), _this._getFile)]).then(function () {
var _iterator2 = _createForOfIteratorHelper(blocks),
_step2;
try {
for (_iterator2.s(); !(_step2 = _iterator2.n()).done;) {
var block = _step2.value;
try {
var cachedFile = _this._getFileName(block.value);
var data = _fs["default"].readFileSync(cachedFile, "utf8");
if (data !== null && data !== void 0 && data.length) {
out.push(_this.params.parseLine(block.inetnum, data).map(function (i) {
return _objectSpread(_objectSpread({}, i), {}, {
af: _ipSub["default"].getAddressFamily(i.inetnum)
});
}));
}
} catch (error) {
// Nothing - these are files that are not CSV
}
}
} catch (err) {
_iterator2.e(err);
} finally {
_iterator2.f();
}
return out;
});
});
_defineProperty(this, "getMostUpdatedInetnums", function (inetnums) {
var index = {};
var _iterator3 = _createForOfIteratorHelper(inetnums),
_step3;
try {
for (_iterator3.s(); !(_step3 = _iterator3.n()).done;) {
var inetnum = _step3.value;
index[inetnum.inetnum] = !index[inetnum.inetnum] || index[inetnum.inetnum].lastUpdate < inetnum.lastUpdate ? inetnum : index[inetnum.inetnum];
}
} catch (err) {
_iterator3.e(err);
} finally {
_iterator3.f();
}
return Object.values(index);
});
_defineProperty(this, "setEntryPriority", function () {
var entries = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : [];
console.log("Validating prefix ownership");
return [].concat(_toConsumableArray(_this.params.af.includes(4) ? _this._setPriority(entries.filter(function (i) {
return i.af === 4;
})) : []), _toConsumableArray(_this.params.af.includes(6) ? _this._setPriority(entries.filter(function (i) {
return i.af === 6;
})) : [])).flat();
});
_defineProperty(this, "_setPriority", function () {
var entries = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : [];
var longestPrefixMatch = new _longestPrefixMatch["default"]();
var tmp = {};
for (var _i = 0, _arr = _toConsumableArray(new Set(entries.map(function (i) {
return i.inetnum;
}))); _i < _arr.length; _i++) {
var inetnum = _arr[_i];
longestPrefixMatch.addPrefix(inetnum, inetnum);
}
var _iterator4 = _createForOfIteratorHelper(entries),
_step4;
try {
for (_iterator4.s(); !(_step4 = _iterator4.n()).done;) {
var entry = _step4.value;
var _inetnum = longestPrefixMatch.getMatch(entry.prefix, false);
if (_inetnum && _inetnum.length === 1 && entry.inetnum === _inetnum[0]) {
tmp[entry.prefix] = entry;
}
}
} catch (err) {
_iterator4.e(err);
} finally {
_iterator4.f();
}
return Object.values(tmp);
});
_defineProperty(this, "testKeys", function (remark, key) {
var re = new RegExp("^".concat(key, " https?://S+"), "gi");
return re.test(remark);
});
_defineProperty(this, "matchUrl", function (remark) {
return remark.match(/\bhttps?:\/\/\S+/gi) || [];
});
_defineProperty(this, "translateObject", function (object, key) {
var _object$remarks, _object$key;
var inetnum = object.inetnum || object.inet6num;
var remarks = (_object$remarks = object.remarks) !== null && _object$remarks !== void 0 ? _object$remarks : [];
var keyField = object !== null && object !== void 0 && (_object$key = object[key]) !== null && _object$key !== void 0 && _object$key.length ? _this.matchUrl(object[key]).pop() : null;
var inetnums = [inetnum];
if (!inetnum.includes("/")) {
var ips = inetnum.split("-").map(function (ip) {
return ip.trim();
});
inetnums = _ipSub["default"].ipRangeToCidr(ips[0], ips[1]);
}
var lastUpdate = (0, _moment["default"])(object["last-updated"]);
var remark = remarks.find(function (i) {
return i.startsWith(key);
});
var fieldContent = null;
if (keyField) {
fieldContent = keyField;
} else if (remark) {
fieldContent = _this.matchUrl(remark).pop();
}
return inetnums.map(function (inetnum) {
return {
inetnum: inetnum,
value: fieldContent,
lastUpdate: lastUpdate
};
});
});
_defineProperty(this, "getPairs", function () {
return _this.getBlocks().then(function () {
var objects = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : [];
return _this.specialKeys.map(function (key) {
return objects.map(function (i) {
return _this.translateObject(i, key);
});
}).flat().flat().filter(function (i) {
return i.value;
});
}).then(_this.getMostUpdatedInetnums);
});
_defineProperty(this, "run", function () {
return _this.getPairs().then(_this.getFiles).then(function (data) {
_this._persistCacheIndex();
return data.flat();
});
});
var defaults = {
cacheDir: ".cache/",
whoisCacheDays: 3,
fileCacheDays: 7,
af: [4, 6],
silent: false,
specialKeys: [],
userAgent: "whois-link-finder",
include: ["ripe", "afrinic", "apnic", "arin", "lacnic"],
output: "result.csv",
test: null,
downloadTimeout: 10,
daysWhoisSuballocationsCache: 7,
// Cannot be less than this
skipSuballocations: false,
compileSuballocationLocally: false,
parseLine: function parseLine(inetnum, data) {
return {
inetnum: inetnum,
data: data
};
}
};
this.params = _objectSpread(_objectSpread({}, defaults), params !== null && params !== void 0 ? params : {});
this.logger = this.params.logger;
this.cacheDir = this.params.cacheDir.split("/").filter(function (i) {
return !!i;
}).join("/") + "/";
this.startTime = (0, _moment["default"])();
this.cacheHeadersIndexFileName = this.cacheDir + "cache-index.json";
this._importCacheHeaderIndex();
this.connectors = defaults.include.filter(function (key) {
return _this.params.include.includes(key);
});
this.whois = new _bulkWhoisParser["default"]({
cacheDir: this.cacheDir,
repos: this.connectors,
daysWhoisSuballocationsCache: this.params.daysWhoisSuballocationsCache,
skipSuballocations: this.params.skipSuballocations,
defaultCacheDays: this.params.whoisCacheDays,
compileSuballocationLocally: this.params.compileSuballocationLocally,
userAgent: this.params.userAgent,
deleteCorruptedCacheFile: true
});
this.specialKeys = this.params.specialKeys;
});