@tricoteuses/assemblee
Version:
Retrieve, clean up & handle French Assemblée nationale's open data
547 lines (545 loc) • 91 kB
JavaScript
"use strict";
function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
require("core-js/modules/es.symbol.js");
require("core-js/modules/es.symbol.description.js");
require("core-js/modules/es.symbol.async-iterator.js");
require("core-js/modules/es.symbol.iterator.js");
require("core-js/modules/es.symbol.to-string-tag.js");
require("core-js/modules/es.array.for-each.js");
require("core-js/modules/es.array.from.js");
require("core-js/modules/es.array.is-array.js");
require("core-js/modules/es.array.reverse.js");
require("core-js/modules/es.array.slice.js");
require("core-js/modules/es.function.name.js");
require("core-js/modules/es.json.to-string-tag.js");
require("core-js/modules/es.math.to-string-tag.js");
require("core-js/modules/es.object.create.js");
require("core-js/modules/es.object.define-property.js");
require("core-js/modules/es.object.get-own-property-descriptor.js");
require("core-js/modules/es.object.get-prototype-of.js");
require("core-js/modules/es.object.set-prototype-of.js");
require("core-js/modules/es.regexp.exec.js");
require("core-js/modules/es.weak-map.js");
require("core-js/modules/web.dom-collections.for-each.js");
Object.defineProperty(exports, "__esModule", {
value: true
});
exports.downloadAndParse = downloadAndParse;
require("core-js/modules/es.array.concat.js");
require("core-js/modules/es.array.find.js");
require("core-js/modules/es.array.includes.js");
require("core-js/modules/es.array.iterator.js");
require("core-js/modules/es.array.join.js");
require("core-js/modules/es.date.to-json.js");
require("core-js/modules/es.date.to-string.js");
require("core-js/modules/es.object.keys.js");
require("core-js/modules/es.object.to-string.js");
require("core-js/modules/es.promise.js");
require("core-js/modules/es.regexp.to-string.js");
require("core-js/modules/es.string.ends-with.js");
require("core-js/modules/es.string.includes.js");
require("core-js/modules/es.string.iterator.js");
require("core-js/modules/es.string.starts-with.js");
require("core-js/modules/web.dom-collections.iterator.js");
require("core-js/modules/web.timers.js");
var _assert = _interopRequireDefault(require("assert"));
var _commandLineArgs = _interopRequireDefault(require("command-line-args"));
var _crypto = require("crypto");
var _dateFns = require("date-fns");
var _fsExtra = _interopRequireDefault(require("fs-extra"));
var _path = _interopRequireDefault(require("path"));
var _dossiers_legislatifs = require("../dossiers_legislatifs");
var git = _interopRequireWildcard(require("../git"));
var _loaders = require("../loaders");
var _parsers = require("../parsers");
var _urls = require("../urls");
var _cli_helpers = require("./shared/cli_helpers");
function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(e) { return e ? t : r; })(e); }
function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != _typeof(e) && "function" != typeof e) return { "default": e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n["default"] = e, t && t.set(e, n), n; }
function _interopRequireDefault(e) { return e && e.__esModule ? e : { "default": e }; }
function _createForOfIteratorHelper(r, e) { var t = "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"]; if (!t) { if (Array.isArray(r) || (t = _unsupportedIterableToArray(r)) || e && r && "number" == typeof r.length) { t && (r = t); var _n = 0, F = function F() {}; return { s: F, n: function n() { return _n >= r.length ? { done: !0 } : { done: !1, value: r[_n++] }; }, e: function e(r) { throw r; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var o, a = !0, u = !1; return { s: function s() { t = t.call(r); }, n: function n() { var r = t.next(); return a = r.done, r; }, e: function e(r) { u = !0, o = r; }, f: function f() { try { a || null == t["return"] || t["return"](); } finally { if (u) throw o; } } }; }
function _unsupportedIterableToArray(r, a) { if (r) { if ("string" == typeof r) return _arrayLikeToArray(r, a); var t = {}.toString.call(r).slice(8, -1); return "Object" === t && r.constructor && (t = r.constructor.name), "Map" === t || "Set" === t ? Array.from(r) : "Arguments" === t || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(t) ? _arrayLikeToArray(r, a) : void 0; } }
function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length); for (var e = 0, n = Array(a); e < a; e++) n[e] = r[e]; return n; }
function _regeneratorRuntime() { "use strict"; /*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */ _regeneratorRuntime = function _regeneratorRuntime() { return e; }; var t, e = {}, r = Object.prototype, n = r.hasOwnProperty, o = Object.defineProperty || function (t, e, r) { t[e] = r.value; }, i = "function" == typeof Symbol ? Symbol : {}, a = i.iterator || "@@iterator", c = i.asyncIterator || "@@asyncIterator", u = i.toStringTag || "@@toStringTag"; function define(t, e, r) { return Object.defineProperty(t, e, { value: r, enumerable: !0, configurable: !0, writable: !0 }), t[e]; } try { define({}, ""); } catch (t) { define = function define(t, e, r) { return t[e] = r; }; } function wrap(t, e, r, n) { var i = e && e.prototype instanceof Generator ? e : Generator, a = Object.create(i.prototype), c = new Context(n || []); return o(a, "_invoke", { value: makeInvokeMethod(t, r, c) }), a; } function tryCatch(t, e, r) { try { return { type: "normal", arg: t.call(e, r) }; } catch (t) { return { type: "throw", arg: t }; } } e.wrap = wrap; var h = "suspendedStart", l = "suspendedYield", f = "executing", s = "completed", y = {}; function Generator() {} function GeneratorFunction() {} function GeneratorFunctionPrototype() {} var p = {}; define(p, a, function () { return this; }); var d = Object.getPrototypeOf, v = d && d(d(values([]))); v && v !== r && n.call(v, a) && (p = v); var g = GeneratorFunctionPrototype.prototype = Generator.prototype = Object.create(p); function defineIteratorMethods(t) { ["next", "throw", "return"].forEach(function (e) { define(t, e, function (t) { return this._invoke(e, t); }); }); } function AsyncIterator(t, e) { function invoke(r, o, i, a) { var c = tryCatch(t[r], t, o); if ("throw" !== c.type) { var u = c.arg, h = u.value; return h && "object" == _typeof(h) && n.call(h, "__await") ? e.resolve(h.__await).then(function (t) { invoke("next", t, i, a); }, function (t) { invoke("throw", t, i, a); }) : e.resolve(h).then(function (t) { u.value = t, i(u); }, function (t) { return invoke("throw", t, i, a); }); } a(c.arg); } var r; o(this, "_invoke", { value: function value(t, n) { function callInvokeWithMethodAndArg() { return new e(function (e, r) { invoke(t, n, e, r); }); } return r = r ? r.then(callInvokeWithMethodAndArg, callInvokeWithMethodAndArg) : callInvokeWithMethodAndArg(); } }); } function makeInvokeMethod(e, r, n) { var o = h; return function (i, a) { if (o === f) throw Error("Generator is already running"); if (o === s) { if ("throw" === i) throw a; return { value: t, done: !0 }; } for (n.method = i, n.arg = a;;) { var c = n.delegate; if (c) { var u = maybeInvokeDelegate(c, n); if (u) { if (u === y) continue; return u; } } if ("next" === n.method) n.sent = n._sent = n.arg;else if ("throw" === n.method) { if (o === h) throw o = s, n.arg; n.dispatchException(n.arg); } else "return" === n.method && n.abrupt("return", n.arg); o = f; var p = tryCatch(e, r, n); if ("normal" === p.type) { if (o = n.done ? s : l, p.arg === y) continue; return { value: p.arg, done: n.done }; } "throw" === p.type && (o = s, n.method = "throw", n.arg = p.arg); } }; } function maybeInvokeDelegate(e, r) { var n = r.method, o = e.iterator[n]; if (o === t) return r.delegate = null, "throw" === n && e.iterator["return"] && (r.method = "return", r.arg = t, maybeInvokeDelegate(e, r), "throw" === r.method) || "return" !== n && (r.method = "throw", r.arg = new TypeError("The iterator does not provide a '" + n + "' method")), y; var i = tryCatch(o, e.iterator, r.arg); if ("throw" === i.type) return r.method = "throw", r.arg = i.arg, r.delegate = null, y; var a = i.arg; return a ? a.done ? (r[e.resultName] = a.value, r.next = e.nextLoc, "return" !== r.method && (r.method = "next", r.arg = t), r.delegate = null, y) : a : (r.method = "throw", r.arg = new TypeError("iterator result is not an object"), r.delegate = null, y); } function pushTryEntry(t) { var e = { tryLoc: t[0] }; 1 in t && (e.catchLoc = t[1]), 2 in t && (e.finallyLoc = t[2], e.afterLoc = t[3]), this.tryEntries.push(e); } function resetTryEntry(t) { var e = t.completion || {}; e.type = "normal", delete e.arg, t.completion = e; } function Context(t) { this.tryEntries = [{ tryLoc: "root" }], t.forEach(pushTryEntry, this), this.reset(!0); } function values(e) { if (e || "" === e) { var r = e[a]; if (r) return r.call(e); if ("function" == typeof e.next) return e; if (!isNaN(e.length)) { var o = -1, i = function next() { for (; ++o < e.length;) if (n.call(e, o)) return next.value = e[o], next.done = !1, next; return next.value = t, next.done = !0, next; }; return i.next = i; } } throw new TypeError(_typeof(e) + " is not iterable"); } return GeneratorFunction.prototype = GeneratorFunctionPrototype, o(g, "constructor", { value: GeneratorFunctionPrototype, configurable: !0 }), o(GeneratorFunctionPrototype, "constructor", { value: GeneratorFunction, configurable: !0 }), GeneratorFunction.displayName = define(GeneratorFunctionPrototype, u, "GeneratorFunction"), e.isGeneratorFunction = function (t) { var e = "function" == typeof t && t.constructor; return !!e && (e === GeneratorFunction || "GeneratorFunction" === (e.displayName || e.name)); }, e.mark = function (t) { return Object.setPrototypeOf ? Object.setPrototypeOf(t, GeneratorFunctionPrototype) : (t.__proto__ = GeneratorFunctionPrototype, define(t, u, "GeneratorFunction")), t.prototype = Object.create(g), t; }, e.awrap = function (t) { return { __await: t }; }, defineIteratorMethods(AsyncIterator.prototype), define(AsyncIterator.prototype, c, function () { return this; }), e.AsyncIterator = AsyncIterator, e.async = function (t, r, n, o, i) { void 0 === i && (i = Promise); var a = new AsyncIterator(wrap(t, r, n, o), i); return e.isGeneratorFunction(r) ? a : a.next().then(function (t) { return t.done ? t.value : a.next(); }); }, defineIteratorMethods(g), define(g, u, "Generator"), define(g, a, function () { return this; }), define(g, "toString", function () { return "[object Generator]"; }), e.keys = function (t) { var e = Object(t), r = []; for (var n in e) r.push(n); return r.reverse(), function next() { for (; r.length;) { var t = r.pop(); if (t in e) return next.value = t, next.done = !1, next; } return next.done = !0, next; }; }, e.values = values, Context.prototype = { constructor: Context, reset: function reset(e) { if (this.prev = 0, this.next = 0, this.sent = this._sent = t, this.done = !1, this.delegate = null, this.method = "next", this.arg = t, this.tryEntries.forEach(resetTryEntry), !e) for (var r in this) "t" === r.charAt(0) && n.call(this, r) && !isNaN(+r.slice(1)) && (this[r] = t); }, stop: function stop() { this.done = !0; var t = this.tryEntries[0].completion; if ("throw" === t.type) throw t.arg; return this.rval; }, dispatchException: function dispatchException(e) { if (this.done) throw e; var r = this; function handle(n, o) { return a.type = "throw", a.arg = e, r.next = n, o && (r.method = "next", r.arg = t), !!o; } for (var o = this.tryEntries.length - 1; o >= 0; --o) { var i = this.tryEntries[o], a = i.completion; if ("root" === i.tryLoc) return handle("end"); if (i.tryLoc <= this.prev) { var c = n.call(i, "catchLoc"), u = n.call(i, "finallyLoc"); if (c && u) { if (this.prev < i.catchLoc) return handle(i.catchLoc, !0); if (this.prev < i.finallyLoc) return handle(i.finallyLoc); } else if (c) { if (this.prev < i.catchLoc) return handle(i.catchLoc, !0); } else { if (!u) throw Error("try statement without catch or finally"); if (this.prev < i.finallyLoc) return handle(i.finallyLoc); } } } }, abrupt: function abrupt(t, e) { for (var r = this.tryEntries.length - 1; r >= 0; --r) { var o = this.tryEntries[r]; if (o.tryLoc <= this.prev && n.call(o, "finallyLoc") && this.prev < o.finallyLoc) { var i = o; break; } } i && ("break" === t || "continue" === t) && i.tryLoc <= e && e <= i.finallyLoc && (i = null); var a = i ? i.completion : {}; return a.type = t, a.arg = e, i ? (this.method = "next", this.next = i.finallyLoc, y) : this.complete(a); }, complete: function complete(t, e) { if ("throw" === t.type) throw t.arg; return "break" === t.type || "continue" === t.type ? this.next = t.arg : "return" === t.type ? (this.rval = this.arg = t.arg, this.method = "return", this.next = "end") : "normal" === t.type && e && (this.next = e), y; }, finish: function finish(t) { for (var e = this.tryEntries.length - 1; e >= 0; --e) { var r = this.tryEntries[e]; if (r.finallyLoc === t) return this.complete(r.completion, r.afterLoc), resetTryEntry(r), y; } }, "catch": function _catch(t) { for (var e = this.tryEntries.length - 1; e >= 0; --e) { var r = this.tryEntries[e]; if (r.tryLoc === t) { var n = r.completion; if ("throw" === n.type) { var o = n.arg; resetTryEntry(r); } return o; } } throw Error("illegal catch attempt"); }, delegateYield: function delegateYield(e, r, n) { return this.delegate = { iterator: values(e), resultName: r, nextLoc: n }, "next" === this.method && (this.arg = t), y; } }, e; }
function asyncGeneratorStep(n, t, e, r, o, a, c) { try { var i = n[a](c), u = i.value; } catch (n) { return void e(n); } i.done ? t(u) : Promise.resolve(u).then(r, o); }
function _asyncToGenerator(n) { return function () { var t = this, e = arguments; return new Promise(function (r, o) { var a = n.apply(t, e); function _next(n) { asyncGeneratorStep(a, r, o, _next, _throw, "next", n); } function _throw(n) { asyncGeneratorStep(a, r, o, _next, _throw, "throw", n); } _next(void 0); }); }; }
function parseArgs(argv) {
var optionsDefinitions = [_cli_helpers.commitOption, _cli_helpers.legislatureOption, _cli_helpers.remoteOption, _cli_helpers.silentOption, _cli_helpers.verboseOption, _cli_helpers.dataDirDefaultOption, _cli_helpers.pullOption, {
alias: "f",
help: "retrieve all documents, even already retrieved ones",
name: "full",
type: Boolean
}, {
alias: "n",
help: "try to also retrieve documents that were previously not found",
name: "not-found",
type: Boolean
}, _cli_helpers.documentFormatOption, {
alias: "T",
help: "type of documents to retrieve (for example: PION)",
multiple: true,
name: "document-type",
type: String
}];
var options = (0, _commandLineArgs["default"])(optionsDefinitions, {
argv: argv
});
return options;
}
var today = new Date();
function fetchWithRetry(_x) {
return _fetchWithRetry.apply(this, arguments);
}
function _fetchWithRetry() {
_fetchWithRetry = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(url) {
var retries,
backoff,
attempt,
_args = arguments;
return _regeneratorRuntime().wrap(function _callee$(_context) {
while (1) switch (_context.prev = _context.next) {
case 0:
retries = _args.length > 1 && _args[1] !== undefined ? _args[1] : 3;
backoff = _args.length > 2 && _args[2] !== undefined ? _args[2] : 300;
attempt = 0;
case 3:
if (!(attempt < retries)) {
_context.next = 21;
break;
}
_context.prev = 4;
_context.next = 7;
return fetch(url);
case 7:
return _context.abrupt("return", _context.sent);
case 10:
_context.prev = 10;
_context.t0 = _context["catch"](4);
if (!(attempt === retries - 1)) {
_context.next = 14;
break;
}
throw _context.t0;
case 14:
console.warn("Fetch attempt ".concat(attempt + 1, " for ").concat(url, " failed. Retrying in ").concat(backoff, "ms..."));
_context.next = 17;
return new Promise(function (resolve) {
return setTimeout(resolve, backoff);
});
case 17:
backoff *= 2;
case 18:
attempt++;
_context.next = 3;
break;
case 21:
throw new Error("Failed to fetch ".concat(url, " after ").concat(retries, " attempts"));
case 22:
case "end":
return _context.stop();
}
}, _callee, null, [[4, 10]]);
}));
return _fetchWithRetry.apply(this, arguments);
}
function downloadAndParse(_x2, _x3, _x4) {
return _downloadAndParse.apply(this, arguments);
}
function _downloadAndParse() {
_downloadAndParse = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee2(document, documentsDir, options) {
var _options$legislature;
var dataDir, documentsFilesDir, documentDir, documentFileDir, documentPath, htmlPath, html, parsedDocument, json, documentJson;
return _regeneratorRuntime().wrap(function _callee2$(_context2) {
while (1) switch (_context2.prev = _context2.next) {
case 0:
if (!(document.uid.substring(4, 6) === "SN")) {
_context2.next = 2;
break;
}
return _context2.abrupt("return");
case 2:
if (!["RAPP", "RINF"].includes(document.uid.substring(0, 4))) {
_context2.next = 4;
break;
}
return _context2.abrupt("return");
case 4:
if (!(((_options$legislature = options.legislature) === null || _options$legislature === void 0 ? void 0 : _options$legislature.toString()) !== document.legislature)) {
_context2.next = 6;
break;
}
return _context2.abrupt("return");
case 6:
if (!options.fetchDocuments) {
_context2.next = 28;
break;
}
dataDir = options.dataDir;
documentsFilesDir = _path["default"].join(dataDir, "Documents");
documentDir = (0, _loaders.pathFromDocumentUid)("".concat(documentsDir, "/documents"), document.uid);
documentFileDir = (0, _loaders.pathFromDocumentUid)(documentsFilesDir, document.uid);
_fsExtra["default"].ensureDirSync(documentsFilesDir);
_context2.next = 14;
return processDocumentOrDivision(document, documentsFilesDir, options);
case 14:
if (!options.parseDocuments) {
_context2.next = 28;
break;
}
documentPath = "".concat(documentDir, ".json");
htmlPath = _path["default"].join(documentFileDir, "dyn-opendata.html");
if (!_fsExtra["default"].existsSync(htmlPath)) {
_context2.next = 28;
break;
}
html = _fsExtra["default"].readFileSync(htmlPath, {
encoding: "utf-8"
});
_context2.prev = 19;
parsedDocument = (0, _parsers.parseTexte)("https://www.assemblee-nationale.fr", html);
if (parsedDocument && _fsExtra["default"].existsSync(documentPath)) {
json = _fsExtra["default"].readFileSync(documentPath, {
encoding: "utf-8"
});
documentJson = JSON.parse(json);
documentJson.subdivisions = parsedDocument === null || parsedDocument === void 0 ? void 0 : parsedDocument.subdivisions;
_fsExtra["default"].writeFileSync(documentPath, JSON.stringify(documentJson, null, 2));
}
_context2.next = 28;
break;
case 24:
_context2.prev = 24;
_context2.t0 = _context2["catch"](19);
console.log("Unable to parse document ".concat(document.uid));
return _context2.abrupt("return");
case 28:
case "end":
return _context2.stop();
}
}, _callee2, null, [[19, 24]]);
}));
return _downloadAndParse.apply(this, arguments);
}
function retrieveDocuments(_x5) {
return _retrieveDocuments.apply(this, arguments);
}
function _retrieveDocuments() {
_retrieveDocuments = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee3(options) {
var dataDir, documentsDir, _iterator, _step, filename, firstUid, skip, _iterator2, _step2, document, _iterator3, _step3, documentOrDivision;
return _regeneratorRuntime().wrap(function _callee3$(_context3) {
while (1) switch (_context3.prev = _context3.next) {
case 0:
(0, _assert["default"])(!options.commit || !options.uid, 'Options "commit" & "uid" are incompatible');
dataDir = options.dataDir;
documentsDir = _path["default"].join(dataDir, "Documents");
if (options.pull) {
git.resetAndPull(documentsDir);
}
_fsExtra["default"].ensureDirSync(documentsDir);
if (!(options.full && !options.uid)) {
_context3.next = 24;
break;
}
_iterator = _createForOfIteratorHelper(_fsExtra["default"].readdirSync(documentsDir));
_context3.prev = 7;
_iterator.s();
case 9:
if ((_step = _iterator.n()).done) {
_context3.next = 16;
break;
}
filename = _step.value;
if (!(filename[0] === ".")) {
_context3.next = 13;
break;
}
return _context3.abrupt("continue", 14);
case 13:
_fsExtra["default"].removeSync(_path["default"].join(documentsDir, filename));
case 14:
_context3.next = 9;
break;
case 16:
_context3.next = 21;
break;
case 18:
_context3.prev = 18;
_context3.t0 = _context3["catch"](7);
_iterator.e(_context3.t0);
case 21:
_context3.prev = 21;
_iterator.f();
return _context3.finish(21);
case 24:
firstUid = options.uid;
skip = Boolean(firstUid);
_iterator2 = _createForOfIteratorHelper((0, _loaders.iterLoadAssembleeDocuments)(dataDir, options.legislature));
_context3.prev = 27;
_iterator2.s();
case 29:
if ((_step2 = _iterator2.n()).done) {
_context3.next = 58;
break;
}
document = _step2.value.document;
_iterator3 = _createForOfIteratorHelper((0, _dossiers_legislatifs.walkDocumentAndDivisions)(document));
_context3.prev = 32;
_iterator3.s();
case 34:
if ((_step3 = _iterator3.n()).done) {
_context3.next = 48;
break;
}
documentOrDivision = _step3.value;
if (!(documentOrDivision.uid.substring(4, 6) === "SN")) {
_context3.next = 38;
break;
}
return _context3.abrupt("continue", 46);
case 38:
if (!skip) {
_context3.next = 44;
break;
}
if (!(documentOrDivision.uid === firstUid)) {
_context3.next = 43;
break;
}
skip = false;
_context3.next = 44;
break;
case 43:
return _context3.abrupt("continue", 46);
case 44:
_context3.next = 46;
return processDocumentOrDivision(documentOrDivision, documentsDir, options);
case 46:
_context3.next = 34;
break;
case 48:
_context3.next = 53;
break;
case 50:
_context3.prev = 50;
_context3.t1 = _context3["catch"](32);
_iterator3.e(_context3.t1);
case 53:
_context3.prev = 53;
_iterator3.f();
return _context3.finish(53);
case 56:
_context3.next = 29;
break;
case 58:
_context3.next = 63;
break;
case 60:
_context3.prev = 60;
_context3.t2 = _context3["catch"](27);
_iterator2.e(_context3.t2);
case 63:
_context3.prev = 63;
_iterator2.f();
return _context3.finish(63);
case 66:
if (!options.commit) {
_context3.next = 68;
break;
}
return _context3.abrupt("return", git.commitAndPush(documentsDir, "Nouvelle moisson", options.remote));
case 68:
return _context3.abrupt("return", 0);
case 69:
case "end":
return _context3.stop();
}
}, _callee3, null, [[7, 18, 21, 24], [27, 60, 63, 66], [32, 50, 53, 56]]);
}));
return _retrieveDocuments.apply(this, arguments);
}
function processDocumentOrDivision(_x6, _x7, _x8) {
return _processDocumentOrDivision.apply(this, arguments);
}
function _processDocumentOrDivision() {
_processDocumentOrDivision = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee4(documentOrDivision, documentsDir, options) {
var documentDir, filenameBySha256, indexPath, index, _iterator4, _step4, _loop, _ret;
return _regeneratorRuntime().wrap(function _callee4$(_context5) {
while (1) switch (_context5.prev = _context5.next) {
case 0:
documentDir = (0, _loaders.pathFromDocumentUid)(documentsDir, documentOrDivision.uid);
_fsExtra["default"].ensureDirSync(documentDir);
filenameBySha256 = {};
indexPath = _path["default"].join(documentDir, "index.json");
index = _fsExtra["default"].pathExistsSync(indexPath) ? _fsExtra["default"].readJsonSync(indexPath) : {};
_iterator4 = _createForOfIteratorHelper((0, _urls.iterDocumentOrDivisionUrls)(documentOrDivision));
_context5.prev = 6;
_loop = /*#__PURE__*/_regeneratorRuntime().mark(function _loop() {
var _documentOrDivision$c, _index$format, _formatFilesInfos$fin, _documentOrDivision$c2;
var _step4$value, format, type, url, filename, fileExtension, documentType, formatFilesInfos, fileInfos, response, filePath, arrayBuffer, buffer, sha256, existingFilename;
return _regeneratorRuntime().wrap(function _loop$(_context4) {
while (1) switch (_context4.prev = _context4.next) {
case 0:
_step4$value = _step4.value, format = _step4$value.format, type = _step4$value.type, url = _step4$value.url;
filename = "".concat(type, ".").concat(format === _urls.DocumentUrlFormat.Pdf ? "pdf" : "html"); // Filter by file extension if option is passed
fileExtension = url.split(".").pop();
if (!(options.format && fileExtension && options.format !== fileExtension)) {
_context4.next = 5;
break;
}
return _context4.abrupt("return", 0);
case 5:
if (!(options.legislature && options.legislature.toString() !== documentOrDivision.legislature)) {
_context4.next = 7;
break;
}
return _context4.abrupt("return", 0);
case 7:
// Filter by document type if option is passed
documentType = (_documentOrDivision$c = documentOrDivision.classification) === null || _documentOrDivision$c === void 0 || (_documentOrDivision$c = _documentOrDivision$c.type) === null || _documentOrDivision$c === void 0 ? void 0 : _documentOrDivision$c.code;
if (!(options["document-type"] !== undefined && !options["document-type"].includes(documentType))) {
_context4.next = 10;
break;
}
return _context4.abrupt("return", 0);
case 10:
formatFilesInfos = (_index$format = index[format]) !== null && _index$format !== void 0 ? _index$format : index[format] = [];
fileInfos = (_formatFilesInfos$fin = formatFilesInfos.find(function (file) {
return file.url === url;
})) !== null && _formatFilesInfos$fin !== void 0 ? _formatFilesInfos$fin : {};
if (!formatFilesInfos.includes(fileInfos)) {
formatFilesInfos.push(fileInfos);
}
fileInfos.url = url;
if (!(fileInfos.status === 200 && !options.full)) {
_context4.next = 17;
break;
}
filenameBySha256[fileInfos.sha256] = filename;
return _context4.abrupt("return", 0);
case 17:
if (!(fileInfos.status === 404 && !options["not-found"] && (0, _dateFns.differenceInDays)(today, (_documentOrDivision$c2 = documentOrDivision.cycleDeVie.chrono.dateCreation) !== null && _documentOrDivision$c2 !== void 0 ? _documentOrDivision$c2 : documentOrDivision.cycleDeVie.chrono.dateDepot) > 10)) {
_context4.next = 19;
break;
}
return _context4.abrupt("return", 0);
case 19:
if (!options.silent) {
console.log("Retrieving document or division ".concat(documentOrDivision.uid, " at ").concat(url, "\u2026"));
}
_context4.next = 22;
return fetchWithRetry(url);
case 22:
response = _context4.sent;
filePath = _path["default"].join(documentDir, filename);
if (!response.ok) {
_context4.next = 32;
break;
}
_context4.next = 27;
return response.arrayBuffer();
case 27:
arrayBuffer = _context4.sent;
buffer = Buffer.from(arrayBuffer);
if (format === _urls.DocumentUrlFormat.Pdf && !buffer.subarray(0, 4).toString().startsWith("%PDF")) {
// Instead of a PDF, the received data may be an HTML page with a message like
// "Document non encore publié".
if (!options.silent) {
console.warn(" PDF \"".concat(url, "\" not found."));
}
_fsExtra["default"].removeSync(filePath);
delete fileInfos.filename;
delete fileInfos.sha256;
fileInfos.status = 404;
} else {
sha256 = (0, _crypto.createHash)("sha256").update(buffer).digest("hex");
existingFilename = filenameBySha256[sha256];
if (existingFilename === undefined) {
_fsExtra["default"].writeFileSync(filePath, buffer, {
encoding: "utf8"
});
fileInfos.filename = filename;
filenameBySha256[sha256] = filename;
} else {
fileInfos.filename = existingFilename;
}
fileInfos.sha256 = sha256;
fileInfos.status = response.status;
}
_context4.next = 37;
break;
case 32:
if (response.status === 404) {
if (!options.silent) {
console.warn(" Page \"".concat(url, "\" not found."));
}
} else {
console.error(" Error:\n".concat(JSON.stringify({
code: response.status,
message: response.statusText
}, null, 2)));
}
_fsExtra["default"].removeSync(filePath);
delete fileInfos.filename;
delete fileInfos.sha256;
fileInfos.status = response.status;
case 37:
case "end":
return _context4.stop();
}
}, _loop);
});
_iterator4.s();
case 9:
if ((_step4 = _iterator4.n()).done) {
_context5.next = 16;
break;
}
return _context5.delegateYield(_loop(), "t0", 11);
case 11:
_ret = _context5.t0;
if (!(_ret === 0)) {
_context5.next = 14;
break;
}
return _context5.abrupt("continue", 14);
case 14:
_context5.next = 9;
break;
case 16:
_context5.next = 21;
break;
case 18:
_context5.prev = 18;
_context5.t1 = _context5["catch"](6);
_iterator4.e(_context5.t1);
case 21:
_context5.prev = 21;
_iterator4.f();
return _context5.finish(21);
case 24:
_fsExtra["default"].writeJsonSync(indexPath, index, {
encoding: "utf-8",
spaces: 2
});
case 25:
case "end":
return _context5.stop();
}
}, _callee4, null, [[6, 18, 21, 24]]);
}));
return _processDocumentOrDivision.apply(this, arguments);
}
function main(argv) {
var options = parseArgs(argv);
return retrieveDocuments(options);
}
/* istanbul ignore if */
if (process.argv[1].endsWith("retrieve_documents.ts")) {
main(process.argv).then(function (exitCode) {
return process.exit(exitCode);
})["catch"](function (error) {
console.log(error);
process.exit(1);
});
}
//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJuYW1lcyI6WyJfYXNzZXJ0IiwiX2ludGVyb3BSZXF1aXJlRGVmYXVsdCIsInJlcXVpcmUiLCJfY29tbWFuZExpbmVBcmdzIiwiX2NyeXB0byIsIl9kYXRlRm5zIiwiX2ZzRXh0cmEiLCJfcGF0aCIsIl9kb3NzaWVyc19sZWdpc2xhdGlmcyIsImdpdCIsIl9pbnRlcm9wUmVxdWlyZVdpbGRjYXJkIiwiX2xvYWRlcnMiLCJfcGFyc2VycyIsIl91cmxzIiwiX2NsaV9oZWxwZXJzIiwiX2dldFJlcXVpcmVXaWxkY2FyZENhY2hlIiwiZSIsIldlYWtNYXAiLCJyIiwidCIsIl9fZXNNb2R1bGUiLCJfdHlwZW9mIiwiaGFzIiwiZ2V0IiwibiIsIl9fcHJvdG9fXyIsImEiLCJPYmplY3QiLCJkZWZpbmVQcm9wZXJ0eSIsImdldE93blByb3BlcnR5RGVzY3JpcHRvciIsInUiLCJoYXNPd25Qcm9wZXJ0eSIsImNhbGwiLCJpIiwic2V0IiwiX2NyZWF0ZUZvck9mSXRlcmF0b3JIZWxwZXIiLCJTeW1ib2wiLCJpdGVyYXRvciIsIkFycmF5IiwiaXNBcnJheSIsIl91bnN1cHBvcnRlZEl0ZXJhYmxlVG9BcnJheSIsImxlbmd0aCIsIl9uIiwiRiIsInMiLCJkb25lIiwidmFsdWUiLCJmIiwiVHlwZUVycm9yIiwibyIsIm5leHQiLCJfYXJyYXlMaWtlVG9BcnJheSIsInRvU3RyaW5nIiwic2xpY2UiLCJjb25zdHJ1Y3RvciIsIm5hbWUiLCJmcm9tIiwidGVzdCIsIl9yZWdlbmVyYXRvclJ1bnRpbWUiLCJwcm90b3R5cGUiLCJjIiwiYXN5bmNJdGVyYXRvciIsInRvU3RyaW5nVGFnIiwiZGVmaW5lIiwiZW51bWVyYWJsZSIsImNvbmZpZ3VyYWJsZSIsIndyaXRhYmxlIiwid3JhcCIsIkdlbmVyYXRvciIsImNyZWF0ZSIsIkNvbnRleHQiLCJtYWtlSW52b2tlTWV0aG9kIiwidHJ5Q2F0Y2giLCJ0eXBlIiwiYXJnIiwiaCIsImwiLCJ5IiwiR2VuZXJhdG9yRnVuY3Rpb24iLCJHZW5lcmF0b3JGdW5jdGlvblByb3RvdHlwZSIsInAiLCJkIiwiZ2V0UHJvdG90eXBlT2YiLCJ2IiwidmFsdWVzIiwiZyIsImRlZmluZUl0ZXJhdG9yTWV0aG9kcyIsImZvckVhY2giLCJfaW52b2tlIiwiQXN5bmNJdGVyYXRvciIsImludm9rZSIsInJlc29sdmUiLCJfX2F3YWl0IiwidGhlbiIsImNhbGxJbnZva2VXaXRoTWV0aG9kQW5kQXJnIiwiRXJyb3IiLCJtZXRob2QiLCJkZWxlZ2F0ZSIsIm1heWJlSW52b2tlRGVsZWdhdGUiLCJzZW50IiwiX3NlbnQiLCJkaXNwYXRjaEV4Y2VwdGlvbiIsImFicnVwdCIsInJlc3VsdE5hbWUiLCJuZXh0TG9jIiwicHVzaFRyeUVudHJ5IiwidHJ5TG9jIiwiY2F0Y2hMb2MiLCJmaW5hbGx5TG9jIiwiYWZ0ZXJMb2MiLCJ0cnlFbnRyaWVzIiwicHVzaCIsInJlc2V0VHJ5RW50cnkiLCJjb21wbGV0aW9uIiwicmVzZXQiLCJpc05hTiIsImRpc3BsYXlOYW1lIiwiaXNHZW5lcmF0b3JGdW5jdGlvbiIsIm1hcmsiLCJzZXRQcm90b3R5cGVPZiIsImF3cmFwIiwiYXN5bmMiLCJQcm9taXNlIiwia2V5cyIsInJldmVyc2UiLCJwb3AiLCJwcmV2IiwiY2hhckF0Iiwic3RvcCIsInJ2YWwiLCJoYW5kbGUiLCJjb21wbGV0ZSIsImZpbmlzaCIsIl9jYXRjaCIsImRlbGVnYXRlWWllbGQiLCJhc3luY0dlbmVyYXRvclN0ZXAiLCJfYXN5bmNUb0dlbmVyYXRvciIsImFyZ3VtZW50cyIsImFwcGx5IiwiX25leHQiLCJfdGhyb3ciLCJwYXJzZUFyZ3MiLCJhcmd2Iiwib3B0aW9uc0RlZmluaXRpb25zIiwiY29tbWl0T3B0aW9uIiwibGVnaXNsYXR1cmVPcHRpb24iLCJyZW1vdGVPcHRpb24iLCJzaWxlbnRPcHRpb24iLCJ2ZXJib3NlT3B0aW9uIiwiZGF0YURpckRlZmF1bHRPcHRpb24iLCJwdWxsT3B0aW9uIiwiYWxpYXMiLCJoZWxwIiwiQm9vbGVhbiIsImRvY3VtZW50Rm9ybWF0T3B0aW9uIiwibXVsdGlwbGUiLCJTdHJpbmciLCJvcHRpb25zIiwiY29tbWFuZExpbmVBcmdzIiwidG9kYXkiLCJEYXRlIiwiZmV0Y2hXaXRoUmV0cnkiLCJfeCIsIl9mZXRjaFdpdGhSZXRyeSIsIl9jYWxsZWUiLCJ1cmwiLCJyZXRyaWVzIiwiYmFja29mZiIsImF0dGVtcHQiLCJfYXJncyIsIl9jYWxsZWUkIiwiX2NvbnRleHQiLCJ1bmRlZmluZWQiLCJmZXRjaCIsInQwIiwiY29uc29sZSIsIndhcm4iLCJjb25jYXQiLCJzZXRUaW1lb3V0IiwiZG93bmxvYWRBbmRQYXJzZSIsIl94MiIsIl94MyIsIl94NCIsIl9kb3dubG9hZEFuZFBhcnNlIiwiX2NhbGxlZTIiLCJkb2N1bWVudCIsImRvY3VtZW50c0RpciIsIl9vcHRpb25zJGxlZ2lzbGF0dXJlIiwiZGF0YURpciIsImRvY3VtZW50c0ZpbGVzRGlyIiwiZG9jdW1lbnREaXIiLCJkb2N1bWVudEZpbGVEaXIiLCJkb2N1bWVudFBhdGgiLCJodG1sUGF0aCIsImh0bWwiLCJwYXJzZWREb2N1bWVudCIsImpzb24iLCJkb2N1bWVudEpzb24iLCJfY2FsbGVlMiQiLCJfY29udGV4dDIiLCJ1aWQiLCJzdWJzdHJpbmciLCJpbmNsdWRlcyIsImxlZ2lzbGF0dXJlIiwiZmV0Y2hEb2N1bWVudHMiLCJwYXRoIiwiam9pbiIsInBhdGhGcm9tRG9jdW1lbnRVaWQiLCJmcyIsImVuc3VyZURpclN5bmMiLCJwcm9jZXNzRG9jdW1lbnRPckRpdmlzaW9uIiwicGFyc2VEb2N1bWVudHMiLCJleGlzdHNTeW5jIiwicmVhZEZpbGVTeW5jIiwiZW5jb2RpbmciLCJwYXJzZVRleHRlIiwiSlNPTiIsInBhcnNlIiwic3ViZGl2aXNpb25zIiwid3JpdGVGaWxlU3luYyIsInN0cmluZ2lmeSIsImxvZyIsInJldHJpZXZlRG9jdW1lbnRzIiwiX3g1IiwiX3JldHJpZXZlRG9jdW1lbnRzIiwiX2NhbGxlZTMiLCJfaXRlcmF0b3IiLCJfc3RlcCIsImZpbGVuYW1lIiwiZmlyc3RVaWQiLCJza2lwIiwiX2l0ZXJhdG9yMiIsIl9zdGVwMiIsIl9pdGVyYXRvcjMiLCJfc3RlcDMiLCJkb2N1bWVudE9yRGl2aXNpb24iLCJfY2FsbGVlMyQiLCJfY29udGV4dDMiLCJhc3NlcnQiLCJjb21taXQiLCJwdWxsIiwicmVzZXRBbmRQdWxsIiwiZnVsbCIsInJlYWRkaXJTeW5jIiwicmVtb3ZlU3luYyIsIml0ZXJMb2FkQXNzZW1ibGVlRG9jdW1lbnRzIiwid2Fsa0RvY3VtZW50QW5kRGl2aXNpb25zIiwidDEiLCJ0MiIsImNvbW1pdEFuZFB1c2giLCJyZW1vdGUiLCJfeDYiLCJfeDciLCJfeDgiLCJfcHJvY2Vzc0RvY3VtZW50T3JEaXZpc2lvbiIsIl9jYWxsZWU0IiwiZmlsZW5hbWVCeVNoYTI1NiIsImluZGV4UGF0aCIsImluZGV4IiwiX2l0ZXJhdG9yNCIsIl9zdGVwNCIsIl9sb29wIiwiX3JldCIsIl9jYWxsZWU0JCIsIl9jb250ZXh0NSIsInBhdGhFeGlzdHNTeW5jIiwicmVhZEpzb25TeW5jIiwiaXRlckRvY3VtZW50T3JEaXZpc2lvblVybHMiLCJfZG9jdW1lbnRPckRpdmlzaW9uJGMiLCJfaW5kZXgkZm9ybWF0IiwiX2Zvcm1hdEZpbGVzSW5mb3MkZmluIiwiX2RvY3VtZW50T3JEaXZpc2lvbiRjMiIsIl9zdGVwNCR2YWx1ZSIsImZvcm1hdCIsImZpbGVFeHRlbnNpb24iLCJkb2N1bWVudFR5cGUiLCJmb3JtYXRGaWxlc0luZm9zIiwiZmlsZUluZm9zIiwicmVzcG9uc2UiLCJmaWxlUGF0aCIsImFycmF5QnVmZmVyIiwiYnVmZmVyIiwic2hhMjU2IiwiZXhpc3RpbmdGaWxlbmFtZSIsIl9sb29wJCIsIl9jb250ZXh0NCIsIkRvY3VtZW50VXJsRm9ybWF0IiwiUGRmIiwic3BsaXQiLCJjbGFzc2lmaWNhdGlvbiIsImNvZGUiLCJmaW5kIiwiZmlsZSIsInN0YXR1cyIsImRpZmZlcmVuY2VJbkRheXMiLCJjeWNsZURlVmllIiwiY2hyb25vIiwiZGF0ZUNyZWF0aW9uIiwiZGF0ZURlcG90Iiwic2lsZW50Iiwib2siLCJCdWZmZXIiLCJzdWJhcnJheSIsInN0YXJ0c1dpdGgiLCJjcmVhdGVIYXNoIiwidXBkYXRlIiwiZGlnZXN0IiwiZXJyb3IiLCJtZXNzYWdlIiwic3RhdHVzVGV4dCIsIndyaXRlSnNvblN5bmMiLCJzcGFjZXMiLCJtYWluIiwicHJvY2VzcyIsImVuZHNXaXRoIiwiZXhpdENvZGUiLCJleGl0Il0sInNvdXJjZXMiOlsiLi4vLi4vc3JjL3NjcmlwdHMvcmV0cmlldmVfZG9jdW1lbnRzLnRzIl0sInNvdXJjZXNDb250ZW50IjpbImltcG9ydCBhc3NlcnQgZnJvbSBcImFzc2VydFwiXG5pbXBvcnQgY29tbWFuZExpbmVBcmdzIGZyb20gXCJjb21tYW5kLWxpbmUtYXJnc1wiXG5pbXBvcnQgeyBjcmVhdGVIYXNoIH0gZnJvbSBcImNyeXB0b1wiXG5pbXBvcnQgeyBkaWZmZXJlbmNlSW5EYXlzIH0gZnJvbSBcImRhdGUtZm5zXCJcbmltcG9ydCBmcyBmcm9tIFwiZnMtZXh0cmFcIlxuaW1wb3J0IHBhdGggZnJvbSBcInBhdGhcIlxuXG5pbXBvcnQge1xuICBEb2N1bWVudEZpbGVJbmZvcyxcbiAgRG9jdW1lbnRGaWxlc0luZGV4LFxuICB3YWxrRG9jdW1lbnRBbmREaXZpc2lvbnMsXG59IGZyb20gXCIuLi9kb3NzaWVyc19sZWdpc2xhdGlmc1wiXG5pbXBvcnQgKiBhcyBnaXQgZnJvbSBcIi4uL2dpdFwiXG5pbXBvcnQgeyBpdGVyTG9hZEFzc2VtYmxlZURvY3VtZW50cywgcGF0aEZyb21Eb2N1bWVudFVpZCB9IGZyb20gXCIuLi9sb2FkZXJzXCJcbmltcG9ydCB7IHBhcnNlVGV4dGUgfSBmcm9tIFwiLi4vcGFyc2Vyc1wiXG5pbXBvcnQgeyBEb2N1bWVudFVybEZvcm1hdCwgaXRlckRvY3VtZW50T3JEaXZpc2lvblVybHMgfSBmcm9tIFwiLi4vdXJsc1wiXG5pbXBvcnQge1xuICBjb21taXRPcHRpb24sXG4gIGRhdGFEaXJEZWZhdWx0T3B0aW9uLFxuICBsZWdpc2xhdHVyZU9wdGlvbixcbiAgcHVsbE9wdGlvbixcbiAgcmVtb3RlT3B0aW9uLFxuICBzaWxlbnRPcHRpb24sXG4gIHZlcmJvc2VPcHRpb24sXG4gIGRvY3VtZW50Rm9ybWF0T3B0aW9uLFxufSBmcm9tIFwiLi9zaGFyZWQvY2xpX2hlbHBlcnNcIlxuXG5mdW5jdGlvbiBwYXJzZUFyZ3MoYXJndjogc3RyaW5nW10pOiBhbnkge1xuICBjb25zdCBvcHRpb25zRGVmaW5pdGlvbnMgPSBbXG4gICAgY29tbWl0T3B0aW9uLFxuICAgIGxlZ2lzbGF0dXJlT3B0aW9uLFxuICAgIHJlbW90ZU9wdGlvbixcbiAgICBzaWxlbnRPcHRpb24sXG4gICAgdmVyYm9zZU9wdGlvbixcbiAgICBkYXRhRGlyRGVmYXVsdE9wdGlvbixcbiAgICBwdWxsT3B0aW9uLFxuICAgIHtcbiAgICAgIGFsaWFzOiBcImZcIixcbiAgICAgIGhlbHA6IFwicmV0cmlldmUgYWxsIGRvY3VtZW50cywgZXZlbiBhbHJlYWR5IHJldHJpZXZlZCBvbmVzXCIsXG4gICAgICBuYW1lOiBcImZ1bGxcIixcbiAgICAgIHR5cGU6IEJvb2xlYW4sXG4gICAgfSxcbiAgICB7XG4gICAgICBhbGlhczogXCJuXCIsXG4gICAgICBoZWxwOiBcInRyeSB0byBhbHNvIHJldHJpZXZlIGRvY3VtZW50cyB0aGF0IHdlcmUgcHJldmlvdXNseSBub3QgZm91bmRcIixcbiAgICAgIG5hbWU6IFwibm90LWZvdW5kXCIsXG4gICAgICB0eXBlOiBCb29sZWFuLFxuICAgIH0sXG4gICAgZG9jdW1lbnRGb3JtYXRPcHRpb24sXG4gICAge1xuICAgICAgYWxpYXM6IFwiVFwiLFxuICAgICAgaGVscDogXCJ0eXBlIG9mIGRvY3VtZW50cyB0byByZXRyaWV2ZSAoZm9yIGV4YW1wbGU6IFBJT04pXCIsXG4gICAgICBtdWx0aXBsZTogdHJ1ZSxcbiAgICAgIG5hbWU6IFwiZG9jdW1lbnQtdHlwZVwiLFxuICAgICAgdHlwZTogU3RyaW5nLFxuICAgIH0sXG4gIF1cbiAgY29uc3Qgb3B0aW9ucyA9IGNvbW1hbmRMaW5lQXJncyhvcHRpb25zRGVmaW5pdGlvbnMsIHtcbiAgICBhcmd2OiBhcmd2LFxuICB9KVxuICByZXR1cm4gb3B0aW9uc1xufVxuXG5jb25zdCB0b2RheSA9IG5ldyBEYXRlKClcblxuYXN5bmMgZnVuY3Rpb24gZmV0Y2hXaXRoUmV0cnkoXG4gIHVybDogc3RyaW5nLFxuICByZXRyaWVzOiBudW1iZXIgPSAzLFxuICBiYWNrb2ZmOiBudW1iZXIgPSAzMDAsXG4pOiBQcm9taXNlPFJlc3BvbnNlPiB7XG4gIGZvciAobGV0IGF0dGVtcHQgPSAwOyBhdHRlbXB0IDwgcmV0cmllczsgYXR0ZW1wdCsrKSB7XG4gICAgdHJ5IHtcbiAgICAgIHJldHVybiBhd2FpdCBmZXRjaCh1cmwpXG4gICAgfSBjYXRjaCAoZXJyb3IpIHtcbiAgICAgIGlmIChhdHRlbXB0ID09PSByZXRyaWVzIC0gMSkge1xuICAgICAgICB0aHJvdyBlcnJvclxuICAgICAgfVxuICAgICAgY29uc29sZS53YXJuKFxuICAgICAgICBgRmV0Y2ggYXR0ZW1wdCAke2F0dGVtcHQgKyAxfSBmb3IgJHt1cmx9IGZhaWxlZC4gUmV0cnlpbmcgaW4gJHtiYWNrb2ZmfW1zLi4uYCxcbiAgICAgIClcbiAgICAgIGF3YWl0IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiBzZXRUaW1lb3V0KHJlc29sdmUsIGJhY2tvZmYpKVxuICAgICAgYmFja29mZiAqPSAyXG4gICAgfVxuICB9XG4gIHRocm93IG5ldyBFcnJvcihgRmFpbGVkIHRvIGZldGNoICR7dXJsfSBhZnRlciAke3JldHJpZXN9IGF0dGVtcHRzYClcbn1cblxuZXhwb3J0IGFzeW5jIGZ1bmN0aW9uIGRvd25sb2FkQW5kUGFyc2UoXG4gIGRvY3VtZW50OiBhbnksXG4gIGRvY3VtZW50c0Rpcjogc3RyaW5nLFxuICBvcHRpb25zOiBhbnksXG4pOiBQcm9taXNlPHZvaWQ+IHtcbiAgaWYgKGRvY3VtZW50LnVpZC5zdWJzdHJpbmcoNCwgNikgPT09IFwiU05cIikge1xuICAgIHJldHVyblxuICB9XG5cbiAgaWYgKFtcIlJBUFBcIiwgXCJSSU5GXCJdLmluY2x1ZGVzKGRvY3VtZW50LnVpZC5zdWJzdHJpbmcoMCwgNCkpKSB7XG4gICAgcmV0dXJuXG4gIH1cblxuICBpZiAob3B0aW9ucy5sZWdpc2xhdHVyZT8udG9TdHJpbmcoKSAhPT0gZG9jdW1lbnQubGVnaXNsYXR1cmUpIHtcbiAgICByZXR1cm5cbiAgfVxuXG4gIGlmIChvcHRpb25zLmZldGNoRG9jdW1lbnRzKSB7XG4gICAgY29uc3QgZGF0YURpciA9IG9wdGlvbnMuZGF0YURpclxuICAgIGNvbnN0IGRvY3VtZW50c0ZpbGVzRGlyID0gcGF0aC5qb2luKGRhdGFEaXIsIFwiRG9jdW1lbnRzXCIpXG4gICAgY29uc3QgZG9jdW1lbnREaXIgPSBwYXRoRnJvbURvY3VtZW50VWlkKFxuICAgICAgYCR7ZG9jdW1lbnRzRGlyfS9kb2N1bWVudHNgLFxuICAgICAgZG9jdW1lbnQudWlkLFxuICAgIClcbiAgICBjb25zdCBkb2N1bWVudEZpbGVEaXIgPSBwYXRoRnJvbURvY3VtZW50VWlkKGRvY3VtZW50c0ZpbGVzRGlyLCBkb2N1bWVudC51aWQpXG5cbiAgICBmcy5lbnN1cmVEaXJTeW5jKGRvY3VtZW50c0ZpbGVzRGlyKVxuICAgIGF3YWl0IHByb2Nlc3NEb2N1bWVudE9yRGl2aXNpb24oZG9jdW1lbnQsIGRvY3VtZW50c0ZpbGVzRGlyLCBvcHRpb25zKVxuXG4gICAgaWYgKG9wdGlvbnMucGFyc2VEb2N1bWVudHMpIHtcbiAgICAgIGNvbnN0IGRvY3VtZW50UGF0aCA9IGAke2RvY3VtZW50RGlyfS5qc29uYFxuICAgICAgY29uc3QgaHRtbFBhdGggPSBwYXRoLmpvaW4oZG9jdW1lbnRGaWxlRGlyLCBgZHluLW9wZW5kYXRhLmh0bWxgKVxuXG4gICAgICBpZiAoZnMuZXhpc3RzU3luYyhodG1sUGF0aCkpIHtcbiAgICAgICAgY29uc3QgaHRtbCA9IGZzLnJlYWRGaWxlU3luYyhodG1sUGF0aCwgeyBlbmNvZGluZzogXCJ1dGYtOFwiIH0pXG4gICAgICAgIHRyeSB7XG4gICAgICAgICAgY29uc3QgcGFyc2VkRG9jdW1lbnQgPSBwYXJzZVRleHRlKFxuICAgICAgICAgICAgXCJodHRwczovL3d3dy5hc3NlbWJsZWUtbmF0aW9uYWxlLmZyXCIsXG4gICAgICAgICAgICBodG1sLFxuICAgICAgICAgIClcbiAgICAgICAgICBpZiAocGFyc2VkRG9jdW1lbnQgJiYgZnMuZXhpc3RzU3luYyhkb2N1bWVudFBhdGgpKSB7XG4gICAgICAgICAgICBjb25zdCBqc29uID0gZnMucmVhZEZpbGVTeW5jKGRvY3VtZW50UGF0aCwgeyBlbmNvZGluZzogXCJ1dGYtOFwiIH0pXG4gICAgICAgICAgICBjb25zdCBkb2N1bWVudEpzb24gPSBKU09OLnBhcnNlKGpzb24pXG4gICAgICAgICAgICBkb2N1bWVudEpzb24uc3ViZGl2aXNpb25zID0gcGFyc2VkRG9jdW1lbnQ/LnN1YmRpdmlzaW9uc1xuICAgICAgICAgICAgZnMud3JpdGVGaWxlU3luYyhcbiAgICAgICAgICAgICAgZG9jdW1lbnRQYXRoLFxuICAgICAgICAgICAgICBKU09OLnN0cmluZ2lmeShkb2N1bWVudEpzb24sIG51bGwsIDIpLFxuICAgICAgICAgICAgKVxuICAgICAgICAgIH1cbiAgICAgICAgfSBjYXRjaCAoZSkge1xuICAgICAgICAgIGNvbnNvbGUubG9nKGBVbmFibGUgdG8gcGFyc2UgZG9jdW1lbnQgJHtkb2N1bWVudC51aWR9YClcbiAgICAgICAgICByZXR1cm5cbiAgICAgICAgfVxuICAgICAgfVxuICAgIH1cbiAgfVxufVxuXG5hc3luYyBmdW5jdGlvbiByZXRyaWV2ZURvY3VtZW50cyhvcHRpb25zOiBhbnkpOiBQcm9taXNlPG51bWJlcj4ge1xuICBhc3NlcnQoXG4gICAgIW9wdGlvbnMuY29tbWl0IHx8ICFvcHRpb25zLnVpZCxcbiAgICAnT3B0aW9ucyBcImNvbW1pdFwiICYgXCJ1aWRcIiBhcmUgaW5jb21wYXRpYmxlJyxcbiAgKVxuXG4gIGNvbnN0IGRhdGFEaXIgPSBvcHRpb25zLmRhdGFEaXJcbiAgY29uc3QgZG9jdW1lbnRzRGlyID0gcGF0aC5qb2luKGRhdGFEaXIsIFwiRG9jdW1lbnRzXCIpXG4gIGlmIChvcHRpb25zLnB1bGwpIHtcbiAgICBnaXQucmVzZXRBbmRQdWxsKGRvY3VtZW50c0RpcilcbiAgfVxuICBmcy5lbnN1cmVEaXJTeW5jKGRvY3VtZW50c0RpcilcblxuICBpZiAob3B0aW9ucy5mdWxsICYmICFvcHRpb25zLnVpZCkge1xuICAgIGZvciAoY29uc3QgZmlsZW5hbWUgb2YgZnMucmVhZGRpclN5bmMoZG9jdW1lbnRzRGlyKSkge1xuICAgICAgaWYgKGZpbGVuYW1lWzBdID09PSBcIi5cIikge1xuICAgICAgICBjb250aW51ZVxuICAgICAgfVxuICAgICAgZnMucmVtb3ZlU3luYyhwYXRoLmpvaW4oZG9jdW1lbnRzRGlyLCBmaWxlbmFtZSkpXG4gICAgfVxuICB9XG5cbiAgY29uc3QgZmlyc3RVaWQgPSBvcHRpb25zLnVpZFxuICBsZXQgc2tpcCA9IEJvb2xlYW4oZmlyc3RVaWQpXG4gIGZvciAoY29uc3QgeyBkb2N1bWVudCB9IG9mIGl0ZXJMb2FkQXNzZW1ibGVlRG9jdW1lbnRzKFxuICAgIGRhdGFEaXIsXG4gICAgb3B0aW9ucy5sZWdpc2xhdHVyZSxcbiAgKSkge1xuICAgIGZvciAoY29uc3QgZG9jdW1lbnRPckRpdmlzaW9uIG9mIHdhbGtEb2N1bWVudEFuZERpdmlzaW9ucyhkb2N1bWVudCkpIHtcbiAgICAgIC8vIElnbm9yZSBkb2N1bWVudHMgZnJvbSBTw6luYXQuXG4gICAgICBpZiAoZG9jdW1lbnRPckRpdmlzaW9uLnVpZC5zdWJzdHJpbmcoNCwgNikgPT09IFwiU05cIikge1xuICAgICAgICBjb250aW51ZVxuICAgICAgfVxuXG4gICAgICBpZiAoc2tpcCkge1xuICAgICAgICBpZiAoZG9jdW1lbnRPckRpdmlzaW9uLnVpZCA9PT0gZmlyc3RVaWQpIHtcbiAgICAgICAgICBza2lwID0gZmFsc2VcbiAgICAgICAgfSBlbHNlIHtcbiAgICAgICAgICBjb250aW51ZVxuICAgICAgICB9XG4gICAgICB9XG5cbiAgICAgIGF3YWl0IHByb2Nlc3NEb2N1bWVudE9yRGl2aXNpb24oZG9jdW1lbnRPckRpdmlzaW9uLCBkb2N1bWVudHNEaXIsIG9wdGlvbnMpXG4gICAgfVxuICB9XG5cbiAgaWYgKG9wdGlvbnMuY29tbWl0KSB7XG4gICAgcmV0dXJuIGdpdC5jb21taXRBbmRQdXNoKGRvY3VtZW50c0RpciwgXCJOb3V2ZWxsZSBtb2lzc29uXCIsIG9wdGlvbnMucmVtb3RlKVxuICB9XG4gIHJldHVybiAwXG59XG5cbmFzeW5jIGZ1bmN0aW9uIHByb2Nlc3NEb2N1bWVudE9yRGl2aXNpb24oXG4gIGRvY3VtZW50T3JEaXZpc2lvbjogYW55LFxuICBkb2N1bWVudHNEaXI6IHN0cmluZyxcbiAgb3B0aW9uczogYW55LFxuKTogUHJvbWlzZTx2b2lkPiB7XG4gIGNvbnN0IGRvY3VtZW50RGlyID0gcGF0aEZyb21Eb2N1bWVudFVpZChkb2N1bWVudHNEaXIsIGRvY3VtZW50T3JEaXZpc2lvbi51aWQpXG4gIGZzLmVuc3VyZURpclN5bmMoZG9jdW1lbnREaXIpXG5cbiAgY29uc3QgZmlsZW5hbWVCeVNoYTI1NjogeyBbZGlnZXN0OiBzdHJpbmddOiBzdHJpbmcgfSA9IHt9XG4gIGNvbnN0IGluZGV4UGF0aCA9IHBhdGguam9pbihkb2N1bWVudERpciwgXCJpbmRleC5qc29uXCIpXG4gIGNvbnN0IGluZGV4ID0gKFxuICAgIGZzLnBhdGhFeGlzdHNTeW5jKGluZGV4UGF0aCkgPyBmcy5yZWFkSnNvblN5bmMoaW5kZXhQYXRoKSA6IHt9XG4gICkgYXMgRG9jdW1lbnRGaWxlc0luZGV4XG5cbiAgZm9yIChjb25zdCB7IGZvcm1hdCwgdHlwZSwgdXJsIH0gb2YgaXRlckRvY3VtZW50T3JEaXZpc2lvblVybHMoXG4gICAgZG9jdW1lbnRPckRpdmlzaW9uLFxuICApKSB7XG4gICAgY29uc3QgZmlsZW5hbWUgPSBgJHt0eXBlfS4ke2Zvcm1hdCA9PT0gRG9jdW1lbnRVcmxGb3JtYXQuUGRmID8gXCJwZGZcIiA6IFwiaHRtbFwifWBcblxuICAgIC8vIEZpbHRlciBieSBmaWxlIGV4dGVuc2lvbiBpZiBvcHRpb24gaXMgcGFzc2VkXG4gICAgY29uc3QgZmlsZUV4dGVuc2lvbiA9IHVybC5zcGxpdChcIi5cIikucG9wKClcbiAgICBpZiAob3B0aW9ucy5mb3JtYXQgJiYgZmlsZUV4dGVuc2lvbiAmJiBvcHRpb25zLmZvcm1hdCAhPT0gZmlsZUV4dGVuc2lvbikge1xuICAgICAgY29udGludWVcbiAgICB9XG5cbiAgICBpZiAoXG4gICAgICBvcHRpb25zLmxlZ2lzbGF0dXJlICYmXG4gICAgICBvcHRpb25zLmxlZ2lzbGF0dXJlLnRvU3RyaW5nKCkgIT09IGRvY3VtZW50T3JEaXZpc2lvbi5sZWdpc2xhdHVyZVxuICAgICkge1xuICAgICAgY29udGludWVcbiAgICB9XG5cbiAgICAvLyBGaWx0ZXIgYnkgZG9jdW1lbnQgdHlwZSBpZiBvcHRpb24gaXMgcGFzc2VkXG4gICAgY29uc3QgZG9jdW1lbnRUeXBlID0gZG9jdW1lbnRPckRpdmlzaW9uLmNsYXNzaWZpY2F0aW9uPy50eXBlPy5jb2RlXG4gICAgaWYgKFxuICAgICAgb3B0aW9uc1tcImRvY3VtZW50LXR5cGVcIl0gIT09IHVuZGVmaW5lZCAmJlxuICAgICAgIW9wdGlvbnNbXCJkb2N1bWVudC10eXBlXCJdLmluY2x1ZGVzKGRvY3VtZW50VHlwZSlcbiAgICApIHtcbiAgICAgIGNvbnRpbnVlXG4gICAgfVxuXG4gICAgbGV0IGZvcm1hdEZpbGVzSW5mb3MgPSBpbmRleFtmb3JtYXRdID8/IChpbmRleFtmb3JtYXRdID0gW10pXG4gICAgbGV0IGZpbGVJbmZvcyA9XG4gICAgICBmb3JtYXRGaWxlc0luZm9zLmZpbmQoKGZpbGUpID0+IGZpbGUudXJsID09PSB1cmwpID8/XG4gICAgICAoe30gYXMgRG9jdW1lbnRGaWxlSW5mb3MpXG4gICAgaWYgKCFmb3JtYXRGaWxlc0luZm9zLmluY2x1ZGVzKGZpbGVJbmZvcykpIHtcbiAgICAgIGZvcm1hdEZpbGVzSW5mb3MucHVzaChmaWxlSW5mb3MpXG4gICAgfVxuICAgIGZpbGVJbmZvcy51cmwgPSB1cmxcblxuICAgIGlmIChmaWxlSW5mb3Muc3RhdHVzID09PSAyMDAgJiYgIW9wdGlvbnMuZnVsbCkge1xuICAgICAgZmlsZW5hbWVCeVNoYTI1NltmaWxlSW5mb3Muc2hhMjU2IGFzIHN0cmluZ10gPSBmaWxlbmFtZVxuICAgICAgY29udGludWVcbiAgICB9XG4gICAgaWYgKFxuICAgICAgZmlsZUluZm9zLnN0YXR1cyA9PT0gNDA0ICYmXG4gICAgICAhb3B0aW9uc1tcIm5vdC1mb3VuZFwiXSAmJlxuICAgICAgZGlmZmVyZW5jZUluRGF5cyhcbiAgICAgICAgdG9kYXksXG4gICAgICAgIGRvY3VtZW50T3JEaXZpc2lvbi5jeWNsZURlVmllLmNocm9uby5kYXRlQ3JlYXRpb24gPz9cbiAgICAgICAgICAoZG9jdW1lbnRPckRpdmlzaW9uLmN5Y2xlRGVWaWUuY2hyb25vLmRhdGVEZXBvdCBhcyBEYXRlIHwgc3RyaW5nKSxcbiAgICAgICkgPiAxMFxuICAgICkge1xuICAgICAgY29udGludWVcbiAgICB9XG5cbiAgICBpZiAoIW9wdGlvbnMuc2lsZW50KSB7XG4gICAgICBjb25zb2xlLmxvZyhcbiAgICAgICAgYFJldHJpZXZpbmcgZG9jdW1lbnQgb3IgZGl2aXNpb24gJHtkb2N1bWVudE9yRGl2aXNpb24udWlkfSBhdCAke3VybH3igKZgLFxuICAgICAgKVxuICAgIH1cblxuICAgIGNvbnN0IHJlc3BvbnNlID0gYXdhaXQgZmV0Y2hXaXRoUmV0cnkodXJsKVxuICAgIGNvbnN0IGZpbGVQYXRoID0gcGF0aC5qb2luKGRvY3VtZW50RGlyLCBmaWxlbmFtZSlcblxuICAgIGlmIChyZXNwb25zZS5vaykge1xuICAgICAgY29uc3QgYXJyYXlCdWZmZXIgPSBhd