@tricoteuses/assemblee
Version:
Retrieve, clean up & handle French Assemblée nationale's open data
613 lines (611 loc) • 105 kB
JavaScript
"use strict";
require("core-js/modules/es.symbol.async-iterator.js");
require("core-js/modules/es.symbol.to-string-tag.js");
require("core-js/modules/es.array.for-each.js");
require("core-js/modules/es.array.from.js");
require("core-js/modules/es.array.is-array.js");
require("core-js/modules/es.array.reverse.js");
require("core-js/modules/es.array.slice.js");
require("core-js/modules/es.function.name.js");
require("core-js/modules/es.json.to-string-tag.js");
require("core-js/modules/es.math.to-string-tag.js");
require("core-js/modules/es.object.create.js");
require("core-js/modules/es.object.define-property.js");
require("core-js/modules/es.object.get-own-property-descriptor.js");
require("core-js/modules/es.object.get-prototype-of.js");
require("core-js/modules/es.object.set-prototype-of.js");
require("core-js/modules/es.promise.js");
require("core-js/modules/es.regexp.to-string.js");
require("core-js/modules/es.weak-map.js");
require("core-js/modules/web.dom-collections.for-each.js");
require("core-js/modules/es.symbol.js");
require("core-js/modules/es.symbol.description.js");
require("core-js/modules/es.symbol.iterator.js");
require("core-js/modules/es.array.concat.js");
require("core-js/modules/es.array.iterator.js");
require("core-js/modules/es.array.join.js");
require("core-js/modules/es.date.to-json.js");
require("core-js/modules/es.date.to-string.js");
require("core-js/modules/es.object.keys.js");
require("core-js/modules/es.object.to-string.js");
require("core-js/modules/es.regexp.exec.js");
require("core-js/modules/es.string.ends-with.js");
require("core-js/modules/es.string.iterator.js");
require("core-js/modules/es.string.replace.js");
require("core-js/modules/es.string.trim.js");
require("core-js/modules/web.dom-collections.iterator.js");
var _commandLineArgs = _interopRequireDefault(require("command-line-args"));
var _fsExtra = _interopRequireDefault(require("fs-extra"));
var _path = _interopRequireDefault(require("path"));
var _cleaners = require("../cleaners");
var _datasets = require("../datasets");
var _retrieve_documents = require("./retrieve_documents");
var _dates = require("../dates");
var _file_systems = require("../file_systems");
var git = _interopRequireWildcard(require("../git"));
var _acteurs_et_organes = require("../types/acteurs_et_organes");
var _agendas = require("../types/agendas");
var _amendements = require("../types/amendements");
var _dossiers_legislatifs = require("../types/dossiers_legislatifs");
var _scrutins = require("../types/scrutins");
var _questions = require("../types/questions");
var _debats = require("../types/debats");
var _cli_helpers = require("./shared/cli_helpers");
function _getRequireWildcardCache(e) { if ("function" != typeof WeakMap) return null; var r = new WeakMap(), t = new WeakMap(); return (_getRequireWildcardCache = function _getRequireWildcardCache(e) { return e ? t : r; })(e); }
function _interopRequireWildcard(e, r) { if (!r && e && e.__esModule) return e; if (null === e || "object" != _typeof(e) && "function" != typeof e) return { "default": e }; var t = _getRequireWildcardCache(r); if (t && t.has(e)) return t.get(e); var n = { __proto__: null }, a = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var u in e) if ("default" !== u && {}.hasOwnProperty.call(e, u)) { var i = a ? Object.getOwnPropertyDescriptor(e, u) : null; i && (i.get || i.set) ? Object.defineProperty(n, u, i) : n[u] = e[u]; } return n["default"] = e, t && t.set(e, n), n; }
function _interopRequireDefault(e) { return e && e.__esModule ? e : { "default": e }; }
function _regeneratorRuntime() { "use strict"; /*! regenerator-runtime -- Copyright (c) 2014-present, Facebook, Inc. -- license (MIT): https://github.com/facebook/regenerator/blob/main/LICENSE */ _regeneratorRuntime = function _regeneratorRuntime() { return e; }; var t, e = {}, r = Object.prototype, n = r.hasOwnProperty, o = Object.defineProperty || function (t, e, r) { t[e] = r.value; }, i = "function" == typeof Symbol ? Symbol : {}, a = i.iterator || "@@iterator", c = i.asyncIterator || "@@asyncIterator", u = i.toStringTag || "@@toStringTag"; function define(t, e, r) { return Object.defineProperty(t, e, { value: r, enumerable: !0, configurable: !0, writable: !0 }), t[e]; } try { define({}, ""); } catch (t) { define = function define(t, e, r) { return t[e] = r; }; } function wrap(t, e, r, n) { var i = e && e.prototype instanceof Generator ? e : Generator, a = Object.create(i.prototype), c = new Context(n || []); return o(a, "_invoke", { value: makeInvokeMethod(t, r, c) }), a; } function tryCatch(t, e, r) { try { return { type: "normal", arg: t.call(e, r) }; } catch (t) { return { type: "throw", arg: t }; } } e.wrap = wrap; var h = "suspendedStart", l = "suspendedYield", f = "executing", s = "completed", y = {}; function Generator() {} function GeneratorFunction() {} function GeneratorFunctionPrototype() {} var p = {}; define(p, a, function () { return this; }); var d = Object.getPrototypeOf, v = d && d(d(values([]))); v && v !== r && n.call(v, a) && (p = v); var g = GeneratorFunctionPrototype.prototype = Generator.prototype = Object.create(p); function defineIteratorMethods(t) { ["next", "throw", "return"].forEach(function (e) { define(t, e, function (t) { return this._invoke(e, t); }); }); } function AsyncIterator(t, e) { function invoke(r, o, i, a) { var c = tryCatch(t[r], t, o); if ("throw" !== c.type) { var u = c.arg, h = u.value; return h && "object" == _typeof(h) && n.call(h, "__await") ? e.resolve(h.__await).then(function (t) { invoke("next", t, i, a); }, function (t) { invoke("throw", t, i, a); }) : e.resolve(h).then(function (t) { u.value = t, i(u); }, function (t) { return invoke("throw", t, i, a); }); } a(c.arg); } var r; o(this, "_invoke", { value: function value(t, n) { function callInvokeWithMethodAndArg() { return new e(function (e, r) { invoke(t, n, e, r); }); } return r = r ? r.then(callInvokeWithMethodAndArg, callInvokeWithMethodAndArg) : callInvokeWithMethodAndArg(); } }); } function makeInvokeMethod(e, r, n) { var o = h; return function (i, a) { if (o === f) throw Error("Generator is already running"); if (o === s) { if ("throw" === i) throw a; return { value: t, done: !0 }; } for (n.method = i, n.arg = a;;) { var c = n.delegate; if (c) { var u = maybeInvokeDelegate(c, n); if (u) { if (u === y) continue; return u; } } if ("next" === n.method) n.sent = n._sent = n.arg;else if ("throw" === n.method) { if (o === h) throw o = s, n.arg; n.dispatchException(n.arg); } else "return" === n.method && n.abrupt("return", n.arg); o = f; var p = tryCatch(e, r, n); if ("normal" === p.type) { if (o = n.done ? s : l, p.arg === y) continue; return { value: p.arg, done: n.done }; } "throw" === p.type && (o = s, n.method = "throw", n.arg = p.arg); } }; } function maybeInvokeDelegate(e, r) { var n = r.method, o = e.iterator[n]; if (o === t) return r.delegate = null, "throw" === n && e.iterator["return"] && (r.method = "return", r.arg = t, maybeInvokeDelegate(e, r), "throw" === r.method) || "return" !== n && (r.method = "throw", r.arg = new TypeError("The iterator does not provide a '" + n + "' method")), y; var i = tryCatch(o, e.iterator, r.arg); if ("throw" === i.type) return r.method = "throw", r.arg = i.arg, r.delegate = null, y; var a = i.arg; return a ? a.done ? (r[e.resultName] = a.value, r.next = e.nextLoc, "return" !== r.method && (r.method = "next", r.arg = t), r.delegate = null, y) : a : (r.method = "throw", r.arg = new TypeError("iterator result is not an object"), r.delegate = null, y); } function pushTryEntry(t) { var e = { tryLoc: t[0] }; 1 in t && (e.catchLoc = t[1]), 2 in t && (e.finallyLoc = t[2], e.afterLoc = t[3]), this.tryEntries.push(e); } function resetTryEntry(t) { var e = t.completion || {}; e.type = "normal", delete e.arg, t.completion = e; } function Context(t) { this.tryEntries = [{ tryLoc: "root" }], t.forEach(pushTryEntry, this), this.reset(!0); } function values(e) { if (e || "" === e) { var r = e[a]; if (r) return r.call(e); if ("function" == typeof e.next) return e; if (!isNaN(e.length)) { var o = -1, i = function next() { for (; ++o < e.length;) if (n.call(e, o)) return next.value = e[o], next.done = !1, next; return next.value = t, next.done = !0, next; }; return i.next = i; } } throw new TypeError(_typeof(e) + " is not iterable"); } return GeneratorFunction.prototype = GeneratorFunctionPrototype, o(g, "constructor", { value: GeneratorFunctionPrototype, configurable: !0 }), o(GeneratorFunctionPrototype, "constructor", { value: GeneratorFunction, configurable: !0 }), GeneratorFunction.displayName = define(GeneratorFunctionPrototype, u, "GeneratorFunction"), e.isGeneratorFunction = function (t) { var e = "function" == typeof t && t.constructor; return !!e && (e === GeneratorFunction || "GeneratorFunction" === (e.displayName || e.name)); }, e.mark = function (t) { return Object.setPrototypeOf ? Object.setPrototypeOf(t, GeneratorFunctionPrototype) : (t.__proto__ = GeneratorFunctionPrototype, define(t, u, "GeneratorFunction")), t.prototype = Object.create(g), t; }, e.awrap = function (t) { return { __await: t }; }, defineIteratorMethods(AsyncIterator.prototype), define(AsyncIterator.prototype, c, function () { return this; }), e.AsyncIterator = AsyncIterator, e.async = function (t, r, n, o, i) { void 0 === i && (i = Promise); var a = new AsyncIterator(wrap(t, r, n, o), i); return e.isGeneratorFunction(r) ? a : a.next().then(function (t) { return t.done ? t.value : a.next(); }); }, defineIteratorMethods(g), define(g, u, "Generator"), define(g, a, function () { return this; }), define(g, "toString", function () { return "[object Generator]"; }), e.keys = function (t) { var e = Object(t), r = []; for (var n in e) r.push(n); return r.reverse(), function next() { for (; r.length;) { var t = r.pop(); if (t in e) return next.value = t, next.done = !1, next; } return next.done = !0, next; }; }, e.values = values, Context.prototype = { constructor: Context, reset: function reset(e) { if (this.prev = 0, this.next = 0, this.sent = this._sent = t, this.done = !1, this.delegate = null, this.method = "next", this.arg = t, this.tryEntries.forEach(resetTryEntry), !e) for (var r in this) "t" === r.charAt(0) && n.call(this, r) && !isNaN(+r.slice(1)) && (this[r] = t); }, stop: function stop() { this.done = !0; var t = this.tryEntries[0].completion; if ("throw" === t.type) throw t.arg; return this.rval; }, dispatchException: function dispatchException(e) { if (this.done) throw e; var r = this; function handle(n, o) { return a.type = "throw", a.arg = e, r.next = n, o && (r.method = "next", r.arg = t), !!o; } for (var o = this.tryEntries.length - 1; o >= 0; --o) { var i = this.tryEntries[o], a = i.completion; if ("root" === i.tryLoc) return handle("end"); if (i.tryLoc <= this.prev) { var c = n.call(i, "catchLoc"), u = n.call(i, "finallyLoc"); if (c && u) { if (this.prev < i.catchLoc) return handle(i.catchLoc, !0); if (this.prev < i.finallyLoc) return handle(i.finallyLoc); } else if (c) { if (this.prev < i.catchLoc) return handle(i.catchLoc, !0); } else { if (!u) throw Error("try statement without catch or finally"); if (this.prev < i.finallyLoc) return handle(i.finallyLoc); } } } }, abrupt: function abrupt(t, e) { for (var r = this.tryEntries.length - 1; r >= 0; --r) { var o = this.tryEntries[r]; if (o.tryLoc <= this.prev && n.call(o, "finallyLoc") && this.prev < o.finallyLoc) { var i = o; break; } } i && ("break" === t || "continue" === t) && i.tryLoc <= e && e <= i.finallyLoc && (i = null); var a = i ? i.completion : {}; return a.type = t, a.arg = e, i ? (this.method = "next", this.next = i.finallyLoc, y) : this.complete(a); }, complete: function complete(t, e) { if ("throw" === t.type) throw t.arg; return "break" === t.type || "continue" === t.type ? this.next = t.arg : "return" === t.type ? (this.rval = this.arg = t.arg, this.method = "return", this.next = "end") : "normal" === t.type && e && (this.next = e), y; }, finish: function finish(t) { for (var e = this.tryEntries.length - 1; e >= 0; --e) { var r = this.tryEntries[e]; if (r.finallyLoc === t) return this.complete(r.completion, r.afterLoc), resetTryEntry(r), y; } }, "catch": function _catch(t) { for (var e = this.tryEntries.length - 1; e >= 0; --e) { var r = this.tryEntries[e]; if (r.tryLoc === t) { var n = r.completion; if ("throw" === n.type) { var o = n.arg; resetTryEntry(r); } return o; } } throw Error("illegal catch attempt"); }, delegateYield: function delegateYield(e, r, n) { return this.delegate = { iterator: values(e), resultName: r, nextLoc: n }, "next" === this.method && (this.arg = t), y; } }, e; }
function _toConsumableArray(r) { return _arrayWithoutHoles(r) || _iterableToArray(r) || _unsupportedIterableToArray(r) || _nonIterableSpread(); }
function _nonIterableSpread() { throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); }
function _iterableToArray(r) { if ("undefined" != typeof Symbol && null != r[Symbol.iterator] || null != r["@@iterator"]) return Array.from(r); }
function _arrayWithoutHoles(r) { if (Array.isArray(r)) return _arrayLikeToArray(r); }
function _createForOfIteratorHelper(r, e) { var t = "undefined" != typeof Symbol && r[Symbol.iterator] || r["@@iterator"]; if (!t) { if (Array.isArray(r) || (t = _unsupportedIterableToArray(r)) || e && r && "number" == typeof r.length) { t && (r = t); var _n = 0, F = function F() {}; return { s: F, n: function n() { return _n >= r.length ? { done: !0 } : { done: !1, value: r[_n++] }; }, e: function e(r) { throw r; }, f: F }; } throw new TypeError("Invalid attempt to iterate non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method."); } var o, a = !0, u = !1; return { s: function s() { t = t.call(r); }, n: function n() { var r = t.next(); return a = r.done, r; }, e: function e(r) { u = !0, o = r; }, f: function f() { try { a || null == t["return"] || t["return"](); } finally { if (u) throw o; } } }; }
function _unsupportedIterableToArray(r, a) { if (r) { if ("string" == typeof r) return _arrayLikeToArray(r, a); var t = {}.toString.call(r).slice(8, -1); return "Object" === t && r.constructor && (t = r.constructor.name), "Map" === t || "Set" === t ? Array.from(r) : "Arguments" === t || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(t) ? _arrayLikeToArray(r, a) : void 0; } }
function _arrayLikeToArray(r, a) { (null == a || a > r.length) && (a = r.length); for (var e = 0, n = Array(a); e < a; e++) n[e] = r[e]; return n; }
function asyncGeneratorStep(n, t, e, r, o, a, c) { try { var i = n[a](c), u = i.value; } catch (n) { return void e(n); } i.done ? t(u) : Promise.resolve(u).then(r, o); }
function _asyncToGenerator(n) { return function () { var t = this, e = arguments; return new Promise(function (r, o) { var a = n.apply(t, e); function _next(n) { asyncGeneratorStep(a, r, o, _next, _throw, "next", n); } function _throw(n) { asyncGeneratorStep(a, r, o, _next, _throw, "throw", n); } _next(void 0); }); }; }
function _typeof(o) { "@babel/helpers - typeof"; return _typeof = "function" == typeof Symbol && "symbol" == typeof Symbol.iterator ? function (o) { return typeof o; } : function (o) { return o && "function" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? "symbol" : typeof o; }, _typeof(o); }
function parseArgs(argv) {
var optionsDefinitions = [_cli_helpers.cloneOption, _cli_helpers.commitOption, _cli_helpers.remoteOption, _cli_helpers.legislatureOption, _cli_helpers.categoriesOption, _cli_helpers.silentOption, _cli_helpers.verboseOption, _cli_helpers.dataDirDefaultOption, _cli_helpers.pullOption, _cli_helpers.fetchDocumentsOption, _cli_helpers.parseDocumentsOption, _cli_helpers.documentFormatOption, {
alias: "d",
help: "single dataset to reorganize",
name: "dataset",
type: String
}, {
help: "do not reset --hard after commit to save I/O if the working tree is to be discarded",
name: "no-reset-after-commit",
type: Boolean
}, {
alias: "V",
help: "don't validate data",
name: "no-validate",
type: Boolean
}];
var options = (0, _commandLineArgs["default"])(optionsDefinitions, {
argv: argv
});
return options;
}
function cloneGit(gitGroupUrl, datasetName, dataDir) {
git.clone(gitGroupUrl, "".concat(datasetName, "_nettoye"), dataDir);
}
function commitGit(datasetCleanDir, options, exitCode, subdirectories) {
if (options.commit) {
var errorCode = git.commitAndPush(datasetCleanDir, "Nouvelle moisson", options.remote);
if (exitCode === 10 && errorCode !== 10 || exitCode === 0 && errorCode !== 0 && errorCode !== 10) {
exitCode = errorCode;
}
}
if (commitAndPushUid(datasetCleanDir, options, subdirectories) && (exitCode === 0 || exitCode === 10)) {
exitCode = 1;
}
return exitCode;
}
function cleanAndValidateData(data, cleaner, fromJson, toJson, noValidate) {
cleaner(data);
if (_typeof(data) === "object" && Object.keys(data).length === 0) {
// By convention: an empty object must be deleted.
return null;
}
var jsonData = JSON.stringify(data, null, 2);
if (!noValidate) {
return jsonData;
} else {
var validatedData = fromJson(jsonData);
return toJson(validatedData);
}
}
function processFiles(_x, _x2, _x3, _x4, _x5, _x6, _x7) {
return _processFiles.apply(this, arguments);
}
function _processFiles() {
_processFiles = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee(datasetRawDir, datasetCleanDir, walkDirParams, cleaner, dataConverter, options, postProcessor) {
var _iterator3, _step3, filePathArray, filename, rawFilePath, rawJson, data, validateData, cleanJson, cleanFilePath;
return _regeneratorRuntime().wrap(function _callee$(_context) {
while (1) switch (_context.prev = _context.next) {
case 0:
_iterator3 = _createForOfIteratorHelper((0, _file_systems.walkDir)(datasetRawDir, walkDirParams));
_context.prev = 1;
_iterator3.s();
case 3:
if ((_step3 = _iterator3.n()).done) {
_context.next = 23;
break;
}
filePathArray = _step3.value;
filename = filePathArray[filePathArray.length - 1];
if (filename.endsWith(".json")) {
_context.next = 8;
break;
}
return _context.abrupt("continue", 21);
case 8:
rawFilePath = _path["default"].join.apply(_path["default"], [datasetRawDir].concat(_toConsumableArray(filePathArray)));
rawJson = _fsExtra["default"].readFileSync(rawFilePath, {
encoding: "utf8"
});
if (options.verbose) console.log("Cleaning file: ".concat(rawFilePath, "..."));
data = JSON.parse(rawJson);
validateData = !options["no-validate"];
cleanJson = cleanAndValidateData(data, cleaner, dataConverter.fromJson, dataConverter.toJson, validateData);
if (!(cleanJson !== null)) {
_context.next = 21;
break;
}
cleanFilePath = _path["default"].join.apply(_path["default"], [datasetCleanDir].concat(_toConsumableArray(filePathArray)));
_fsExtra["default"].ensureDirSync(_path["default"].dirname(cleanFilePath));
_fsExtra["default"].writeFileSync(cleanFilePath, cleanJson, {
encoding: "utf8"
});
if (!postProcessor) {
_context.next = 21;
break;
}
_context.next = 21;
return postProcessor(data, datasetCleanDir, options);
case 21:
_context.next = 3;
break;
case 23:
_context.next = 28;
break;
case 25:
_context.prev = 25;
_context.t0 = _context["catch"](1);
_iterator3.e(_context.t0);
case 28:
_context.prev = 28;
_iterator3.f();
return _context.finish(28);
case 31:
case "end":
return _context.stop();
}
}, _callee, null, [[1, 25, 28, 31]]);
}));
return _processFiles.apply(this, arguments);
}
function cleanReorganizedData(_x8) {
return _cleanReorganizedData.apply(this, arguments);
}
function _cleanReorganizedData() {
_cleanReorganizedData = _asyncToGenerator(/*#__PURE__*/_regeneratorRuntime().mark(function _callee2(options) {
var _options$clone;
var dataDir, exitCode, gitGroupUrl, enabledDatasets, chosenDatasets, _iterator4, _step4, dataset, datasetName, datasetRawDir, datasetCleanDir, _iterator5, _step5, _dataset, _datasetName, _datasetRawDir, _datasetCleanDir, _iterator6, _step6, _dataset2, _datasetName2, _datasetRawDir2, _datasetCleanDir2, _iterator7, _step7, _dataset3, _datasetName3, _datasetRawDir3, _datasetCleanDir3, _iterator8, _step8, _dataset4, _datasetName4, _datasetRawDir4, _datasetCleanDir4, _iterator9, _step9, _dataset5, _datasetName5, _datasetRawDir5, _datasetCleanDir5, _iterator10, _step10, _dataset6, _datasetName6, _datasetRawDir6, _datasetCleanDir6;
return _regeneratorRuntime().wrap(function _callee2$(_context2) {
while (1) switch (_context2.prev = _context2.next) {
case 0:
dataDir = options.dataDir;
Date.prototype.toJSON = _dates.patchedDateToJson;
exitCode = options.commit ? 10 : 0;
gitGroupUrl = (_options$clone = options.clone) === null || _options$clone === void 0 ? void 0 : _options$clone.trim().replace(/\/+$/, "");
enabledDatasets = (0, _datasets.getEnabledDatasets)(options.categories);
chosenDatasets = (0, _datasets.filterByLegislature)(_datasets.datasets, options.legislature);
if (!(enabledDatasets & _datasets.EnabledDatasets.ActeursEtOrganes)) {
_context2.next = 35;
break;
}
_iterator4 = _createForOfIteratorHelper(chosenDatasets.acteursEtOrganes);
_context2.prev = 8;
_iterator4.s();
case 10:
if ((_step4 = _iterator4.n()).done) {
_context2.next = 27;
break;
}
dataset = _step4.value;
datasetName = _path["default"].basename(dataset.filename, ".json");
if (!(options.dataset && datasetName !== options.dataset)) {
_context2.next = 15;
break;
}
return _context2.abrupt("continue", 25);
case 15:
cloneGit(gitGroupUrl, datasetName, dataDir);
datasetRawDir = _path["default"].join(dataDir, datasetName);
datasetCleanDir = "".concat(datasetRawDir, "_nettoye");
if (options.pull) {
git.resetAndPull(datasetCleanDir);
}
analyzeAndRemove(datasetName, options);
_context2.next = 22;
return processFiles(datasetRawDir, datasetCleanDir, ["acteurs"], _cleaners.cleanActeur, {
fromJson: _acteurs_et_organes.Convert.toActeur,
toJson: _acteurs_et_organes.Convert.acteurToJson
}, options);
case 22:
_context2.next = 24;
return processFiles(datasetRawDir, datasetCleanDir, ["organes"], _cleaners.cleanOrgane, {
fromJson: _acteurs_et_organes.Convert.toOrgane,
toJson: _acteurs_et_organes.Convert.organeToJson
}, options);
case 24:
exitCode = commitGit(datasetCleanDir, options, exitCode, ["acteurs", "organes"]);
case 25:
_context2.next = 10;
break;
case 27:
_context2.next = 32;
break;
case 29:
_context2.prev = 29;
_context2.t0 = _context2["catch"](8);
_iterator4.e(_context2.t0);
case 32:
_context2.prev = 32;
_iterator4.f();
return _context2.finish(32);
case 35:
if (!(enabledDatasets & _datasets.EnabledDatasets.Agendas)) {
_context2.next = 61;
break;
}
_iterator5 = _createForOfIteratorHelper(chosenDatasets.agendas);
_context2.prev = 37;
_iterator5.s();
case 39:
if ((_step5 = _iterator5.n()).done) {
_context2.next = 53;
break;
}
_dataset = _step5.value;
_datasetName = _path["default"].basename(_dataset.filename, ".json");
if (!(options.dataset && _datasetName !== options.dataset)) {
_context2.next = 44;
break;
}
return _context2.abrupt("continue", 51);
case 44:
cloneGit(gitGroupUrl, _datasetName, dataDir);
analyzeAndRemove(_datasetName, options);
_datasetRawDir = _path["default"].join(dataDir, _datasetName);
_datasetCleanDir = "".concat(_datasetRawDir, "_nettoye");
_context2.next = 50;
return processFiles(_datasetRawDir, _datasetCleanDir, [], _cleaners.cleanReunion, {
fromJson: _agendas.Convert.toReunion,
toJson: _agendas.Convert.reunionToJson
}, options);
case 50:
exitCode = commitGit(_datasetCleanDir, options, exitCode, ["."]);
case 51:
_context2.next = 39;
break;
case 53:
_context2.next = 58;
break;
case 55:
_context2.prev = 55;
_context2.t1 = _context2["catch"](37);
_iterator5.e(_context2.t1);
case 58:
_context2.prev = 58;
_iterator5.f();
return _context2.finish(58);
case 61:
if (!(enabledDatasets & _datasets.EnabledDatasets.Amendements)) {
_context2.next = 87;
break;
}
_iterator6 = _createForOfIteratorHelper(chosenDatasets.amendements);
_context2.prev = 63;
_iterator6.s();
case 65:
if ((_step6 = _iterator6.n()).done) {
_context2.next = 79;
break;
}
_dataset2 = _step6.value;
_datasetName2 = _path["default"].basename(_dataset2.filename, ".json");
if (!(options.dataset && _datasetName2 !== options.dataset)) {
_context2.next = 70;
break;
}
return _context2.abrupt("continue", 77);
case 70:
cloneGit(gitGroupUrl, _datasetName2, dataDir);
analyzeAndRemove(_datasetName2, options);
_datasetRawDir2 = _path["default"].join(dataDir, _datasetName2);
_datasetCleanDir2 = "".concat(_datasetRawDir2, "_nettoye");
_context2.next = 76;
return processFiles(_datasetRawDir2, _datasetCleanDir2, [], _cleaners.cleanAmendement, {
fromJson: _amendements.Convert.toAmendement,
toJson: _amendements.Convert.amendementToJson
}, options);
case 76:
exitCode = commitGit(_datasetCleanDir2, options, exitCode, ["."]);
case 77:
_context2.next = 65;
break;
case 79:
_context2.next = 84;
break;
case 81:
_context2.prev = 81;
_context2.t2 = _context2["catch"](63);
_iterator6.e(_context2.t2);
case 84:
_context2.prev = 84;
_iterator6.f();
return _context2.finish(84);
case 87:
if (!(enabledDatasets & _datasets.EnabledDatasets.DossiersLegislatifs)) {
_context2.next = 115;
break;
}
_iterator7 = _createForOfIteratorHelper(chosenDatasets.dossiersLegislatifs);
_context2.prev = 89;
_iterator7.s();
case 91:
if ((_step7 = _iterator7.n()).done) {
_context2.next = 107;
break;
}
_dataset3 = _step7.value;
_datasetName3 = _path["default"].basename(_dataset3.filename, ".json");
if (!(options.dataset && _datasetName3 !== options.dataset)) {
_context2.next = 96;
break;
}
return _context2.abrupt("continue", 105);
case 96:
cloneGit(gitGroupUrl, _datasetName3, dataDir);
analyzeAndRemove(_datasetName3, options);
_datasetRawDir3 = _path["default"].join(dataDir, _datasetName3);
_datasetCleanDir3 = "".concat(_datasetRawDir3, "_nettoye");
_context2.next = 102;
return processFiles(_datasetRawDir3, _datasetCleanDir3, ["documents"], _cleaners.cleanDocumentOrDivision, {
fromJson: _dossiers_legislatifs.Convert.toDocument,
toJson: _dossiers_legislatifs.Convert.documentToJson
}, options, _retrieve_documents.downloadAndParse);
case 102:
_context2.next = 104;
return processFiles(_datasetRawDir3, _datasetCleanDir3, ["dossiers"], _cleaners.cleanDossierParlementaire, {
fromJson: _dossiers_legislatifs.Convert.toDossierParlementaire,
toJson: _dossiers_legislatifs.Convert.dossierParlementaireToJson
}, options);
case 104:
exitCode = commitGit(_datasetCleanDir3, options, exitCode, ["documents", "dossiers"]);
case 105:
_context2.next = 91;
break;
case 107:
_context2.next = 112;
break;
case 109:
_context2.prev = 109;
_context2.t3 = _context2["catch"](89);
_iterator7.e(_context2.t3);
case 112:
_context2.prev = 112;
_iterator7.f();
return _context2.finish(112);
case 115:
if (!(enabledDatasets & _datasets.EnabledDatasets.Scrutins)) {
_context2.next = 141;
break;
}
_iterator8 = _createForOfIteratorHelper(chosenDatasets.scrutins);
_context2.prev = 117;
_iterator8.s();
case 119:
if ((_step8 = _iterator8.n()).done) {
_context2.next = 133;
break;
}
_dataset4 = _step8.value;
_datasetName4 = _path["default"].basename(_dataset4.filename, ".json");
if (!(options.dataset && _datasetName4 !== options.dataset)) {
_context2.next = 124;
break;
}
return _context2.abrupt("continue", 131);
case 124:
cloneGit(gitGroupUrl, _datasetName4, dataDir);
analyzeAndRemove(_datasetName4, options);
_datasetRawDir4 = _path["default"].join(dataDir, _datasetName4);
_datasetCleanDir4 = "".concat(_datasetRawDir4, "_nettoye");
_context2.next = 130;
return processFiles(_datasetRawDir4, _datasetCleanDir4, [], _cleaners.cleanScrutin, {
fromJson: _scrutins.Convert.toScrutin,
toJson: _scrutins.Convert.scrutinToJson
}, options);
case 130:
exitCode = commitGit(_datasetCleanDir4, options, exitCode, ["."]);
case 131:
_context2.next = 119;
break;
case 133:
_context2.next = 138;
break;
case 135:
_context2.prev = 135;
_context2.t4 = _context2["catch"](117);
_iterator8.e(_context2.t4);
case 138:
_context2.prev = 138;
_iterator8.f();
return _context2.finish(138);
case 141:
if (!(enabledDatasets & _datasets.EnabledDatasets.Questions)) {
_context2.next = 167;
break;
}
_iterator9 = _createForOfIteratorHelper(_datasets.datasets.questions);
_context2.prev = 143;
_iterator9.s();
case 145:
if ((_step9 = _iterator9.n()).done) {
_context2.next = 159;
break;
}
_dataset5 = _step9.value;
_datasetName5 = _path["default"].basename(_dataset5.filename, ".json");
if (!(options.dataset && _datasetName5 !== options.dataset)) {
_context2.next = 150;
break;
}
return _context2.abrupt("continue", 157);
case 150:
cloneGit(gitGroupUrl, _datasetName5, dataDir);
analyzeAndRemove(_datasetName5, options);
_datasetRawDir5 = _path["default"].join(dataDir, _datasetName5);
_datasetCleanDir5 = "".concat(_datasetRawDir5, "_nettoye");
_context2.next = 156;
return processFiles(_datasetRawDir5, _datasetCleanDir5, [], _cleaners.cleanQuestion, {
fromJson: _questions.Convert.toQuestion,
toJson: _questions.Convert.questionToJson
}, options);
case 156:
exitCode = commitGit(_datasetCleanDir5, options, exitCode, ["."]);
case 157:
_context2.next = 145;
break;
case 159:
_context2.next = 164;
break;
case 161:
_context2.prev = 161;
_context2.t5 = _context2["catch"](143);
_iterator9.e(_context2.t5);
case 164:
_context2.prev = 164;
_iterator9.f();
return _context2.finish(164);
case 167:
if (!(enabledDatasets & _datasets.EnabledDatasets.ComptesRendusSeances)) {
_context2.next = 193;
break;
}
_iterator10 = _createForOfIteratorHelper(_datasets.datasets.comptesRendusSeances);
_context2.prev = 169;
_iterator10.s();
case 171:
if ((_step10 = _iterator10.n()).done) {
_context2.next = 185;
break;
}
_dataset6 = _step10.value;
_datasetName6 = _path["default"].basename(_dataset6.filename, ".xml");
if (!(options.dataset && _datasetName6 !== options.dataset)) {
_context2.next = 176;
break;
}
return _context2.abrupt("continue", 183);
case 176:
cloneGit(gitGroupUrl, _datasetName6, dataDir);
analyzeAndRemove(_datasetName6, options);
_datasetRawDir6 = _path["default"].join(dataDir, _datasetName6);
_datasetCleanDir6 = "".concat(_datasetRawDir6, "_nettoye");
_context2.next = 182;
return processFiles(_datasetRawDir6, _datasetCleanDir6, [], _cleaners.cleanCompteRendu, {
fromJson: _debats.Convert.toCompteRendu,
toJson: _debats.Convert.compteRenduToJson
}, options);
case 182:
exitCode = commitGit(_datasetCleanDir6, options, exitCode, ["."]);
case 183:
_context2.next = 171;
break;
case 185:
_context2.next = 190;
break;
case 187:
_context2.prev = 187;
_context2.t6 = _context2["catch"](169);
_iterator10.e(_context2.t6);
case 190:
_context2.prev = 190;
_iterator10.f();
return _context2.finish(190);
case 193:
// Restore standard conversion of dates to JSON.
Date.prototype.toJSON = _dates.existingDateToJson;
if (!options.silent) {
console.log("Finished !");
}
return _context2.abrupt("return", exitCode);
case 196:
case "end":
return _context2.stop();
}
}, _callee2, null, [[8, 29, 32, 35], [37, 55, 58, 61], [63, 81, 84, 87], [89, 109, 112, 115], [117, 135, 138, 141], [143, 161, 164, 167], [169, 187, 190, 193]]);
}));
return _cleanReorganizedData.apply(this, arguments);
}
function analyzeAndRemove(datasetName, options) {
var dataDir = options.dataDir;
var datasetRawDir = _path["default"].join(dataDir, datasetName);
var datasetCleanDir = "".concat(datasetRawDir, "_nettoye");
if (!options.silent) {
console.log("Cleaning directory: ".concat(datasetCleanDir, "\u2026"));
}
_fsExtra["default"].ensureDirSync(datasetCleanDir);
var _iterator = _createForOfIteratorHelper(_fsExtra["default"].readdirSync(datasetCleanDir)),
_step;
try {
for (_iterator.s(); !(_step = _iterator.n()).done;) {
var filename = _step.value;
if (filename[0] === ".") {
continue;
}
var pathname = _path["default"].join(datasetCleanDir, filename);
_fsExtra["default"].removeSync(pathname);
}
} catch (err) {
_iterator.e(err);
} finally {
_iterator.f();
}
}
function commitAndPushUid(repositoryDir, options, subdirectories) {
var error = false;
if (options.commit) {
git.uidBranchUpdate(repositoryDir, subdirectories, options["no-reset-after-commit"]);
var _iterator2 = _createForOfIteratorHelper(options.remote || []),
_step2;
try {
for (_iterator2.s(); !(_step2 = _iterator2.n()).done;) {
var remote = _step2.value;
try {
git.run(repositoryDir, "push --quiet ".concat(remote, " uid:uid"));
} catch (childProcess) {
error = true;
}
}
} catch (err) {
_iterator2.e(err);
} finally {
_iterator2.f();
}
}
return error;
}
function main(argv) {
var options = parseArgs(argv);
return cleanReorganizedData(options);
}
/* istanbul ignore if */
if (process.argv[1].endsWith("clean_reorganized_data.ts")) {
main(process.argv).then(process.exit);
}
//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJuYW1lcyI6WyJfY29tbWFuZExpbmVBcmdzIiwiX2ludGVyb3BSZXF1aXJlRGVmYXVsdCIsInJlcXVpcmUiLCJfZnNFeHRyYSIsIl9wYXRoIiwiX2NsZWFuZXJzIiwiX2RhdGFzZXRzIiwiX3JldHJpZXZlX2RvY3VtZW50cyIsIl9kYXRlcyIsIl9maWxlX3N5c3RlbXMiLCJnaXQiLCJfaW50ZXJvcFJlcXVpcmVXaWxkY2FyZCIsIl9hY3RldXJzX2V0X29yZ2FuZXMiLCJfYWdlbmRhcyIsIl9hbWVuZGVtZW50cyIsIl9kb3NzaWVyc19sZWdpc2xhdGlmcyIsIl9zY3J1dGlucyIsIl9xdWVzdGlvbnMiLCJfZGViYXRzIiwiX2NsaV9oZWxwZXJzIiwiX2dldFJlcXVpcmVXaWxkY2FyZENhY2hlIiwiZSIsIldlYWtNYXAiLCJyIiwidCIsIl9fZXNNb2R1bGUiLCJfdHlwZW9mIiwiaGFzIiwiZ2V0IiwibiIsIl9fcHJvdG9fXyIsImEiLCJPYmplY3QiLCJkZWZpbmVQcm9wZXJ0eSIsImdldE93blByb3BlcnR5RGVzY3JpcHRvciIsInUiLCJoYXNPd25Qcm9wZXJ0eSIsImNhbGwiLCJpIiwic2V0IiwiX3JlZ2VuZXJhdG9yUnVudGltZSIsInByb3RvdHlwZSIsIm8iLCJ2YWx1ZSIsIlN5bWJvbCIsIml0ZXJhdG9yIiwiYyIsImFzeW5jSXRlcmF0b3IiLCJ0b1N0cmluZ1RhZyIsImRlZmluZSIsImVudW1lcmFibGUiLCJjb25maWd1cmFibGUiLCJ3cml0YWJsZSIsIndyYXAiLCJHZW5lcmF0b3IiLCJjcmVhdGUiLCJDb250ZXh0IiwibWFrZUludm9rZU1ldGhvZCIsInRyeUNhdGNoIiwidHlwZSIsImFyZyIsImgiLCJsIiwiZiIsInMiLCJ5IiwiR2VuZXJhdG9yRnVuY3Rpb24iLCJHZW5lcmF0b3JGdW5jdGlvblByb3RvdHlwZSIsInAiLCJkIiwiZ2V0UHJvdG90eXBlT2YiLCJ2IiwidmFsdWVzIiwiZyIsImRlZmluZUl0ZXJhdG9yTWV0aG9kcyIsImZvckVhY2giLCJfaW52b2tlIiwiQXN5bmNJdGVyYXRvciIsImludm9rZSIsInJlc29sdmUiLCJfX2F3YWl0IiwidGhlbiIsImNhbGxJbnZva2VXaXRoTWV0aG9kQW5kQXJnIiwiRXJyb3IiLCJkb25lIiwibWV0aG9kIiwiZGVsZWdhdGUiLCJtYXliZUludm9rZURlbGVnYXRlIiwic2VudCIsIl9zZW50IiwiZGlzcGF0Y2hFeGNlcHRpb24iLCJhYnJ1cHQiLCJUeXBlRXJyb3IiLCJyZXN1bHROYW1lIiwibmV4dCIsIm5leHRMb2MiLCJwdXNoVHJ5RW50cnkiLCJ0cnlMb2MiLCJjYXRjaExvYyIsImZpbmFsbHlMb2MiLCJhZnRlckxvYyIsInRyeUVudHJpZXMiLCJwdXNoIiwicmVzZXRUcnlFbnRyeSIsImNvbXBsZXRpb24iLCJyZXNldCIsImlzTmFOIiwibGVuZ3RoIiwiZGlzcGxheU5hbWUiLCJpc0dlbmVyYXRvckZ1bmN0aW9uIiwiY29uc3RydWN0b3IiLCJuYW1lIiwibWFyayIsInNldFByb3RvdHlwZU9mIiwiYXdyYXAiLCJhc3luYyIsIlByb21pc2UiLCJrZXlzIiwicmV2ZXJzZSIsInBvcCIsInByZXYiLCJjaGFyQXQiLCJzbGljZSIsInN0b3AiLCJydmFsIiwiaGFuZGxlIiwiY29tcGxldGUiLCJmaW5pc2giLCJfY2F0Y2giLCJkZWxlZ2F0ZVlpZWxkIiwiX3RvQ29uc3VtYWJsZUFycmF5IiwiX2FycmF5V2l0aG91dEhvbGVzIiwiX2l0ZXJhYmxlVG9BcnJheSIsIl91bnN1cHBvcnRlZEl0ZXJhYmxlVG9BcnJheSIsIl9ub25JdGVyYWJsZVNwcmVhZCIsIkFycmF5IiwiZnJvbSIsImlzQXJyYXkiLCJfYXJyYXlMaWtlVG9BcnJheSIsIl9jcmVhdGVGb3JPZkl0ZXJhdG9ySGVscGVyIiwiX24iLCJGIiwidG9TdHJpbmciLCJ0ZXN0IiwiYXN5bmNHZW5lcmF0b3JTdGVwIiwiX2FzeW5jVG9HZW5lcmF0b3IiLCJhcmd1bWVudHMiLCJhcHBseSIsIl9uZXh0IiwiX3Rocm93IiwicGFyc2VBcmdzIiwiYXJndiIsIm9wdGlvbnNEZWZpbml0aW9ucyIsImNsb25lT3B0aW9uIiwiY29tbWl0T3B0aW9uIiwicmVtb3RlT3B0aW9uIiwibGVnaXNsYXR1cmVPcHRpb24iLCJjYXRlZ29yaWVzT3B0aW9uIiwic2lsZW50T3B0aW9uIiwidmVyYm9zZU9wdGlvbiIsImRhdGFEaXJEZWZhdWx0T3B0aW9uIiwicHVsbE9wdGlvbiIsImZldGNoRG9jdW1lbnRzT3B0aW9uIiwicGFyc2VEb2N1bWVudHNPcHRpb24iLCJkb2N1bWVudEZvcm1hdE9wdGlvbiIsImFsaWFzIiwiaGVscCIsIlN0cmluZyIsIkJvb2xlYW4iLCJvcHRpb25zIiwiY29tbWFuZExpbmVBcmdzIiwiY2xvbmVHaXQiLCJnaXRHcm91cFVybCIsImRhdGFzZXROYW1lIiwiZGF0YURpciIsImNsb25lIiwiY29uY2F0IiwiY29tbWl0R2l0IiwiZGF0YXNldENsZWFuRGlyIiwiZXhpdENvZGUiLCJzdWJkaXJlY3RvcmllcyIsImNvbW1pdCIsImVycm9yQ29kZSIsImNvbW1pdEFuZFB1c2giLCJyZW1vdGUiLCJjb21taXRBbmRQdXNoVWlkIiwiY2xlYW5BbmRWYWxpZGF0ZURhdGEiLCJkYXRhIiwiY2xlYW5lciIsImZyb21Kc29uIiwidG9Kc29uIiwibm9WYWxpZGF0ZSIsImpzb25EYXRhIiwiSlNPTiIsInN0cmluZ2lmeSIsInZhbGlkYXRlZERhdGEiLCJwcm9jZXNzRmlsZXMiLCJfeCIsIl94MiIsIl94MyIsIl94NCIsIl94NSIsIl94NiIsIl94NyIsIl9wcm9jZXNzRmlsZXMiLCJfY2FsbGVlIiwiZGF0YXNldFJhd0RpciIsIndhbGtEaXJQYXJhbXMiLCJkYXRhQ29udmVydGVyIiwicG9zdFByb2Nlc3NvciIsIl9pdGVyYXRvcjMiLCJfc3RlcDMiLCJmaWxlUGF0aEFycmF5IiwiZmlsZW5hbWUiLCJyYXdGaWxlUGF0aCIsInJhd0pzb24iLCJ2YWxpZGF0ZURhdGEiLCJjbGVhbkpzb24iLCJjbGVhbkZpbGVQYXRoIiwiX2NhbGxlZSQiLCJfY29udGV4dCIsIndhbGtEaXIiLCJlbmRzV2l0aCIsInBhdGgiLCJqb2luIiwiZnMiLCJyZWFkRmlsZVN5bmMiLCJlbmNvZGluZyIsInZlcmJvc2UiLCJjb25zb2xlIiwibG9nIiwicGFyc2UiLCJlbnN1cmVEaXJTeW5jIiwiZGlybmFtZSIsIndyaXRlRmlsZVN5bmMiLCJ0MCIsImNsZWFuUmVvcmdhbml6ZWREYXRhIiwiX3g4IiwiX2NsZWFuUmVvcmdhbml6ZWREYXRhIiwiX2NhbGxlZTIiLCJfb3B0aW9ucyRjbG9uZSIsImVuYWJsZWREYXRhc2V0cyIsImNob3NlbkRhdGFzZXRzIiwiX2l0ZXJhdG9yNCIsIl9zdGVwNCIsImRhdGFzZXQiLCJfaXRlcmF0b3I1IiwiX3N0ZXA1IiwiX2RhdGFzZXQiLCJfZGF0YXNldE5hbWUiLCJfZGF0YXNldFJhd0RpciIsIl9kYXRhc2V0Q2xlYW5EaXIiLCJfaXRlcmF0b3I2IiwiX3N0ZXA2IiwiX2RhdGFzZXQyIiwiX2RhdGFzZXROYW1lMiIsIl9kYXRhc2V0UmF3RGlyMiIsIl9kYXRhc2V0Q2xlYW5EaXIyIiwiX2l0ZXJhdG9yNyIsIl9zdGVwNyIsIl9kYXRhc2V0MyIsIl9kYXRhc2V0TmFtZTMiLCJfZGF0YXNldFJhd0RpcjMiLCJfZGF0YXNldENsZWFuRGlyMyIsIl9pdGVyYXRvcjgiLCJfc3RlcDgiLCJfZGF0YXNldDQiLCJfZGF0YXNldE5hbWU0IiwiX2RhdGFzZXRSYXdEaXI0IiwiX2RhdGFzZXRDbGVhbkRpcjQiLCJfaXRlcmF0b3I5IiwiX3N0ZXA5IiwiX2RhdGFzZXQ1IiwiX2RhdGFzZXROYW1lNSIsIl9kYXRhc2V0UmF3RGlyNSIsIl9kYXRhc2V0Q2xlYW5EaXI1IiwiX2l0ZXJhdG9yMTAiLCJfc3RlcDEwIiwiX2RhdGFzZXQ2IiwiX2RhdGFzZXROYW1lNiIsIl9kYXRhc2V0UmF3RGlyNiIsIl9kYXRhc2V0Q2xlYW5EaXI2IiwiX2NhbGxlZTIkIiwiX2NvbnRleHQyIiwiRGF0ZSIsInRvSlNPTiIsInBhdGNoZWREYXRlVG9Kc29uIiwidHJpbSIsInJlcGxhY2UiLCJnZXRFbmFibGVkRGF0YXNldHMiLCJjYXRlZ29yaWVzIiwiZmlsdGVyQnlMZWdpc2xhdHVyZSIsImRhdGFzZXRzIiwibGVnaXNsYXR1cmUiLCJFbmFibGVkRGF0YXNldHMiLCJBY3RldXJzRXRPcmdhbmVzIiwiYWN0ZXVyc0V0T3JnYW5lcyIsImJhc2VuYW1lIiwicHVsbCIsInJlc2V0QW5kUHVsbCIsImFuYWx5emVBbmRSZW1vdmUiLCJjbGVhbkFjdGV1ciIsIkFjdGV1cnNFdE9yZ2FuZXNDb252ZXJ0IiwidG9BY3RldXIiLCJhY3RldXJUb0pzb24iLCJjbGVhbk9yZ2FuZSIsInRvT3JnYW5lIiwib3JnYW5lVG9Kc29uIiwiQWdlbmRhcyIsImFnZW5kYXMiLCJjbGVhblJldW5pb24iLCJBZ2VuZGFzQ29udmVydCIsInRvUmV1bmlvbiIsInJldW5pb25Ub0pzb24iLCJ0MSIsIkFtZW5kZW1lbnRzIiwiYW1lbmRlbWVudHMiLCJjbGVhbkFtZW5kZW1lbnQiLCJBbWVuZGVtZW50c0NvbnZlcnQiLCJ0b0FtZW5kZW1lbnQiLCJhbWVuZGVtZW50VG9Kc29uIiwidDIiLCJEb3NzaWVyc0xlZ2lzbGF0aWZzIiwiZG9zc2llcnNMZWdpc2xhdGlmcyIsImNsZWFuRG9jdW1lbnRPckRpdmlzaW9uIiwiRG9zc2llcnNMZWdpc2xhdGlmc0NvbnZlcnQiLCJ0b0RvY3VtZW50IiwiZG9jdW1lbnRUb0pzb24iLCJkb3dubG9hZEFuZFBhcnNlIiwiY2xlYW5Eb3NzaWVyUGFybGVtZW50YWlyZSIsInRvRG9zc2llclBhcmxlbWVudGFpcmUiLCJkb3NzaWVyUGFybGVtZW50YWlyZVRvSnNvbiIsInQzIiwiU2NydXRpbnMiLCJzY3J1dGlucyIsImNsZWFuU2NydXRpbiIsIlNjcnV0aW5zQ29udmVydCIsInRvU2NydXRpbiIsInNjcnV0aW5Ub0pzb24iLCJ0NCIsIlF1ZXN0aW9ucyIsInF1ZXN0aW9ucyIsImNsZWFuUXVlc3Rpb24iLCJRdWVzdGlvbnNDb252ZXJ0IiwidG9RdWVzdGlvbiIsInF1ZXN0aW9uVG9Kc29uIiwidDUiLCJDb21wdGVzUmVuZHVzU2VhbmNlcyIsImNvbXB0ZXNSZW5kdXNTZWFuY2VzIiwiY2xlYW5Db21wdGVSZW5kdSIsIkNvbXB0ZXNSZW5kdXNDb252ZXJ0IiwidG9Db21wdGVSZW5kdSIsImNvbXB0ZVJlbmR1VG9Kc29uIiwidDYiLCJleGlzdGluZ0RhdGVUb0pzb24iLCJzaWxlbnQiLCJfaXRlcmF0b3IiLCJyZWFkZGlyU3luYyIsIl9zdGVwIiwicGF0aG5hbWUiLCJyZW1vdmVTeW5jIiwiZXJyIiwicmVwb3NpdG9yeURpciIsImVycm9yIiwidWlkQnJhbmNoVXBkYXRlIiwiX2l0ZXJhdG9yMiIsIl9zdGVwMiIsInJ1biIsImNoaWxkUHJvY2VzcyIsIm1haW4iLCJwcm9jZXNzIiwiZXhpdCJdLCJzb3VyY2VzIjpbIi4uLy4uL3NyYy9zY3JpcHRzL2NsZWFuX3Jlb3JnYW5pemVkX2RhdGEudHMiXSwic291cmNlc0NvbnRlbnQiOlsiaW1wb3J0IGNvbW1hbmRMaW5lQXJncyBmcm9tIFwiY29tbWFuZC1saW5lLWFyZ3NcIlxuaW1wb3J0IGZzIGZyb20gXCJmcy1leHRyYVwiXG5pbXBvcnQgcGF0aCBmcm9tIFwicGF0aFwiXG5pbXBvcnQge1xuICBjbGVhbkFjdGV1cixcbiAgY2xlYW5BbWVuZGVtZW50LFxuICBjbGVhbkRvY3VtZW50T3JEaXZpc2lvbixcbiAgY2xlYW5Eb3NzaWVyUGFybGVtZW50YWlyZSxcbiAgY2xlYW5PcmdhbmUsXG4gIGNsZWFuUmV1bmlvbixcbiAgY2xlYW5TY3J1dGluLFxuICBjbGVhblF1ZXN0aW9uLFxuICBjbGVhbkNvbXB0ZVJlbmR1LFxufSBmcm9tIFwiLi4vY2xlYW5lcnNcIlxuaW1wb3J0IHtcbiAgZGF0YXNldHMsXG4gIEVuYWJsZWREYXRhc2V0cyxcbiAgZ2V0RW5hYmxlZERhdGFzZXRzLFxuICBmaWx0ZXJCeUxlZ2lzbGF0dXJlLFxufSBmcm9tIFwiLi4vZGF0YXNldHNcIlxuaW1wb3J0IHsgZG93bmxvYWRBbmRQYXJzZSB9IGZyb20gXCIuL3JldHJpZXZlX2RvY3VtZW50c1wiXG5pbXBvcnQgeyBleGlzdGluZ0RhdGVUb0pzb24sIHBhdGNoZWREYXRlVG9Kc29uIH0gZnJvbSBcIi4uL2RhdGVzXCJcbmltcG9ydCB7IHdhbGtEaXIgfSBmcm9tIFwiLi4vZmlsZV9zeXN0ZW1zXCJcbmltcG9ydCAqIGFzIGdpdCBmcm9tIFwiLi4vZ2l0XCJcbmltcG9ydCB7XG4gIEFjdGV1cixcbiAgQ29udmVydCBhcyBBY3RldXJzRXRPcmdhbmVzQ29udmVydCxcbiAgT3JnYW5lLFxufSBmcm9tIFwiLi4vdHlwZXMvYWN0ZXVyc19ldF9vcmdhbmVzXCJcbmltcG9ydCB7IENvbnZlcnQgYXMgQWdlbmRhc0NvbnZlcnQsIFJldW5pb24gfSBmcm9tIFwiLi4vdHlwZXMvYWdlbmRhc1wiXG5pbXBvcnQgeyBBbWVuZGVtZW50LCBDb252ZXJ0IGFzIEFtZW5kZW1lbnRzQ29udmVydCB9IGZyb20gXCIuLi90eXBlcy9hbWVuZGVtZW50c1wiXG5pbXBvcnQge1xuICBDb252ZXJ0IGFzIERvc3NpZXJzTGVnaXNsYXRpZnNDb252ZXJ0LFxuICBEb2N1bWVudCxcbiAgRG9zc2llclBhcmxlbWVudGFpcmUsXG59IGZyb20gXCIuLi90eXBlcy9kb3NzaWVyc19sZWdpc2xhdGlmc1wiXG5pbXBvcnQgeyBDb252ZXJ0IGFzIFNjcnV0aW5zQ29udmVydCwgU2NydXRpbiB9IGZyb20gXCIuLi90eXBlcy9zY3J1dGluc1wiXG5pbXBvcnQgeyBDb252ZXJ0IGFzIFF1ZXN0aW9uc0NvbnZlcnQsIFF1ZXN0aW9uIH0gZnJvbSBcIi4uL3R5cGVzL3F1ZXN0aW9uc1wiXG5pbXBvcnQgeyBDb252ZXJ0IGFzIENvbXB0ZXNSZW5kdXNDb252ZXJ0LCBDb21wdGVSZW5kdSB9IGZyb20gXCIuLi90eXBlcy9kZWJhdHNcIlxuaW1wb3J0IHtcbiAgY2xvbmVPcHRpb24sXG4gIGNvbW1pdE9wdGlvbixcbiAgcmVtb3RlT3B0aW9uLFxuICBsZWdpc2xhdHVyZU9wdGlvbixcbiAgY2F0ZWdvcmllc09wdGlvbixcbiAgZGF0YURpckRlZmF1bHRPcHRpb24sXG4gIHNpbGVudE9wdGlvbixcbiAgdmVyYm9zZU9wdGlvbixcbiAgcHVsbE9wdGlvbixcbiAgZmV0Y2hEb2N1bWVudHNPcHRpb24sXG4gIHBhcnNlRG9jdW1lbnRzT3B0aW9uLFxuICBkb2N1bWVudEZvcm1hdE9wdGlvbixcbn0gZnJvbSBcIi4vc2hhcmVkL2NsaV9oZWxwZXJzXCJcblxuZnVuY3Rpb24gcGFyc2VBcmdzKGFyZ3Y6IHN0cmluZ1tdKTogYW55IHtcbiAgY29uc3Qgb3B0aW9uc0RlZmluaXRpb25zID0gW1xuICAgIGNsb25lT3B0aW9uLFxuICAgIGNvbW1pdE9wdGlvbixcbiAgICByZW1vdGVPcHRpb24sXG4gICAgbGVnaXNsYXR1cmVPcHRpb24sXG4gICAgY2F0ZWdvcmllc09wdGlvbixcbiAgICBzaWxlbnRPcHRpb24sXG4gICAgdmVyYm9zZU9wdGlvbixcbiAgICBkYXRhRGlyRGVmYXVsdE9wdGlvbixcbiAgICBwdWxsT3B0aW9uLFxuICAgIGZldGNoRG9jdW1lbnRzT3B0aW9uLFxuICAgIHBhcnNlRG9jdW1lbnRzT3B0aW9uLFxuICAgIGRvY3VtZW50Rm9ybWF0T3B0aW9uLFxuICAgIHtcbiAgICAgIGFsaWFzOiBcImRcIixcbiAgICAgIGhlbHA6IFwic2luZ2xlIGRhdGFzZXQgdG8gcmVvcmdhbml6ZVwiLFxuICAgICAgbmFtZTogXCJkYXRhc2V0XCIsXG4gICAgICB0eXBlOiBTdHJpbmcsXG4gICAgfSxcbiAgICB7XG4gICAgICBoZWxwOiBcImRvIG5vdCByZXNldCAtLWhhcmQgYWZ0ZXIgY29tbWl0IHRvIHNhdmUgSS9PIGlmIHRoZSB3b3JraW5nIHRyZWUgaXMgdG8gYmUgZGlzY2FyZGVkXCIsXG4gICAgICBuYW1lOiBcIm5vLXJlc2V0LWFmdGVyLWNvbW1pdFwiLFxuICAgICAgdHlwZTogQm9vbGVhbixcbiAgICB9LFxuICAgIHtcbiAgICAgIGFsaWFzOiBcIlZcIixcbiAgICAgIGhlbHA6IFwiZG9uJ3QgdmFsaWRhdGUgZGF0YVwiLFxuICAgICAgbmFtZTogXCJuby12YWxpZGF0ZVwiLFxuICAgICAgdHlwZTogQm9vbGVhbixcbiAgICB9LFxuICBdXG4gIGNvbnN0IG9wdGlvbnMgPSBjb21tYW5kTGluZUFyZ3Mob3B0aW9uc0RlZmluaXRpb25zLCB7XG4gICAgYXJndjogYXJndixcbiAgfSlcbiAgcmV0dXJuIG9wdGlvbnNcbn1cblxuZnVuY3Rpb24gY2xvbmVHaXQoXG4gIGdpdEdyb3VwVXJsOiBzdHJpbmcgfCB1bmRlZmluZWQsXG4gIGRhdGFzZXROYW1lOiBzdHJpbmcsXG4gIGRhdGFEaXI6IHN0cmluZyxcbik6IHZvaWQge1xuICBnaXQuY2xvbmUoZ2l0R3JvdXBVcmwsIGAke2RhdGFzZXROYW1lfV9uZXR0b3llYCwgZGF0YURpcilcbn1cblxuZnVuY3Rpb24gY29tbWl0R2l0KFxuICBkYXRhc2V0Q2xlYW5EaXI6IHN0cmluZyxcbiAgb3B0aW9uczogYW55LFxuICBleGl0Q29kZTogbnVtYmVyLFxuICBzdWJkaXJlY3RvcmllczogYW55LFxuKTogbnVtYmVyIHtcbiAgaWYgKG9wdGlvbnMuY29tbWl0KSB7XG4gICAgY29uc3QgZXJyb3JDb2RlID0gZ2l0LmNvbW1pdEFuZFB1c2goXG4gICAgICBkYXRhc2V0Q2xlYW5EaXIsXG4gICAgICBcIk5vdXZlbGxlIG1vaXNzb25cIixcbiAgICAgIG9wdGlvbnMucmVtb3RlLFxuICAgIClcbiAgICBpZiAoXG4gICAgICAoZXhpdENvZGUgPT09IDEwICYmIGVycm9yQ29kZSAhPT0gMTApIHx8XG4gICAgICAoZXhpdENvZGUgPT09IDAgJiYgZXJyb3JDb2RlICE9PSAwICYmIGVycm9yQ29kZSAhPT0gMTApXG4gICAgKSB7XG4gICAgICBleGl0Q29kZSA9IGVycm9yQ29kZVxuICAgIH1cbiAgfVxuICBpZiAoXG4gICAgY29tbWl0QW5kUHVzaFVpZChkYXRhc2V0Q2xlYW5EaXIsIG9wdGlvbnMsIHN1YmRpcmVjdG9yaWVzKSAmJlxuICAgIChleGl0Q29kZSA9PT0gMCB8fCBleGl0Q29kZSA9PT0gMTApXG4gICkge1xuICAgIGV4aXRDb2RlID0gMVxuICB9XG4gIHJldHVybiBleGl0Q29kZVxufVxuXG5mdW5jdGlvbiBjbGVhbkFuZFZhbGlkYXRlRGF0YTxUPihcbiAgZGF0YTogYW55LFxuICBjbGVhbmVyOiAoZGF0YTogVCkgPT4gdm9pZCxcbiAgZnJvbUpzb246IChqc29uOiBzdHJpbmcpID0+IFQsXG4gIHRvSnNvbjogKGRhdGE6IFQpID0+IHN0cmluZyxcbiAgbm9WYWxpZGF0ZTogQm9vbGVhbixcbik6IHN0cmluZyB8IG51bGwge1xuICBjbGVhbmVyKGRhdGEpXG4gIGlmICh0eXBlb2YgZGF0YSA9PT0gXCJvYmplY3RcIiAmJiBPYmplY3Qua2V5cyhkYXRhKS5sZW5ndGggPT09IDApIHtcbiAgICAvLyBCeSBjb252ZW50aW9uOiBhbiBlbXB0eSBvYmplY3QgbXVzdCBiZSBkZWxldGVkLlxuICAgIHJldHVybiBudWxsXG4gIH1cbiAgY29uc3QganNvbkRhdGEgPSBKU09OLnN0cmluZ2lmeShkYXRhLCBudWxsLCAyKVxuICBpZiAoIW5vVmFsaWRhdGUpIHtcbiAgICByZXR1cm4ganNvbkRhdGFcbiAgfSBlbHNlIHtcbiAgICBjb25zdCB2YWxpZGF0ZWREYXRhOiBUID0gZnJvbUpzb24oanNvbkRhdGEpXG4gICAgcmV0dXJuIHRvSnNvbih2YWxpZGF0ZWREYXRhKVxuICB9XG59XG5cbmFzeW5jIGZ1bmN0aW9uIHByb2Nlc3NGaWxlczxUPihcbiAgZGF0YXNldFJhd0Rpcjogc3RyaW5nLFxuICBkYXRhc2V0Q2xlYW5EaXI6IHN0cmluZyxcbiAgd2Fsa0RpclBhcmFtczogYW55W10sXG4gIGNsZWFuZXI6IGFueSxcbiAgZGF0YUNvbnZlcnRlcjogeyBmcm9tSnNvbjogYW55OyB0b0pzb246IGFueSB9LFxuICBvcHRpb25zOiBhbnksXG4gIHBvc3RQcm9jZXNzb3I/OiBhbnksXG4pOiBQcm9taXNlPHZvaWQ+IHtcbiAgZm9yIChjb25zdCBmaWxlUGF0aEFycmF5IG9mIHdhbGtEaXIoZGF0YXNldFJhd0Rpciwgd2Fsa0RpclBhcmFtcykpIHtcbiAgICBjb25zdCBmaWxlbmFtZSA9IGZpbGVQYXRoQXJyYXlbZmlsZVBhdGhBcnJheS5sZW5ndGggLSAxXVxuICAgIGlmICghZmlsZW5hbWUuZW5kc1dpdGgoXCIuanNvblwiKSkgY29udGludWVcbiAgICBjb25zdCByYXdGaWxlUGF0aCA9IHBhdGguam9pbihkYXRhc2V0UmF3RGlyLCAuLi5maWxlUGF0aEFycmF5KVxuICAgIGNvbnN0IHJhd0pzb24gPSBmcy5yZWFkRmlsZVN5bmMocmF3RmlsZVBhdGgsIHsgZW5jb2Rpbmc6IFwidXRmOFwiIH0pXG4gICAgaWYgKG9wdGlvbnMudmVyYm9zZSkgY29uc29sZS5sb2coYENsZWFuaW5nIGZpbGU6ICR7cmF3RmlsZVBhdGh9Li4uYClcbiAgICBjb25zdCBkYXRhID0gSlNPTi5wYXJzZShyYXdKc29uKVxuICAgIGNvbnN0IHZhbGlkYXRlRGF0YSA9ICFvcHRpb25zW1wibm8tdmFsaWRhdGVcIl1cbiAgICBjb25zdCBjbGVhbkpzb24gPSBjbGVhbkFuZFZhbGlkYXRlRGF0YTxUPihcbiAgICAgIGRhdGEsXG4gICAgICBjbGVhbmVyLFxuICAgICAgZGF0YUNvbnZlcnRlci5mcm9tSnNvbixcbiAgICAgIGRhdGFDb252ZXJ0ZXIudG9Kc29uLFxuICAgICAgdmFsaWRhdGVEYXRhLFxuICAgIClcblxuICAgIGlmIChjbGVhbkpzb24gIT09IG51bGwpIHtcbiAgICAgIGNvbnN0IGNsZWFuRmlsZVBhdGggPSBwYXRoLmpvaW4oZGF0YXNld