@tricoteuses/assemblee
Version:
Retrieve, clean up & handle French Assemblée nationale's open data
57 lines • 11.1 kB
JavaScript
import commandLineArgs from "command-line-args";
import deepEqual from "deep-equal";
import fs from "fs-extra";
import path from "path";
import * as git from "../git.mjs";
import { iterRechercheRawAmendements } from "../parsers/index.mjs";
import { cloneOption, commitOption, dataDirDefaultOption, legislatureOption, pullOption, remoteOption, silentOption, verboseOption } from "./shared/cli_helpers.mjs";
const optionsDefinitions = [cloneOption, commitOption, legislatureOption, remoteOption, silentOption, verboseOption, dataDirDefaultOption, pullOption, {
alias: "i",
help: "incremental mode: retrieve only amendments whose position in search has changed",
name: "incremental",
type: Boolean
}];
const options = commandLineArgs(optionsDefinitions);
async function main() {
const gitGroupUrl = options.clone ? options.clone.trim().replace(/\/+$/, "") : undefined;
git.clone(gitGroupUrl, "Amendements_en_attente", options.dataDir);
const amendementsDir = path.join(options.dataDir, "Amendements_en_attente");
if (options.pull) {
git.resetAndPull(amendementsDir);
}
await fs.ensureDir(amendementsDir);
const cacheDir = path.join(options.dataDir, "Amendements_en_attente_cache");
for (const etat of ["En recevabilité", "En traitement"]) {
const url = `https://www.assemblee-nationale.fr/dyn/${options.legislature}/amendements?etat=${encodeURIComponent(etat)}`;
for await (const [amendementUrlPath, amendement] of iterRechercheRawAmendements(cacheDir, url, options.incremental, null, null, {
silent: options.silent,
verbose: options.verbose
})) {
const amendementUrlPathSplitted = amendementUrlPath.split("/");
const amendementFilePath = path.join(amendementsDir, ...amendementUrlPathSplitted.slice(2)) + ".json";
await fs.ensureDir(path.dirname(amendementFilePath));
const existingAmendement = (await fs.pathExists(amendementFilePath)) ? await fs.readJson(amendementFilePath) : null;
if (!deepEqual(amendement, existingAmendement)) {
if (!options.silent) {
if (existingAmendement === null) {
console.log(`Adding amendement JSON: ${amendementFilePath}…`);
} else {
console.log(`Updating amendement JSON: ${amendementFilePath}…`);
}
}
await fs.writeJson(amendementFilePath, amendement, {
spaces: 2
});
}
}
}
if (options.commit) {
return git.commitAndPush(amendementsDir, "Nouvelle moisson", options.remote);
}
return 0;
}
main().then(exitCode => process.exit(exitCode)).catch(error => {
console.log(error);
process.exit(1);
});
//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJuYW1lcyI6WyJjb21tYW5kTGluZUFyZ3MiLCJkZWVwRXF1YWwiLCJmcyIsInBhdGgiLCJnaXQiLCJpdGVyUmVjaGVyY2hlUmF3QW1lbmRlbWVudHMiLCJjbG9uZU9wdGlvbiIsImNvbW1pdE9wdGlvbiIsImRhdGFEaXJEZWZhdWx0T3B0aW9uIiwibGVnaXNsYXR1cmVPcHRpb24iLCJwdWxsT3B0aW9uIiwicmVtb3RlT3B0aW9uIiwic2lsZW50T3B0aW9uIiwidmVyYm9zZU9wdGlvbiIsIm9wdGlvbnNEZWZpbml0aW9ucyIsImFsaWFzIiwiaGVscCIsIm5hbWUiLCJ0eXBlIiwiQm9vbGVhbiIsIm9wdGlvbnMiLCJtYWluIiwiZ2l0R3JvdXBVcmwiLCJjbG9uZSIsInRyaW0iLCJyZXBsYWNlIiwidW5kZWZpbmVkIiwiZGF0YURpciIsImFtZW5kZW1lbnRzRGlyIiwiam9pbiIsInB1bGwiLCJyZXNldEFuZFB1bGwiLCJlbnN1cmVEaXIiLCJjYWNoZURpciIsImV0YXQiLCJ1cmwiLCJsZWdpc2xhdHVyZSIsImVuY29kZVVSSUNvbXBvbmVudCIsImFtZW5kZW1lbnRVcmxQYXRoIiwiYW1lbmRlbWVudCIsImluY3JlbWVudGFsIiwic2lsZW50IiwidmVyYm9zZSIsImFtZW5kZW1lbnRVcmxQYXRoU3BsaXR0ZWQiLCJzcGxpdCIsImFtZW5kZW1lbnRGaWxlUGF0aCIsInNsaWNlIiwiZGlybmFtZSIsImV4aXN0aW5nQW1lbmRlbWVudCIsInBhdGhFeGlzdHMiLCJyZWFkSnNvbiIsImNvbnNvbGUiLCJsb2ciLCJ3cml0ZUpzb24iLCJzcGFjZXMiLCJjb21taXQiLCJjb21taXRBbmRQdXNoIiwicmVtb3RlIiwidGhlbiIsImV4aXRDb2RlIiwicHJvY2VzcyIsImV4aXQiLCJjYXRjaCIsImVycm9yIl0sInNvdXJjZXMiOlsiLi4vLi4vc3JjL3NjcmlwdHMvcmV0cmlldmVfcGVuZGluZ19hbWVuZG1lbnRzLnRzIl0sInNvdXJjZXNDb250ZW50IjpbImltcG9ydCBjb21tYW5kTGluZUFyZ3MgZnJvbSBcImNvbW1hbmQtbGluZS1hcmdzXCJcbmltcG9ydCBkZWVwRXF1YWwgZnJvbSBcImRlZXAtZXF1YWxcIlxuaW1wb3J0IGZzIGZyb20gXCJmcy1leHRyYVwiXG5pbXBvcnQgcGF0aCBmcm9tIFwicGF0aFwiXG5cbmltcG9ydCAqIGFzIGdpdCBmcm9tIFwiLi4vZ2l0XCJcbmltcG9ydCB7IGl0ZXJSZWNoZXJjaGVSYXdBbWVuZGVtZW50cyB9IGZyb20gXCIuLi9wYXJzZXJzXCJcbmltcG9ydCB7XG4gIGNsb25lT3B0aW9uLFxuICBjb21taXRPcHRpb24sXG4gIGRhdGFEaXJEZWZhdWx0T3B0aW9uLFxuICBsZWdpc2xhdHVyZU9wdGlvbixcbiAgcHVsbE9wdGlvbixcbiAgcmVtb3RlT3B0aW9uLFxuICBzaWxlbnRPcHRpb24sXG4gIHZlcmJvc2VPcHRpb24sXG59IGZyb20gXCIuL3NoYXJlZC9jbGlfaGVscGVyc1wiXG5cbmNvbnN0IG9wdGlvbnNEZWZpbml0aW9ucyA9IFtcbiAgY2xvbmVPcHRpb24sXG4gIGNvbW1pdE9wdGlvbixcbiAgbGVnaXNsYXR1cmVPcHRpb24sXG4gIHJlbW90ZU9wdGlvbixcbiAgc2lsZW50T3B0aW9uLFxuICB2ZXJib3NlT3B0aW9uLFxuICBkYXRhRGlyRGVmYXVsdE9wdGlvbixcbiAgcHVsbE9wdGlvbixcbiAge1xuICAgIGFsaWFzOiBcImlcIixcbiAgICBoZWxwOiBcImluY3JlbWVudGFsIG1vZGU6IHJldHJpZXZlIG9ubHkgYW1lbmRtZW50cyB3aG9zZSBwb3NpdGlvbiBpbiBzZWFyY2ggaGFzIGNoYW5nZWRcIixcbiAgICBuYW1lOiBcImluY3JlbWVudGFsXCIsXG4gICAgdHlwZTogQm9vbGVhbixcbiAgfSxcbl1cbmNvbnN0IG9wdGlvbnMgPSBjb21tYW5kTGluZUFyZ3Mob3B0aW9uc0RlZmluaXRpb25zKVxuXG5hc3luYyBmdW5jdGlvbiBtYWluKCk6IFByb21pc2U8bnVtYmVyPiB7XG4gIGNvbnN0IGdpdEdyb3VwVXJsID0gb3B0aW9ucy5jbG9uZVxuICAgID8gb3B0aW9ucy5jbG9uZS50cmltKCkucmVwbGFjZSgvXFwvKyQvLCBcIlwiKVxuICAgIDogdW5kZWZpbmVkXG5cbiAgZ2l0LmNsb25lKGdpdEdyb3VwVXJsLCBcIkFtZW5kZW1lbnRzX2VuX2F0dGVudGVcIiwgb3B0aW9ucy5kYXRhRGlyKVxuXG4gIGNvbnN0IGFtZW5kZW1lbnRzRGlyID0gcGF0aC5qb2luKG9wdGlvbnMuZGF0YURpciwgXCJBbWVuZGVtZW50c19lbl9hdHRlbnRlXCIpXG4gIGlmIChvcHRpb25zLnB1bGwpIHtcbiAgICBnaXQucmVzZXRBbmRQdWxsKGFtZW5kZW1lbnRzRGlyKVxuICB9XG5cbiAgYXdhaXQgZnMuZW5zdXJlRGlyKGFtZW5kZW1lbnRzRGlyKVxuICBjb25zdCBjYWNoZURpciA9IHBhdGguam9pbihvcHRpb25zLmRhdGFEaXIsIFwiQW1lbmRlbWVudHNfZW5fYXR0ZW50ZV9jYWNoZVwiKVxuICBmb3IgKGNvbnN0IGV0YXQgb2YgW1wiRW4gcmVjZXZhYmlsaXTDqVwiLCBcIkVuIHRyYWl0ZW1lbnRcIl0pIHtcbiAgICBjb25zdCB1cmwgPSBgaHR0cHM6Ly93d3cuYXNzZW1ibGVlLW5hdGlvbmFsZS5mci9keW4vJHtcbiAgICAgIG9wdGlvbnMubGVnaXNsYXR1cmVcbiAgICB9L2FtZW5kZW1lbnRzP2V0YXQ9JHtlbmNvZGVVUklDb21wb25lbnQoZXRhdCl9YFxuICAgIGZvciBhd2FpdCAoY29uc3QgW1xuICAgICAgYW1lbmRlbWVudFVybFBhdGgsXG4gICAgICBhbWVuZGVtZW50LFxuICAgIF0gb2YgaXRlclJlY2hlcmNoZVJhd0FtZW5kZW1lbnRzKFxuICAgICAgY2FjaGVEaXIsXG4gICAgICB1cmwsXG4gICAgICBvcHRpb25zLmluY3JlbWVudGFsLFxuICAgICAgbnVsbCxcbiAgICAgIG51bGwsXG4gICAgICB7IHNpbGVudDogb3B0aW9ucy5zaWxlbnQsIHZlcmJvc2U6IG9wdGlvbnMudmVyYm9zZSB9LFxuICAgICkpIHtcbiAgICAgIGNvbnN0IGFtZW5kZW1lbnRVcmxQYXRoU3BsaXR0ZWQgPSBhbWVuZGVtZW50VXJsUGF0aCEuc3BsaXQoXCIvXCIpXG4gICAgICBjb25zdCBhbWVuZGVtZW50RmlsZVBhdGggPVxuICAgICAgICBwYXRoLmpvaW4oYW1lbmRlbWVudHNEaXIsIC4uLmFtZW5kZW1lbnRVcmxQYXRoU3BsaXR0ZWQuc2xpY2UoMikpICtcbiAgICAgICAgXCIuanNvblwiXG4gICAgICBhd2FpdCBmcy5lbnN1cmVEaXIocGF0aC5kaXJuYW1lKGFtZW5kZW1lbnRGaWxlUGF0aCkpXG4gICAgICBjb25zdCBleGlzdGluZ0FtZW5kZW1lbnQgPSAoYXdhaXQgZnMucGF0aEV4aXN0cyhhbWVuZGVtZW50RmlsZVBhdGgpKVxuICAgICAgICA/IGF3YWl0IGZzLnJlYWRKc29uKGFtZW5kZW1lbnRGaWxlUGF0aClcbiAgICAgICAgOiBudWxsXG4gICAgICBpZiAoIWRlZXBFcXVhbChhbWVuZGVtZW50LCBleGlzdGluZ0FtZW5kZW1lbnQpKSB7XG4gICAgICAgIGlmICghb3B0aW9ucy5zaWxlbnQpIHtcbiAgICAgICAgICBpZiAoZXhpc3RpbmdBbWVuZGVtZW50ID09PSBudWxsKSB7XG4gICAgICAgICAgICBjb25zb2xlLmxvZyhgQWRkaW5nIGFtZW5kZW1lbnQgSlNPTjogJHthbWVuZGVtZW50RmlsZVBhdGh94oCmYClcbiAgICAgICAgICB9IGVsc2Uge1xuICAgICAgICAgICAgY29uc29sZS5sb2coYFVwZGF0aW5nIGFtZW5kZW1lbnQgSlNPTjogJHthbWVuZGVtZW50RmlsZVBhdGh94oCmYClcbiAgICAgICAgICB9XG4gICAgICAgIH1cbiAgICAgICAgYXdhaXQgZnMud3JpdGVKc29uKGFtZW5kZW1lbnRGaWxlUGF0aCwgYW1lbmRlbWVudCwgeyBzcGFjZXM6IDIgfSlcbiAgICAgIH1cbiAgICB9XG4gIH1cblxuICBpZiAob3B0aW9ucy5jb21taXQpIHtcbiAgICByZXR1cm4gZ2l0LmNvbW1pdEFuZFB1c2goYW1lbmRlbWVudHNEaXIsIFwiTm91dmVsbGUgbW9pc3NvblwiLCBvcHRpb25zLnJlbW90ZSlcbiAgfVxuICByZXR1cm4gMFxufVxuXG5tYWluKClcbiAgLnRoZW4oKGV4aXRDb2RlKSA9PiBwcm9jZXNzLmV4aXQoZXhpdENvZGUpKVxuICAuY2F0Y2goKGVycm9yKSA9PiB7XG4gICAgY29uc29sZS5sb2coZXJyb3IpXG4gICAgcHJvY2Vzcy5leGl0KDEpXG4gIH0pXG4iXSwibWFwcGluZ3MiOiJBQUFBLE9BQU9BLGVBQWUsTUFBTSxtQkFBbUI7QUFDL0MsT0FBT0MsU0FBUyxNQUFNLFlBQVk7QUFDbEMsT0FBT0MsRUFBRSxNQUFNLFVBQVU7QUFDekIsT0FBT0MsSUFBSSxNQUFNLE1BQU07QUFBQSxPQUVoQixLQUFLQyxHQUFHO0FBQUEsU0FDTkMsMkJBQTJCO0FBQUEsU0FFbENDLFdBQVcsRUFDWEMsWUFBWSxFQUNaQyxvQkFBb0IsRUFDcEJDLGlCQUFpQixFQUNqQkMsVUFBVSxFQUNWQyxZQUFZLEVBQ1pDLFlBQVksRUFDWkMsYUFBYTtBQUdmLE1BQU1DLGtCQUFrQixHQUFHLENBQ3pCUixXQUFXLEVBQ1hDLFlBQVksRUFDWkUsaUJBQWlCLEVBQ2pCRSxZQUFZLEVBQ1pDLFlBQVksRUFDWkMsYUFBYSxFQUNiTCxvQkFBb0IsRUFDcEJFLFVBQVUsRUFDVjtFQUNFSyxLQUFLLEVBQUUsR0FBRztFQUNWQyxJQUFJLEVBQUUsaUZBQWlGO0VBQ3ZGQyxJQUFJLEVBQUUsYUFBYTtFQUNuQkMsSUFBSSxFQUFFQztBQUNSLENBQUMsQ0FDRjtBQUNELE1BQU1DLE9BQU8sR0FBR3BCLGVBQWUsQ0FBQ2Msa0JBQWtCLENBQUM7QUFFbkQsZUFBZU8sSUFBSUEsQ0FBQSxFQUFvQjtFQUNyQyxNQUFNQyxXQUFXLEdBQUdGLE9BQU8sQ0FBQ0csS0FBSyxHQUM3QkgsT0FBTyxDQUFDRyxLQUFLLENBQUNDLElBQUksQ0FBQyxDQUFDLENBQUNDLE9BQU8sQ0FBQyxNQUFNLEVBQUUsRUFBRSxDQUFDLEdBQ3hDQyxTQUFTO0VBRWJ0QixHQUFHLENBQUNtQixLQUFLLENBQUNELFdBQVcsRUFBRSx3QkFBd0IsRUFBRUYsT0FBTyxDQUFDTyxPQUFPLENBQUM7RUFFakUsTUFBTUMsY0FBYyxHQUFHekIsSUFBSSxDQUFDMEIsSUFBSSxDQUFDVCxPQUFPLENBQUNPLE9BQU8sRUFBRSx3QkFBd0IsQ0FBQztFQUMzRSxJQUFJUCxPQUFPLENBQUNVLElBQUksRUFBRTtJQUNoQjFCLEdBQUcsQ0FBQzJCLFlBQVksQ0FBQ0gsY0FBYyxDQUFDO0VBQ2xDO0VBRUEsTUFBTTFCLEVBQUUsQ0FBQzhCLFNBQVMsQ0FBQ0osY0FBYyxDQUFDO0VBQ2xDLE1BQU1LLFFBQVEsR0FBRzlCLElBQUksQ0FBQzBCLElBQUksQ0FBQ1QsT0FBTyxDQUFDTyxPQUFPLEVBQUUsOEJBQThCLENBQUM7RUFDM0UsS0FBSyxNQUFNTyxJQUFJLElBQUksQ0FBQyxpQkFBaUIsRUFBRSxlQUFlLENBQUMsRUFBRTtJQUN2RCxNQUFNQyxHQUFHLEdBQUcsMENBQ1ZmLE9BQU8sQ0FBQ2dCLFdBQVcscUJBQ0FDLGtCQUFrQixDQUFDSCxJQUFJLENBQUMsRUFBRTtJQUMvQyxXQUFXLE1BQU0sQ0FDZkksaUJBQWlCLEVBQ2pCQyxVQUFVLENBQ1gsSUFBSWxDLDJCQUEyQixDQUM5QjRCLFFBQVEsRUFDUkUsR0FBRyxFQUNIZixPQUFPLENBQUNvQixXQUFXLEVBQ25CLElBQUksRUFDSixJQUFJLEVBQ0o7TUFBRUMsTUFBTSxFQUFFckIsT0FBTyxDQUFDcUIsTUFBTTtNQUFFQyxPQUFPLEVBQUV0QixPQUFPLENBQUNzQjtJQUFRLENBQ3JELENBQUMsRUFBRTtNQUNELE1BQU1DLHlCQUF5QixHQUFHTCxpQkFBaUIsQ0FBRU0sS0FBSyxDQUFDLEdBQUcsQ0FBQztNQUMvRCxNQUFNQyxrQkFBa0IsR0FDdEIxQyxJQUFJLENBQUMwQixJQUFJLENBQUNELGNBQWMsRUFBRSxHQUFHZSx5QkFBeUIsQ0FBQ0csS0FBSyxDQUFDLENBQUMsQ0FBQyxDQUFDLEdBQ2hFLE9BQU87TUFDVCxNQUFNNUMsRUFBRSxDQUFDOEIsU0FBUyxDQUFDN0IsSUFBSSxDQUFDNEMsT0FBTyxDQUFDRixrQkFBa0IsQ0FBQyxDQUFDO01BQ3BELE1BQU1HLGtCQUFrQixHQUFHLENBQUMsTUFBTTlDLEVBQUUsQ0FBQytDLFVBQVUsQ0FBQ0osa0JBQWtCLENBQUMsSUFDL0QsTUFBTTNDLEVBQUUsQ0FBQ2dELFFBQVEsQ0FBQ0wsa0JBQWtCLENBQUMsR0FDckMsSUFBSTtNQUNSLElBQUksQ0FBQzVDLFNBQVMsQ0FBQ3NDLFVBQVUsRUFBRVMsa0JBQWtCLENBQUMsRUFBRTtRQUM5QyxJQUFJLENBQUM1QixPQUFPLENBQUNxQixNQUFNLEVBQUU7VUFDbkIsSUFBSU8sa0JBQWtCLEtBQUssSUFBSSxFQUFFO1lBQy9CRyxPQUFPLENBQUNDLEdBQUcsQ0FBQywyQkFBMkJQLGtCQUFrQixHQUFHLENBQUM7VUFDL0QsQ0FBQyxNQUFNO1lBQ0xNLE9BQU8sQ0FBQ0MsR0FBRyxDQUFDLDZCQUE2QlAsa0JBQWtCLEdBQUcsQ0FBQztVQUNqRTtRQUNGO1FBQ0EsTUFBTTNDLEVBQUUsQ0FBQ21ELFNBQVMsQ0FBQ1Isa0JBQWtCLEVBQUVOLFVBQVUsRUFBRTtVQUFFZSxNQUFNLEVBQUU7UUFBRSxDQUFDLENBQUM7TUFDbkU7SUFDRjtFQUNGO0VBRUEsSUFBSWxDLE9BQU8sQ0FBQ21DLE1BQU0sRUFBRTtJQUNsQixPQUFPbkQsR0FBRyxDQUFDb0QsYUFBYSxDQUFDNUIsY0FBYyxFQUFFLGtCQUFrQixFQUFFUixPQUFPLENBQUNxQyxNQUFNLENBQUM7RUFDOUU7RUFDQSxPQUFPLENBQUM7QUFDVjtBQUVBcEMsSUFBSSxDQUFDLENBQUMsQ0FDSHFDLElBQUksQ0FBRUMsUUFBUSxJQUFLQyxPQUFPLENBQUNDLElBQUksQ0FBQ0YsUUFBUSxDQUFDLENBQUMsQ0FDMUNHLEtBQUssQ0FBRUMsS0FBSyxJQUFLO0VBQ2hCWixPQUFPLENBQUNDLEdBQUcsQ0FBQ1csS0FBSyxDQUFDO0VBQ2xCSCxPQUFPLENBQUNDLElBQUksQ0FBQyxDQUFDLENBQUM7QUFDakIsQ0FBQyxDQUFDIiwiaWdub3JlTGlzdCI6W119