@tricoteuses/assemblee
Version:
Retrieve, clean up & handle French Assemblée nationale's open data
47 lines • 8.34 kB
JavaScript
import commandLineArgs from "command-line-args";
import fs from "fs-extra";
import path from "path";
import { datasets, DatasetStructure } from "../datasets.mjs";
import { walkDir } from "../file_systems.mjs";
import { dataDirDefaultOption, silentOption, verboseOption } from "./shared/cli_helpers.mjs";
const optionsDefinitions = [silentOption, verboseOption, dataDirDefaultOption];
const options = commandLineArgs(optionsDefinitions);
function mergeScrutins(dataDir) {
for (const dataset of datasets.scrutins) {
switch (dataset.structure) {
case DatasetStructure.SegmentedFiles:
{
const originalJsonDir = path.join(dataDir, dataset.filename);
if (!options.silent) {
console.log(`Merging ${originalJsonDir}`);
}
const datasetMergedFilePath = path.join(dataDir, path.basename(originalJsonDir, ".json") + "_fusionne.json");
const scrutins = [];
for (const scrutinSplitPath of walkDir(originalJsonDir)) {
const scrutinFilename = scrutinSplitPath[scrutinSplitPath.length - 1];
if (!scrutinFilename.endsWith(".json")) {
continue;
}
const scrutinOriginalFilePath = path.join(originalJsonDir, ...scrutinSplitPath);
const scrutinOriginalJson = fs.readFileSync(scrutinOriginalFilePath, {
encoding: "utf8"
});
const scrutinOriginal = JSON.parse(scrutinOriginalJson);
const scrutin = scrutinOriginal.scrutin;
scrutins.push(scrutin);
}
scrutins.sort((a, b) => a.uid.length === b.uid.length ? a.uid.localeCompare(b.uid) : a.uid.length - b.uid.length);
fs.writeFileSync(datasetMergedFilePath, JSON.stringify({
scrutins: {
scrutin: scrutins
}
}, null, 2), {
encoding: "utf8"
});
}
break;
}
}
}
mergeScrutins(options.dataDir);
//# sourceMappingURL=data:application/json;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJuYW1lcyI6WyJjb21tYW5kTGluZUFyZ3MiLCJmcyIsInBhdGgiLCJkYXRhc2V0cyIsIkRhdGFzZXRTdHJ1Y3R1cmUiLCJ3YWxrRGlyIiwiZGF0YURpckRlZmF1bHRPcHRpb24iLCJzaWxlbnRPcHRpb24iLCJ2ZXJib3NlT3B0aW9uIiwib3B0aW9uc0RlZmluaXRpb25zIiwib3B0aW9ucyIsIm1lcmdlU2NydXRpbnMiLCJkYXRhRGlyIiwiZGF0YXNldCIsInNjcnV0aW5zIiwic3RydWN0dXJlIiwiU2VnbWVudGVkRmlsZXMiLCJvcmlnaW5hbEpzb25EaXIiLCJqb2luIiwiZmlsZW5hbWUiLCJzaWxlbnQiLCJjb25zb2xlIiwibG9nIiwiZGF0YXNldE1lcmdlZEZpbGVQYXRoIiwiYmFzZW5hbWUiLCJzY3J1dGluU3BsaXRQYXRoIiwic2NydXRpbkZpbGVuYW1lIiwibGVuZ3RoIiwiZW5kc1dpdGgiLCJzY3J1dGluT3JpZ2luYWxGaWxlUGF0aCIsInNjcnV0aW5PcmlnaW5hbEpzb24iLCJyZWFkRmlsZVN5bmMiLCJlbmNvZGluZyIsInNjcnV0aW5PcmlnaW5hbCIsIkpTT04iLCJwYXJzZSIsInNjcnV0aW4iLCJwdXNoIiwic29ydCIsImEiLCJiIiwidWlkIiwibG9jYWxlQ29tcGFyZSIsIndyaXRlRmlsZVN5bmMiLCJzdHJpbmdpZnkiXSwic291cmNlcyI6WyIuLi8uLi9zcmMvc2NyaXB0cy9tZXJnZV9zY3J1dGlucy50cyJdLCJzb3VyY2VzQ29udGVudCI6WyJpbXBvcnQgY29tbWFuZExpbmVBcmdzIGZyb20gXCJjb21tYW5kLWxpbmUtYXJnc1wiXG5pbXBvcnQgZnMgZnJvbSBcImZzLWV4dHJhXCJcbmltcG9ydCBwYXRoIGZyb20gXCJwYXRoXCJcblxuaW1wb3J0IHsgZGF0YXNldHMsIERhdGFzZXRTdHJ1Y3R1cmUgfSBmcm9tIFwiLi4vZGF0YXNldHNcIlxuaW1wb3J0IHsgd2Fsa0RpciB9IGZyb20gXCIuLi9maWxlX3N5c3RlbXNcIlxuaW1wb3J0IHtcbiAgZGF0YURpckRlZmF1bHRPcHRpb24sXG4gIHNpbGVudE9wdGlvbixcbiAgdmVyYm9zZU9wdGlvbixcbn0gZnJvbSBcIi4vc2hhcmVkL2NsaV9oZWxwZXJzXCJcblxuY29uc3Qgb3B0aW9uc0RlZmluaXRpb25zID0gW3NpbGVudE9wdGlvbiwgdmVyYm9zZU9wdGlvbiwgZGF0YURpckRlZmF1bHRPcHRpb25dXG5jb25zdCBvcHRpb25zID0gY29tbWFuZExpbmVBcmdzKG9wdGlvbnNEZWZpbml0aW9ucylcblxuZnVuY3Rpb24gbWVyZ2VTY3J1dGlucyhkYXRhRGlyOiBzdHJpbmcpIHtcbiAgZm9yIChjb25zdCBkYXRhc2V0IG9mIGRhdGFzZXRzLnNjcnV0aW5zKSB7XG4gICAgc3dpdGNoIChkYXRhc2V0LnN0cnVjdHVyZSkge1xuICAgICAgY2FzZSBEYXRhc2V0U3RydWN0dXJlLlNlZ21lbnRlZEZpbGVzOlxuICAgICAgICB7XG4gICAgICAgICAgY29uc3Qgb3JpZ2luYWxKc29uRGlyOiBzdHJpbmcgPSBwYXRoLmpvaW4oZGF0YURpciwgZGF0YXNldC5maWxlbmFtZSlcbiAgICAgICAgICBpZiAoIW9wdGlvbnMuc2lsZW50KSB7XG4gICAgICAgICAgICBjb25zb2xlLmxvZyhgTWVyZ2luZyAke29yaWdpbmFsSnNvbkRpcn1gKVxuICAgICAgICAgIH1cblxuICAgICAgICAgIGNvbnN0IGRhdGFzZXRNZXJnZWRGaWxlUGF0aCA9IHBhdGguam9pbihcbiAgICAgICAgICAgIGRhdGFEaXIsXG4gICAgICAgICAgICBwYXRoLmJhc2VuYW1lKG9yaWdpbmFsSnNvbkRpciwgXCIuanNvblwiKSArIFwiX2Z1c2lvbm5lLmpzb25cIixcbiAgICAgICAgICApXG5cbiAgICAgICAgICBjb25zdCBzY3J1dGlucyA9IFtdXG4gICAgICAgICAgZm9yIChjb25zdCBzY3J1dGluU3BsaXRQYXRoIG9mIHdhbGtEaXIob3JpZ2luYWxKc29uRGlyKSkge1xuICAgICAgICAgICAgY29uc3Qgc2NydXRpbkZpbGVuYW1lID1cbiAgICAgICAgICAgICAgc2NydXRpblNwbGl0UGF0aFtzY3J1dGluU3BsaXRQYXRoLmxlbmd0aCAtIDFdXG4gICAgICAgICAgICBpZiAoIXNjcnV0aW5GaWxlbmFtZS5lbmRzV2l0aChcIi5qc29uXCIpKSB7XG4gICAgICAgICAgICAgIGNvbnRpbnVlXG4gICAgICAgICAgICB9XG4gICAgICAgICAgICBjb25zdCBzY3J1dGluT3JpZ2luYWxGaWxlUGF0aCA9IHBhdGguam9pbihcbiAgICAgICAgICAgICAgb3JpZ2luYWxKc29uRGlyLFxuICAgICAgICAgICAgICAuLi5zY3J1dGluU3BsaXRQYXRoLFxuICAgICAgICAgICAgKVxuICAgICAgICAgICAgY29uc3Qgc2NydXRpbk9yaWdpbmFsSnNvbjogc3RyaW5nID0gZnMucmVhZEZpbGVTeW5jKFxuICAgICAgICAgICAgICBzY3J1dGluT3JpZ2luYWxGaWxlUGF0aCxcbiAgICAgICAgICAgICAge1xuICAgICAgICAgICAgICAgIGVuY29kaW5nOiBcInV0ZjhcIixcbiAgICAgICAgICAgICAgfSxcbiAgICAgICAgICAgIClcbiAgICAgICAgICAgIGNvbnN0IHNjcnV0aW5PcmlnaW5hbDogYW55ID0gSlNPTi5wYXJzZShzY3J1dGluT3JpZ2luYWxKc29uKVxuICAgICAgICAgICAgY29uc3Qgc2NydXRpbiA9IHNjcnV0aW5PcmlnaW5hbC5zY3J1dGluXG4gICAgICAgICAgICBzY3J1dGlucy5wdXNoKHNjcnV0aW4pXG4gICAgICAgICAgfVxuICAgICAgICAgIHNjcnV0aW5zLnNvcnQoKGE6IHsgdWlkOiBzdHJpbmcgfSwgYjogeyB1aWQ6IHN0cmluZyB9KSA9PlxuICAgICAgICAgICAgYS51aWQubGVuZ3RoID09PSBiLnVpZC5sZW5ndGhcbiAgICAgICAgICAgICAgPyBhLnVpZC5sb2NhbGVDb21wYXJlKGIudWlkKVxuICAgICAgICAgICAgICA6IGEudWlkLmxlbmd0aCAtIGIudWlkLmxlbmd0aCxcbiAgICAgICAgICApXG5cbiAgICAgICAgICBmcy53cml0ZUZpbGVTeW5jKFxuICAgICAgICAgICAgZGF0YXNldE1lcmdlZEZpbGVQYXRoLFxuICAgICAgICAgICAgSlNPTi5zdHJpbmdpZnkoeyBzY3J1dGluczogeyBzY3J1dGluOiBzY3J1dGlucyB9IH0sIG51bGwsIDIpLFxuICAgICAgICAgICAge1xuICAgICAgICAgICAgICBlbmNvZGluZzogXCJ1dGY4XCIsXG4gICAgICAgICAgICB9LFxuICAgICAgICAgIClcbiAgICAgICAgfVxuICAgICAgICBicmVha1xuICAgIH1cbiAgfVxufVxuXG5tZXJnZVNjcnV0aW5zKG9wdGlvbnMuZGF0YURpcilcbiJdLCJtYXBwaW5ncyI6IkFBQUEsT0FBT0EsZUFBZSxNQUFNLG1CQUFtQjtBQUMvQyxPQUFPQyxFQUFFLE1BQU0sVUFBVTtBQUN6QixPQUFPQyxJQUFJLE1BQU0sTUFBTTtBQUFBLFNBRWRDLFFBQVEsRUFBRUMsZ0JBQWdCO0FBQUEsU0FDMUJDLE9BQU87QUFBQSxTQUVkQyxvQkFBb0IsRUFDcEJDLFlBQVksRUFDWkMsYUFBYTtBQUdmLE1BQU1DLGtCQUFrQixHQUFHLENBQUNGLFlBQVksRUFBRUMsYUFBYSxFQUFFRixvQkFBb0IsQ0FBQztBQUM5RSxNQUFNSSxPQUFPLEdBQUdWLGVBQWUsQ0FBQ1Msa0JBQWtCLENBQUM7QUFFbkQsU0FBU0UsYUFBYUEsQ0FBQ0MsT0FBZSxFQUFFO0VBQ3RDLEtBQUssTUFBTUMsT0FBTyxJQUFJVixRQUFRLENBQUNXLFFBQVEsRUFBRTtJQUN2QyxRQUFRRCxPQUFPLENBQUNFLFNBQVM7TUFDdkIsS0FBS1gsZ0JBQWdCLENBQUNZLGNBQWM7UUFDbEM7VUFDRSxNQUFNQyxlQUF1QixHQUFHZixJQUFJLENBQUNnQixJQUFJLENBQUNOLE9BQU8sRUFBRUMsT0FBTyxDQUFDTSxRQUFRLENBQUM7VUFDcEUsSUFBSSxDQUFDVCxPQUFPLENBQUNVLE1BQU0sRUFBRTtZQUNuQkMsT0FBTyxDQUFDQyxHQUFHLENBQUMsV0FBV0wsZUFBZSxFQUFFLENBQUM7VUFDM0M7VUFFQSxNQUFNTSxxQkFBcUIsR0FBR3JCLElBQUksQ0FBQ2dCLElBQUksQ0FDckNOLE9BQU8sRUFDUFYsSUFBSSxDQUFDc0IsUUFBUSxDQUFDUCxlQUFlLEVBQUUsT0FBTyxDQUFDLEdBQUcsZ0JBQzVDLENBQUM7VUFFRCxNQUFNSCxRQUFRLEdBQUcsRUFBRTtVQUNuQixLQUFLLE1BQU1XLGdCQUFnQixJQUFJcEIsT0FBTyxDQUFDWSxlQUFlLENBQUMsRUFBRTtZQUN2RCxNQUFNUyxlQUFlLEdBQ25CRCxnQkFBZ0IsQ0FBQ0EsZ0JBQWdCLENBQUNFLE1BQU0sR0FBRyxDQUFDLENBQUM7WUFDL0MsSUFBSSxDQUFDRCxlQUFlLENBQUNFLFFBQVEsQ0FBQyxPQUFPLENBQUMsRUFBRTtjQUN0QztZQUNGO1lBQ0EsTUFBTUMsdUJBQXVCLEdBQUczQixJQUFJLENBQUNnQixJQUFJLENBQ3ZDRCxlQUFlLEVBQ2YsR0FBR1EsZ0JBQ0wsQ0FBQztZQUNELE1BQU1LLG1CQUEyQixHQUFHN0IsRUFBRSxDQUFDOEIsWUFBWSxDQUNqREYsdUJBQXVCLEVBQ3ZCO2NBQ0VHLFFBQVEsRUFBRTtZQUNaLENBQ0YsQ0FBQztZQUNELE1BQU1DLGVBQW9CLEdBQUdDLElBQUksQ0FBQ0MsS0FBSyxDQUFDTCxtQkFBbUIsQ0FBQztZQUM1RCxNQUFNTSxPQUFPLEdBQUdILGVBQWUsQ0FBQ0csT0FBTztZQUN2Q3RCLFFBQVEsQ0FBQ3VCLElBQUksQ0FBQ0QsT0FBTyxDQUFDO1VBQ3hCO1VBQ0F0QixRQUFRLENBQUN3QixJQUFJLENBQUMsQ0FBQ0MsQ0FBa0IsRUFBRUMsQ0FBa0IsS0FDbkRELENBQUMsQ0FBQ0UsR0FBRyxDQUFDZCxNQUFNLEtBQUthLENBQUMsQ0FBQ0MsR0FBRyxDQUFDZCxNQUFNLEdBQ3pCWSxDQUFDLENBQUNFLEdBQUcsQ0FBQ0MsYUFBYSxDQUFDRixDQUFDLENBQUNDLEdBQUcsQ0FBQyxHQUMxQkYsQ0FBQyxDQUFDRSxHQUFHLENBQUNkLE1BQU0sR0FBR2EsQ0FBQyxDQUFDQyxHQUFHLENBQUNkLE1BQzNCLENBQUM7VUFFRDFCLEVBQUUsQ0FBQzBDLGFBQWEsQ0FDZHBCLHFCQUFxQixFQUNyQlcsSUFBSSxDQUFDVSxTQUFTLENBQUM7WUFBRTlCLFFBQVEsRUFBRTtjQUFFc0IsT0FBTyxFQUFFdEI7WUFBUztVQUFFLENBQUMsRUFBRSxJQUFJLEVBQUUsQ0FBQyxDQUFDLEVBQzVEO1lBQ0VrQixRQUFRLEVBQUU7VUFDWixDQUNGLENBQUM7UUFDSDtRQUNBO0lBQ0o7RUFDRjtBQUNGO0FBRUFyQixhQUFhLENBQUNELE9BQU8sQ0FBQ0UsT0FBTyxDQUFDIiwiaWdub3JlTGlzdCI6W119