@tricoteuses/senat
Version:
Handle French Sénat's open data
147 lines (146 loc) • 5.63 kB
JavaScript
import fs from 'fs-extra';
import path from 'path';
import commandLineArgs from 'command-line-args';
import { datasets, EnabledDatasets, getEnabledDatasets, } from '../datasets';
import { Aggregator, allFollows, getAllLois, getAllQuestions, getAllSens } from '../model';
import { insertLoiReferences, insertQuestionReferences, insertSenReferences } from '../inserters';
const optionsDefinitions = [
{
alias: 'k',
defaultValue: ['All'],
help: 'categories of datasets to reorganize',
multiple: true,
name: 'categories',
type: String,
},
{
alias: 's',
help: 'don\'t log anything',
name: 'silent',
type: Boolean,
},
{
alias: "v",
help: "verbose logs",
name: "verbose",
type: Boolean,
},
{
defaultOption: true,
help: 'directory containing Sénat open data files',
name: 'dataDir',
type: String,
},
];
const options = commandLineArgs(optionsDefinitions);
const SIGNET_STRUCTURE_REGEXP = /^(?<type>[a-z]+)(?<session>\d{2,4})-?(?<numTexte>\d+)?/;
function ensureAndClearDir(path) {
if (!fs.existsSync(path)) {
fs.mkdirSync(path);
}
else {
fs.emptyDirSync(path);
}
}
async function convertData() {
const enabledDatasets = getEnabledDatasets(options.categories);
const dataDir = options.dataDir;
if (enabledDatasets & EnabledDatasets.DosLeg) {
const dataset = datasets.dosleg;
if (!options.silent) {
console.log(`Converting database ${dataset.database} data into files…`);
}
const lois = await getAllLois();
const aggregator = new Aggregator(new Set(allFollows));
for (const loi of lois) {
aggregator.addLoi(loi);
}
if (options.verbose) {
console.log("Starting getting all the data…");
console.time("get all data");
}
await aggregator.getAll();
if (options.verbose) {
console.timeEnd("get all data");
}
const allData = aggregator.toJson();
const loiReorganizedRootDir = path.join(dataDir, dataset.database);
ensureAndClearDir(loiReorganizedRootDir);
for (const loi of lois) {
if (options.verbose) {
console.log(`Converting ${loi.signet} file…`);
}
insertLoiReferences(loi, allData, {});
let loiFileName = `${loi.loicod}.json`;
let loiReorganizedDir = path.join(loiReorganizedRootDir, loi.typloicod);
if (loi.signet) {
loiFileName = `${loi.signet}.json`;
let signetParts = SIGNET_STRUCTURE_REGEXP.exec(loi.signet)?.groups;
if (signetParts) {
const { session } = signetParts;
loiReorganizedDir = path.join(loiReorganizedDir, session);
}
else {
loiReorganizedDir = path.join(loiReorganizedDir, "non defini");
}
fs.ensureDirSync(loiReorganizedDir);
}
fs.writeJSONSync(path.join(loiReorganizedDir, loiFileName), loi, { spaces: 2 });
}
}
if (enabledDatasets & EnabledDatasets.Questions) {
const dataset = datasets.questions;
if (!options.silent) {
console.log(`Converting database ${dataset.database} data into files…`);
}
const questions = await getAllQuestions();
const aggregator = new Aggregator(new Set(allFollows));
for (const question of questions) {
aggregator.addQuestion(question);
}
await aggregator.getAll();
const allData = aggregator.toJson();
const questionReorganizedRootDir = path.join(dataDir, dataset.database);
ensureAndClearDir(questionReorganizedRootDir);
for (const question of questions) {
if (options.verbose) {
console.log(`Converting ${question.id} file…`);
}
insertQuestionReferences(question, allData, {});
const legislature = question.legislature ? question.legislature : 0;
const questionReorganizedDir = path.join(questionReorganizedRootDir, String(legislature));
fs.ensureDirSync(questionReorganizedDir);
let questionFileName = `${question.reference}.json`;
fs.writeJSONSync(path.join(questionReorganizedDir, questionFileName), question, { spaces: 2 });
}
}
if (enabledDatasets & EnabledDatasets.Sens) {
const dataset = datasets.sens;
if (!options.silent) {
console.log(`Converting database ${dataset.database} data into files…`);
}
const sens = await getAllSens();
const aggregator = new Aggregator(new Set(allFollows));
for (const sen of sens) {
aggregator.addSen(sen);
}
await aggregator.getAll();
const allData = aggregator.toJson();
const senReorganizedRootDir = path.join(dataDir, dataset.database);
ensureAndClearDir(senReorganizedRootDir);
for (const sen of sens) {
if (options.verbose) {
console.log(`Converting ${sen.senmat} file…`);
}
insertSenReferences(sen, allData, {});
let senFileName = `${sen.senmat}.json`;
fs.writeJSONSync(path.join(senReorganizedRootDir, senFileName), sen, { spaces: 2 });
}
}
}
convertData()
.then(() => process.exit(0))
.catch((error) => {
console.log(error);
process.exit(1);
});