@tricoteuses/senat/lib/scripts/convert_data.ts

Handle French Sénat's open data

import fs from 'fs-extra'
import path from 'path'
import commandLineArgs from 'command-line-args'

import { datasets, EnabledDatasets, getEnabledDatasets, } from '../datasets'
import {
  Aggregator,
  allFollows,
  getAllLois,
  getAllQuestions,
  getAllSens
} from '../model'
import {
  insertLoiReferences,
  insertQuestionReferences,
  insertSenReferences
} from '../inserters'

const optionsDefinitions = [
  {
    alias: 'k',
    defaultValue: ['All'],
    help: 'categories of datasets to reorganize',
    multiple: true,
    name: 'categories',
    type: String,
  },
  {
    alias: 's',
    help: 'don\'t log anything',
    name: 'silent',
    type: Boolean,
  },
  {
    alias: "v",
    help: "verbose logs",
    name: "verbose",
    type: Boolean,
  },
  {
    defaultOption: true,
    help: 'directory containing Sénat open data files',
    name: 'dataDir',
    type: String,
  },
]
const options = commandLineArgs(optionsDefinitions)

const SIGNET_STRUCTURE_REGEXP = /^(?<type>[a-z]+)(?<session>\d{2,4})-?(?<numTexte>\d+)?/

function ensureAndClearDir(path: string) {
  if (!fs.existsSync(path)) {
    fs.mkdirSync(path)
  } else {
    fs.emptyDirSync(path)
  }
}

async function convertData() {
  const enabledDatasets = getEnabledDatasets(options.categories)
  const dataDir: string = options.dataDir

  if (enabledDatasets & EnabledDatasets.DosLeg) {
    const dataset = datasets.dosleg

    if (!options.silent) {
      console.log(`Converting database ${dataset.database} data into files…`)
    }

    const lois = await getAllLois()
    const aggregator = new Aggregator(new Set(allFollows))
    for (const loi of lois) {
      aggregator.addLoi(loi)
    }
    if (options.verbose) {
      console.log("Starting getting all the data…")
      console.time("get all data")
    }
    await aggregator.getAll()
    if (options.verbose) {
      console.timeEnd("get all data")
    }
    const allData = aggregator.toJson()

    const loiReorganizedRootDir = path.join(dataDir, dataset.database)
    ensureAndClearDir(loiReorganizedRootDir)

    for (const loi of lois) {
      if (options.verbose) {
        console.log(`Converting ${loi.signet} file…`)
      }

      insertLoiReferences(loi, allData, {})

      let loiFileName = `${loi.loicod}.json`
      let loiReorganizedDir = path.join(loiReorganizedRootDir, loi.typloicod)
      if (loi.signet) {
        loiFileName = `${loi.signet}.json`
        let signetParts = SIGNET_STRUCTURE_REGEXP.exec(loi.signet)?.groups
        if (signetParts) {
          const { session} = signetParts
          loiReorganizedDir = path.join(loiReorganizedDir, session)
        } else {
          loiReorganizedDir = path.join(loiReorganizedDir, "non defini")
        }
        fs.ensureDirSync(loiReorganizedDir)
      }

      fs.writeJSONSync(path.join(loiReorganizedDir, loiFileName), loi, {spaces: 2})
    }
  }

  if (enabledDatasets & EnabledDatasets.Questions) {
    const dataset = datasets.questions

    if (!options.silent) {
      console.log(`Converting database ${dataset.database} data into files…`)
    }

    const questions = await getAllQuestions()
    const aggregator = new Aggregator(new Set(allFollows))
    for (const question of questions) {
      aggregator.addQuestion(question)
    }
    await aggregator.getAll()
    const allData = aggregator.toJson()

    const questionReorganizedRootDir = path.join(dataDir, dataset.database)
    ensureAndClearDir(questionReorganizedRootDir)

    for (const question of questions) {
      if (options.verbose) {
        console.log(`Converting ${question.id} file…`)
      }

      insertQuestionReferences(question, allData, {})

      const legislature = question.legislature ? question.legislature : 0
      const questionReorganizedDir = path.join(questionReorganizedRootDir, String(legislature))
      fs.ensureDirSync(questionReorganizedDir)
      let questionFileName = `${question.reference}.json`
      fs.writeJSONSync(path.join(questionReorganizedDir, questionFileName), question, {spaces: 2})
    }
  }

  if (enabledDatasets & EnabledDatasets.Sens) {
    const dataset = datasets.sens

    if (!options.silent) {
      console.log(`Converting database ${dataset.database} data into files…`)
    }

    const sens = await getAllSens()
    const aggregator = new Aggregator(new Set(allFollows))
    for (const sen of sens) {
      aggregator.addSen(sen)
    }
    await aggregator.getAll()
    const allData = aggregator.toJson()

    const senReorganizedRootDir = path.join(dataDir, dataset.database)
    ensureAndClearDir(senReorganizedRootDir)

    for (const sen of sens) {
      if (options.verbose) {
        console.log(`Converting ${sen.senmat} file…`)
      }

      insertSenReferences(sen, allData, {})

      let senFileName = `${sen.senmat}.json`
      fs.writeJSONSync(path.join(senReorganizedRootDir, senFileName), sen, {spaces: 2})
    }
  }
}

convertData()
  .then(() => process.exit(0))
  .catch((error) => {
    console.log(error)
    process.exit(1)
  })