@tricoteuses/senat/lib/datasets.ts

Handle French Sénat's open data

import fs from "fs-extra"
import path from "path"
import assert from "assert"

export interface Dataset {
  database: string
  repairEncoding: boolean
  repairZip?: (dataset: Dataset, dataDir: string) => void
  schema: string
  title: string
  url: string
}

export interface Datasets {
  ameli: Dataset
  debats: Dataset
  dosleg: Dataset
  questions: Dataset
  sens: Dataset
}

export enum EnabledDatasets {
  None = 0,
  Ameli = 1 << 0,
  Debats = 1 << 1,
  DosLeg = 1 << 2,
  Questions = 1 << 3,
  Sens = 1 << 4,
  PhotosSenateurs = 1 << 5,
  All = Ameli | Debats | DosLeg | Questions | Sens | PhotosSenateurs,
}

export const datasets: Datasets = {
  ameli: {
    database: "ameli",
    repairEncoding: false,
    repairZip: (dataset: Dataset, dataDir: string) => {
      const sqlFilename = `${dataset.database}.sql`
      const sqlFilePath = path.join(dataDir, sqlFilename)
      fs.removeSync(sqlFilePath)
      fs.moveSync(
        path.join(dataDir, "var", "opt", "opendata", sqlFilename),
        sqlFilePath,
      )
    },
    schema: "public",
    title: "Amendements",
    url: "https://data.senat.fr/data/ameli/ameli.zip",
  },
  debats: {
    database: "debats",
    repairEncoding: true,
    schema: "public",
    title:
      "Informations relatives aux comptes rendus intégraux de la séance publique",
    url: "https://data.senat.fr/data/debats/debats.zip",
  },
  dosleg: {
    database: "dosleg",
    repairEncoding: true,
    schema: "public",
    title: "Dossiers législatifs",
    url: "https://data.senat.fr/data/dosleg/dosleg.zip",
  },
  questions: {
    database: "questions",
    repairEncoding: true,
    schema: "questions",
    title:
      "Questions écrites et orales posées par les sénateurs au Gouvernement",
    url: "https://data.senat.fr/data/questions/questions.zip",
  },
  sens: {
    database: "sens",
    repairEncoding: true,
    repairZip: (dataset: Dataset, dataDir: string) => {
      const sqlFilename = `${dataset.database}.sql`
      const sqlFilePath = path.join(dataDir, sqlFilename)
      fs.removeSync(sqlFilePath)
      fs.moveSync(path.join(dataDir, "export_sens.sql"), sqlFilePath)
    },
    schema: "public",
    title: "Sénateurs (y compris organes et présence)",
    url: "https://data.senat.fr/data/senateurs/export_sens.zip",
  },
}

export function getEnabledDatasets(categories: any): EnabledDatasets {
  categories.map((datasetName: string) =>
    assert.notStrictEqual(
      (EnabledDatasets as any as { [name: string]: EnabledDatasets })[
        datasetName
      ],
      undefined,
      `Unknown name of dataset: ${datasetName}`,
    ),
  )

  return categories.reduce(
    (enabledDatasets: EnabledDatasets, datasetName: string): EnabledDatasets =>
      enabledDatasets |
      (EnabledDatasets as any as { [name: string]: EnabledDatasets })[
        datasetName
      ],
    EnabledDatasets.None,
  )
}

export function getChosenFromEnabledDatasets(categories: any): Dataset[] {
  const enabledDatasets = getEnabledDatasets(categories)
  return  [
    enabledDatasets & EnabledDatasets.Ameli ? datasets.ameli : null,
    enabledDatasets & EnabledDatasets.Debats ? datasets.debats : null,
    enabledDatasets & EnabledDatasets.DosLeg ? datasets.dosleg : null,
    enabledDatasets & EnabledDatasets.Questions ? datasets.questions : null,
    enabledDatasets & EnabledDatasets.Sens ? datasets.sens : null,
  ].filter((dataset: Dataset | null) => dataset !== null) as Dataset[]
}