@tricoteuses/senat/lib/scripts/retrieve_senateurs_photos.ts

Handle French Sénat's open data

import assert from "assert"
import { execSync } from "child_process"
import commandLineArgs from "command-line-args"
import fs from "fs-extra"
// import fetch from "node-fetch"
import path from "path"
// import stream from "stream"
// import util from "util"

import { Photo, Sen, senFieldsToParseInt, senFieldsToTrim } from "../types/sens"
import { checkDatabase } from "../databases"
import { parseIntFields, trimFieldsRight } from "../fields"
import { slugify } from "../strings"

const optionsDefinitions = [
  {
    alias: "f",
    help: "fetch sénateurs' pictures instead of retrieving them from files",
    name: "fetch",
    type: Boolean,
  },
  {
    alias: "s",
    help: "don't log anything",
    name: "silent",
    type: Boolean,
  },
  {
    defaultOption: true,
    help: "directory containing Sénat open data files",
    name: "dataDir",
    type: String,
  },
]
const options = commandLineArgs(optionsDefinitions)
// const pipeline = util.promisify(stream.pipeline)

async function retrievePhotosSenateurs(): Promise<void> {
  const dataDir = options.dataDir
  assert(dataDir, "Missing argument: data directory")

  const db = await checkDatabase("sens")

  const sens: Sen[] = (
    await db.any(
      `
      SELECT *
      FROM sen
      WHERE etasencod = 'ACTIF'
    `,
    )
  ).map((sen: Sen) =>
    parseIntFields(senFieldsToParseInt, trimFieldsRight(senFieldsToTrim, sen)),
  )

  const photosDir = path.join(dataDir, "photos_senateurs")
  const missingPhotoFilePath = path.resolve(__dirname, "images", "transparent_155x225.jpg")

  // Download photos.
  fs.ensureDirSync(photosDir)
  if (options.fetch) {
    for (const sen of sens) {
      const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
        sen.senprenomuse,
        "_",
      )}${slugify(sen.senmat, "_")}`
      const photoFilename = photoStem + ".jpg"
      const photoFilePath = path.join(photosDir, photoFilename)
      const photoTempFilename = photoStem + "_temp.jpg"
      const photoTempFilePath = path.join(photosDir, photoTempFilename)
      const urlPhoto = `https://www.senat.fr/senimg/${photoFilename}`
      if (!options.silent) {
        console.log(
          `Loading photo ${urlPhoto} for ${sen.senprenomuse} ${sen.sennomuse}…`,
        )
      }
      // Fetch fails with OpenSSL error: dh key too small.
      // (so does "curl").
      // for (let retries = 0; retries < 3; retries++) {
      //   const response = await fetch(urlPhoto)
      //   if (response.ok) {
      //     await pipeline(response.body, fs.createWriteStream(photoTempFilePath))
      //     fs.renameSync(photoTempFilePath, photoFilePath)
      //     break
      //   }
      //   if (retries >= 2) {
      //     console.warn(`Fetch failed: ${urlPhoto} (${sen.senprenomuse} ${sen.sennomuse})`)
      //     console.warn(response.status, response.statusText)
      //     console.warn(await response.text())
      //     if (fs.existsSync(photoFilePath)) {
      //       console.warn("  => Reusing existing image")
      //     } else {
      //       console.warn("  => Using blank image")
      //       fs.copyFileSync(missingPhotoFilePath, photoFilePath)
      //     }
      //     break
      //   }
      // }
      try {
        execSync(`wget --quiet -O ${photoTempFilename} ${urlPhoto}`, {
          cwd: photosDir,
          env: process.env,
          encoding: "utf-8",
          // stdio: ["ignore", "ignore", "pipe"],
        })
        fs.renameSync(photoTempFilePath, photoFilePath)
      } catch (error) {
        if (typeof error === "object" && error && "status" in error && error.status === 8) {
          console.error(`Unable to load photo for ${sen.senprenomuse} ${sen.sennomuse}`)
          continue
        }
        throw error
      }
    }
  }

  // Resize photos to 155x225, because some haven't exactly this size.
  for (const sen of sens) {
    const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
      sen.senprenomuse,
      "_",
    )}${slugify(sen.senmat, "_")}`
    const photoFilename = photoStem + ".jpg"
    const photoFilePath = path.join(photosDir, photoFilename)
    if (fs.existsSync(photoFilePath)) {
      if (!options.silent) {
        console.log(
          `Resizing photo ${photoStem} for ${sen.senprenomuse} ${sen.sennomuse}…`,
        )
      }

      execSync(
        `gm convert -resize 155x225! ${photoStem}.jpg ${photoStem}_155x225.jpg`,
        {
          cwd: photosDir,
        },
      )
    } else {
      if (!options.silent) {
        console.warn(`Missing photo for ${sen.senprenomuse} ${sen.sennomuse}: using blank image`)
      }
      fs.copyFileSync(missingPhotoFilePath, path.join(photosDir, `${photoStem}_155x225.jpg`))
    }
  }

  // Create a mosaic of photos.
  if (!options.silent) {
    console.log("Creating mosaic of photos…")
  }
  const photoBySenmat: { [senmat: string]: Photo } = {}
  const rowsFilenames: string[] = []
  for (
    let senIndex = 0, rowIndex = 0;
    senIndex < sens.length;
    senIndex += 25, rowIndex++
  ) {
    const row = sens.slice(senIndex, senIndex + 25)
    const photosFilenames: string[] = []
    for (const [columnIndex, sen] of row.entries()) {
      const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(
        sen.senprenomuse,
        "_",
      )}${slugify(sen.senmat, "_")}`
      const photoFilename = `${photoStem}_155x225.jpg`
      photosFilenames.push(photoFilename)
      photoBySenmat[sen.senmat] = {
        chemin: `photos_senateurs/${photoFilename}`,
        cheminMosaique: "photos_senateurs/senateurs.jpg",
        hauteur: 225,
        largeur: 155,
        xMosaique: columnIndex * 155,
        yMosaique: rowIndex * 225,
      }
    }
    const rowFilename = `row-${rowIndex}.jpg`
    execSync(`gm convert ${photosFilenames.join(" ")} +append ${rowFilename}`, {
      cwd: photosDir,
    })
    rowsFilenames.push(rowFilename)
  }
  execSync(`gm convert ${rowsFilenames.join(" ")} -append senateurs.jpg`, {
    cwd: photosDir,
  })
  for (const rowFilename of rowsFilenames) {
    fs.unlinkSync(path.join(photosDir, rowFilename))
  }

  if (!options.silent) {
    console.log("Creating JSON file containing informations on all pictures…")
  }
  const jsonFilePath = path.join(photosDir, "senateurs.json")
  fs.writeFileSync(jsonFilePath, JSON.stringify(photoBySenmat, null, 2))
}

retrievePhotosSenateurs()
  .then(() => process.exit(0))
  .catch((error) => {
    console.log(error)
    process.exit(1)
  })