@tricoteuses/senat
Version:
Handle French Sénat's open data
165 lines (164 loc) • 6.92 kB
JavaScript
import assert from "assert";
import { execSync } from "child_process";
import commandLineArgs from "command-line-args";
import fs from "fs-extra";
// import fetch from "node-fetch"
import path from "path";
// import stream from "stream"
// import util from "util"
import { senFieldsToParseInt, senFieldsToTrim } from "../types/sens";
import { checkDatabase } from "../databases";
import { parseIntFields, trimFieldsRight } from "../fields";
import { slugify } from "../strings";
const optionsDefinitions = [
{
alias: "f",
help: "fetch sénateurs' pictures instead of retrieving them from files",
name: "fetch",
type: Boolean,
},
{
alias: "s",
help: "don't log anything",
name: "silent",
type: Boolean,
},
{
defaultOption: true,
help: "directory containing Sénat open data files",
name: "dataDir",
type: String,
},
];
const options = commandLineArgs(optionsDefinitions);
// const pipeline = util.promisify(stream.pipeline)
async function retrievePhotosSenateurs() {
const dataDir = options.dataDir;
assert(dataDir, "Missing argument: data directory");
const db = await checkDatabase("sens");
const sens = (await db.any(`
SELECT *
FROM sen
WHERE etasencod = 'ACTIF'
`)).map((sen) => parseIntFields(senFieldsToParseInt, trimFieldsRight(senFieldsToTrim, sen)));
const photosDir = path.join(dataDir, "photos_senateurs");
const missingPhotoFilePath = path.resolve(__dirname, "images", "transparent_155x225.jpg");
// Download photos.
fs.ensureDirSync(photosDir);
if (options.fetch) {
for (const sen of sens) {
const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(sen.senprenomuse, "_")}${slugify(sen.senmat, "_")}`;
const photoFilename = photoStem + ".jpg";
const photoFilePath = path.join(photosDir, photoFilename);
const photoTempFilename = photoStem + "_temp.jpg";
const photoTempFilePath = path.join(photosDir, photoTempFilename);
const urlPhoto = `https://www.senat.fr/senimg/${photoFilename}`;
if (!options.silent) {
console.log(`Loading photo ${urlPhoto} for ${sen.senprenomuse} ${sen.sennomuse}…`);
}
// Fetch fails with OpenSSL error: dh key too small.
// (so does "curl").
// for (let retries = 0; retries < 3; retries++) {
// const response = await fetch(urlPhoto)
// if (response.ok) {
// await pipeline(response.body, fs.createWriteStream(photoTempFilePath))
// fs.renameSync(photoTempFilePath, photoFilePath)
// break
// }
// if (retries >= 2) {
// console.warn(`Fetch failed: ${urlPhoto} (${sen.senprenomuse} ${sen.sennomuse})`)
// console.warn(response.status, response.statusText)
// console.warn(await response.text())
// if (fs.existsSync(photoFilePath)) {
// console.warn(" => Reusing existing image")
// } else {
// console.warn(" => Using blank image")
// fs.copyFileSync(missingPhotoFilePath, photoFilePath)
// }
// break
// }
// }
try {
execSync(`wget --quiet -O ${photoTempFilename} ${urlPhoto}`, {
cwd: photosDir,
env: process.env,
encoding: "utf-8",
// stdio: ["ignore", "ignore", "pipe"],
});
fs.renameSync(photoTempFilePath, photoFilePath);
}
catch (error) {
if (typeof error === "object" && error && "status" in error && error.status === 8) {
console.error(`Unable to load photo for ${sen.senprenomuse} ${sen.sennomuse}`);
continue;
}
throw error;
}
}
}
// Resize photos to 155x225, because some haven't exactly this size.
for (const sen of sens) {
const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(sen.senprenomuse, "_")}${slugify(sen.senmat, "_")}`;
const photoFilename = photoStem + ".jpg";
const photoFilePath = path.join(photosDir, photoFilename);
if (fs.existsSync(photoFilePath)) {
if (!options.silent) {
console.log(`Resizing photo ${photoStem} for ${sen.senprenomuse} ${sen.sennomuse}…`);
}
execSync(`gm convert -resize 155x225! ${photoStem}.jpg ${photoStem}_155x225.jpg`, {
cwd: photosDir,
});
}
else {
if (!options.silent) {
console.warn(`Missing photo for ${sen.senprenomuse} ${sen.sennomuse}: using blank image`);
}
fs.copyFileSync(missingPhotoFilePath, path.join(photosDir, `${photoStem}_155x225.jpg`));
}
}
// Create a mosaic of photos.
if (!options.silent) {
console.log("Creating mosaic of photos…");
}
const photoBySenmat = {};
const rowsFilenames = [];
for (let senIndex = 0, rowIndex = 0; senIndex < sens.length; senIndex += 25, rowIndex++) {
const row = sens.slice(senIndex, senIndex + 25);
const photosFilenames = [];
for (const [columnIndex, sen] of row.entries()) {
const photoStem = `${slugify(sen.sennomuse, "_")}_${slugify(sen.senprenomuse, "_")}${slugify(sen.senmat, "_")}`;
const photoFilename = `${photoStem}_155x225.jpg`;
photosFilenames.push(photoFilename);
photoBySenmat[sen.senmat] = {
chemin: `photos_senateurs/${photoFilename}`,
cheminMosaique: "photos_senateurs/senateurs.jpg",
hauteur: 225,
largeur: 155,
xMosaique: columnIndex * 155,
yMosaique: rowIndex * 225,
};
}
const rowFilename = `row-${rowIndex}.jpg`;
execSync(`gm convert ${photosFilenames.join(" ")} +append ${rowFilename}`, {
cwd: photosDir,
});
rowsFilenames.push(rowFilename);
}
execSync(`gm convert ${rowsFilenames.join(" ")} -append senateurs.jpg`, {
cwd: photosDir,
});
for (const rowFilename of rowsFilenames) {
fs.unlinkSync(path.join(photosDir, rowFilename));
}
if (!options.silent) {
console.log("Creating JSON file containing informations on all pictures…");
}
const jsonFilePath = path.join(photosDir, "senateurs.json");
fs.writeFileSync(jsonFilePath, JSON.stringify(photoBySenmat, null, 2));
}
retrievePhotosSenateurs()
.then(() => process.exit(0))
.catch((error) => {
console.log(error);
process.exit(1);
});