@tricoteuses/senat
Version:
Handle French Sénat's open data
173 lines (172 loc) • 7.33 kB
JavaScript
import { AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER } from "../loaders";
import { epochToParisDateTime } from "../utils/date";
import { SENAT_DATAS_ROOT } from "./config";
import { fetchText } from "./search";
import fs from "fs-extra";
import fsp from "fs/promises";
import path from "path";
export async function processOneReunionMatch(args) {
const { agenda, best, baseDir, dataDir, session, options, writeIfChanged, lastByVideo, getAgendaSegmentTimecodes, buildSenatVodMasterM3u8FromNvs, } = args;
const reunionUid = agenda.uid;
let dataTxt;
let finalTxt;
try {
dataTxt = await fsp.readFile(path.join(baseDir, "data.nvs"), "utf-8");
finalTxt = await fsp.readFile(path.join(baseDir, "finalplayer.nvs"), "utf-8");
}
catch (e) {
console.warn(`[skip] Missing NVS files for reunion ${reunionUid}`);
return;
}
const master = buildSenatVodMasterM3u8FromNvs(dataTxt);
if (!master) {
console.warn(`[warn] Cannot build m3u8 for reunion ${reunionUid}`);
return;
}
const agendaJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${agenda.uid}.json`);
// Ensure it exists first.
if (!(await fs.pathExists(agendaJsonPath))) {
console.warn(`[warn] agenda file not found: ${agendaJsonPath}`);
return;
}
let timecodeDebutVideo = null;
let timecodeFinVideo = null;
const agendaKey = agenda.titre || agenda.objet || "";
const seg = getAgendaSegmentTimecodes(dataTxt, finalTxt, agendaKey);
if (seg) {
timecodeDebutVideo = seg.start;
timecodeFinVideo = null; // keep open by default
}
// 1) If we have a start timecode, close the previous agenda for this SAME master
if (timecodeDebutVideo != null) {
const prev = lastByVideo.get(master);
if (prev && prev.agendaJsonPath !== agendaJsonPath) {
// micro-safety: do not close with an earlier timecode
if (timecodeDebutVideo <= prev.start) {
console.warn(`[warn] timecode order inversion on same video: ` +
`prev=${prev.agendaUid}(${prev.start}s) -> cur=${agenda.uid}(${timecodeDebutVideo}s). ` +
`Skip closing prev to avoid negative segment.`);
}
else {
await patchAgendaTimecodeFin({
agendaJsonPath: prev.agendaJsonPath,
timecodeFinVideo: timecodeDebutVideo,
writeIfChanged,
});
}
}
lastByVideo.set(master, { agendaUid: agenda.uid, agendaJsonPath, start: timecodeDebutVideo });
}
// 2) Update current agenda JSON with urlVideo (+ start/end if any)
const raw = await fsp.readFile(agendaJsonPath, "utf-8");
let obj;
try {
obj = JSON.parse(raw);
}
catch (e) {
console.warn(`[warn] invalid JSON in ${agendaJsonPath}`);
return;
}
const next = { ...obj, urlVideo: master, startTime: agenda.startTime, urlPageVideo: best?.pageUrl };
if (timecodeDebutVideo != null) {
next.timecodeDebutVideo = timecodeDebutVideo;
if (timecodeFinVideo != null)
next.timecodeFinVideo = timecodeFinVideo;
else
delete next.timecodeFinVideo;
}
await writeIfChanged(agendaJsonPath, JSON.stringify(next, null, 2));
if (!options["silent"]) {
console.log(`[write] ${agenda.uid} urlVideo ← ${master}` +
(timecodeDebutVideo != null ? ` (timecodeDebutVideo ← ${timecodeDebutVideo}s)` : ""));
}
}
export async function processBisIfNeeded(args) {
const { agenda, secondBest, ctx, skipDownload, options, lastByVideo, writeIfChanged, processOneReunionMatch, getAgendaSegmentTimecodes, buildSenatVodMasterM3u8FromNvs, } = args;
if (skipDownload)
return;
if (!secondBest)
return;
const bisUid = `${agenda.uid}Bis`;
const agendaBis = {
...agenda,
uid: bisUid,
titre: secondBest.vtitle ?? agenda.titre,
startTime: secondBest.epoch
? (epochToParisDateTime(secondBest.epoch)?.startTime ?? agenda.startTime)
: agenda.startTime,
};
const baseDirBis = path.join(path.dirname(ctx.baseDir), bisUid);
await fs.ensureDir(baseDirBis);
await ensureBisAgendaJson({ agenda, agendaBis, dataDir: ctx.dataDir, session: ctx.session, options });
await downloadNvsForMatch(secondBest, baseDirBis);
await processOneReunionMatch({
agenda: agendaBis,
best: secondBest,
baseDir: baseDirBis,
dataDir: ctx.dataDir,
session: ctx.session,
options,
writeIfChanged,
lastByVideo,
getAgendaSegmentTimecodes,
buildSenatVodMasterM3u8FromNvs,
});
}
async function ensureBisAgendaJson(args) {
const { agenda, agendaBis, dataDir, session, options } = args;
const agendaJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${agenda.uid}.json`);
const agendaBisJsonPath = path.join(dataDir, AGENDA_FOLDER, DATA_TRANSFORMED_FOLDER, String(session), `${agendaBis.uid}.json`);
if (!(await fs.pathExists(agendaJsonPath))) {
console.warn(`[bis] original agenda json not found to clone: ${agendaJsonPath}`);
return;
}
try {
const raw = await fsp.readFile(agendaJsonPath, "utf-8");
const obj = JSON.parse(raw);
if (!obj || typeof obj !== "object" || Array.isArray(obj)) {
console.warn(`[bis] cannot clone agenda json: expected object in ${agendaJsonPath}, got ${Array.isArray(obj) ? "array" : typeof obj}`);
return;
}
await writeIfChanged(agendaBisJsonPath, JSON.stringify(agendaBis, null, 2));
console.log(`[bis] created agenda json ${agendaBis.uid}.json from ${agenda.uid}.json`);
}
catch (e) {
console.warn(`[bis] cannot clone agenda json ${agenda.uid}.json -> ${agendaBis.uid}.json:`, e?.message);
}
}
async function downloadNvsForMatch(best, baseDir) {
const dataUrl = `${SENAT_DATAS_ROOT}/${best.id}_${best.hash}/content/data.nvs`;
const finalUrl = `${SENAT_DATAS_ROOT}/${best.id}_${best.hash}/content/finalplayer.nvs`;
const dataTxt = await fetchText(dataUrl);
const finalTxt = await fetchText(finalUrl);
if (dataTxt)
await fsp.writeFile(path.join(baseDir, "data.nvs"), dataTxt, "utf-8");
if (finalTxt)
await fsp.writeFile(path.join(baseDir, "finalplayer.nvs"), finalTxt, "utf-8");
}
export async function writeIfChanged(p, content) {
const exists = await fs.pathExists(p);
if (exists) {
const old = await fsp.readFile(p, "utf-8");
if (old === content)
return;
}
await fsp.writeFile(p, content, "utf-8");
}
async function patchAgendaTimecodeFin(args) {
const { agendaJsonPath, timecodeFinVideo, writeIfChanged } = args;
if (!(await fs.pathExists(agendaJsonPath)))
return;
const raw = await fsp.readFile(agendaJsonPath, "utf-8");
let obj;
try {
obj = JSON.parse(raw);
}
catch {
console.warn(`[warn] invalid JSON in ${agendaJsonPath}`);
return;
}
const next = { ...obj, timecodeFinVideo };
await writeIfChanged(agendaJsonPath, JSON.stringify(next, null, 2));
}