pose-to-video/src/index.js

A library to convert pose estimation data from a custom binary format to a video.

const { Pose } = require("./pose-format/index.js");
const { CanvasPoseRenderer } = require("./renderers/canvas.pose-renderer.js");
const { mkdirSync, writeFileSync, rmSync } = require("fs");
const { spawn } = require("child_process");
const ffmpegPath = require("ffmpeg-static");

/**
 * Cleans and validates pose data.
 * Returns a sanitized pose or null if invalid.
 */
function cleanPoseJson(pose) {
  if (!pose || typeof pose !== "object") {
    console.warn("❌ Invalid pose: not an object");
    return null;
  }

  if (!pose.header || !pose.body) {
    console.warn("❌ Pose missing header or body");
    return null;
  }

  if (typeof pose.body.fps !== "number") {
    console.warn("❌ Pose body.fps is missing or not a number");
    return null;
  }

  const frames = [];
  for (let f = 0; f < pose.body._frames; f++) {
    const frame = pose.body.frames[f];
    let isValid = true;

    if (!Array.isArray(frame.people)) {
      console.warn(`⚠ Frame ${f}: Missing or invalid 'people' array`);
      continue;
    }

    for (const person of frame.people) {
      for (const component of pose.header.components) {
        const points = person[component.name];
        if (
          !Array.isArray(points) ||
          points.length !== component.points.length
        ) {
          isValid = false;
          break;
        }

        for (const point of points) {
          if (Object.keys(point).length !== component.format.length) {
            isValid = false;
            break;
          }

          for (const key of component.format) {
            if (!(key in point)) {
              isValid = false;
              break;
            }
            if (key !== "C" && typeof point[key] !== "number") {
              isValid = false;
              break;
            }
          }
          if (!isValid) break;
        }
        if (!isValid) break;
      }
      if (!isValid) break;
    }

    if (isValid) frames.push(frame);
  }

  console.log(
    `✅ ${frames.length}/${pose.body.frames.length} valid frames found`
  );
  pose.body.frames = frames;
  pose.body._frames = frames.length;
  return pose;
}

/**
 * Helper: Render frames and save them to disk
 */
function generateFrames(pose, renderer) {
  rmSync("frames", { recursive: true, force: true });
  mkdirSync("frames");

  console.log(`Rendering ${pose.body.frames.length} frames...`);
  let frame;
  for (let i = 0; i < pose.body.frames.length; i++) {
    frame = pose.body.frames[i];
    const img = renderer.render(frame);
    const buffer = img.toBuffer("image/png");
    const filename = `frames/frame_${String(i).padStart(5, "0")}.png`;
    writeFileSync(filename, buffer);
  }
}

/**
 * Helper: Use ffmpeg to create a video from rendered frames
 */
async function combineFramesToVideo(fps, outputPath) {
  return new Promise((resolve, reject) => {
    console.log("Combining frames into video...", ffmpegPath);
    const ffmpeg = spawn(ffmpegPath || "ffmpeg", [
      "-framerate",
      String(fps),
      "-i",
      "frames/frame_%05d.png",
      "-c:v",
      "libx264",
      "-pix_fmt",
      "yuv420p",
      outputPath,
    ]);

    ffmpeg.stdout.on("data", (d) => console.log(d.toString()));
    ffmpeg.stderr.on("data", (d) => console.error(d.toString()));
    ffmpeg.on("close", (code) => {
      console.log(`FFmpeg finished with code ${code}`);
      if (code !== 0) {
        console.error("❌ FFmpeg failed to create video");
        reject(new Error("FFmpeg failed"));
        return;
      }
      resolve();
    });
  });
}

/**
 * Core function shared by both poseToVideo and poseJsonToVideo
 */
async function processPose(pose, outputPath) {
  console.log("Cleaning pose..");
  pose = cleanPoseJson(pose);

  if (!pose) {
    console.error("Pose file is invalid!");
    return false;
  }

  const fps = pose.body.fps;
  const renderer = new CanvasPoseRenderer({ width: 640, height: 480, pose });

  generateFrames(pose, renderer);
  await combineFramesToVideo(fps, outputPath);

  return true;
}

/**
 * Converts a .pose file into a video
 */
async function poseToVideo(posePath, outputPath) {
  const pose = await Pose.fromLocal(posePath);
  return processPose(pose, outputPath);
}

/**
 * Converts a JSON pose object into a video
 */
async function poseJsonToVideo(pose, outputPath) {
  return processPose(pose, outputPath);
}

if (require.main === module) poseToVideo("./example.pose", "output.mp4");

module.exports = {
  poseToVideo,
  poseJsonToVideo,
};