peertube-plugin-transposer-connector/main.js

Transposer connector is a PeerTube language tool plugin to transcribe and translate with Whisper

const { initRegisterSettings } = require("./settings.js");

const { LanguageMetadataDAO } = require("./server/dao/LanguageMetadataDAO.js");

const {
  initTransposerController,
} = require("./server/controller/transposerController.js");
const {
  initLanguageMetadata,
} = require("./server/controller/languageMetadataController.js");
const {
  initCaptionMetadata,
} = require("./server/controller/captionMetadataController.js");

async function register({
  getRouter,
  registerHook,
  registerSetting,
  settingsManager,
  peertubeHelpers,
}) {
  peertubeHelpers.logger.info(
    `[transposer-connector] Main: Starting plugin registration and initialization`
  );

  // init Settings
  peertubeHelpers.logger.info(
    `[transposer-connector] Main: Initializing plugin settings`
  );
  initRegisterSettings(registerSetting, peertubeHelpers);

  // init DataAccessObjects
  peertubeHelpers.logger.info(
    `[transposer-connector] Main: Initializing data access objects`
  );
  var languageMetadataDAO = new LanguageMetadataDAO(peertubeHelpers);
  peertubeHelpers.logger.info(
    `[transposer-connector] Main: Successfully initialized LanguageMetadataDAO`
  );

  // init Controllers
  peertubeHelpers.logger.info(
    `[transposer-connector] Main: Initializing API controllers`
  );
  const router = getRouter();
  initTransposerController(router, peertubeHelpers, settingsManager);
  peertubeHelpers.logger.info(
    `[transposer-connector] Main: TransposerController initialized`
  );

  initLanguageMetadata(router, languageMetadataDAO, peertubeHelpers);
  peertubeHelpers.logger.info(
    `[transposer-connector] Main: LanguageMetadataController initialized`
  );

  initCaptionMetadata(router, peertubeHelpers);
  peertubeHelpers.logger.info(
    `[transposer-connector] Main: CaptionMetadataController initialized`
  );

  // We also update the JSON LD context because we added a new field
  // TODO: use https://www.w3.org/TR/json-ld11/#example-71-language-map-expressing-a-property-in-three-languages
  // TODO: use https://docs.joinpeertube.org/contribute/plugins#federation

  peertubeHelpers.logger.info(
    `[transposer-connector] Main: Registering ActivityPub context build hook`
  );
  registerHook({
    target: "filter:activity-pub.activity.context.build.result",
    handler: (jsonld) => {
      peertubeHelpers.logger.info(
        `[transposer-connector] Main: ActivityPub context build hook triggered`
      );

      var newJsonLd = jsonld.concat([
        { translatedMetadata: "https://schema.org/recordedAt" },
      ]);
      peertubeHelpers.logger.info(
        `[transposer-connector] Main: Added translatedMetadata to ActivityPub context`
      );
      return jsonld;
    },
  });

  //Triggers when a video is updated:
  peertubeHelpers.logger.info(
    `[transposer-connector] Main: Registering ActivityPub video JSON-LD build hook`
  );
  registerHook({
    target: "filter:activity-pub.video.json-ld.build.result",
    handler: async (jsonld, { video }) => {
      const videoId = video.id;
      peertubeHelpers.logger.info(
        `[transposer-connector] Main: ActivityPub video JSON-LD build hook triggered for video ${videoId}`
      );

      var languageMetadata = await languageMetadataDAO.findLanguageMetadataById(
        video.id
      );

      Object.assign(jsonld, { translatedMetadata: languageMetadata });

      peertubeHelpers.logger.info(
        `[transposer-connector] Main: Added language metadata to ActivityPub JSON-LD for video ${videoId}`
      );

      //TODO: get the videoCaptions
      /*       
      const response = await fetch(
        `https://tube.dev.displ.eu/api/v1/videos/${videoId}/captions`
      );
      const captions = await response.json();
      peertubeHelpers.logger.info("captionsList", captions); 
      */

      return jsonld;
    },
  });

  peertubeHelpers.logger.info(
    `[transposer-connector] Main: Registering video uploaded hook`
  );
  registerHook({
    target: "action:api.video.uploaded",
    handler: async ({ video }) => {
      if (!video) {
        peertubeHelpers.logger.warn(
          `[transposer-connector] Main: Video missing in uploaded hook`
        );

        return;
      }
      peertubeHelpers.logger.info(
        `[transposer-connector] Main: Video ${video.id} uploaded - transposer only triggered on video update`
      );
    },
  });

  peertubeHelpers.logger.info(
    `[transposer-connector] Main: Registering video updated hook`
  );

  registerHook({
    target: "action:api.video.updated",
    handler: async ({ video, body }) => {
      peertubeHelpers.logger.info(
        `[transposer-connector] Main: Video updated hook triggered`
      );

      if (!video) {
        peertubeHelpers.logger.warn(
          `[transposer-connector] Main: Video missing in updated hook`
        );
        return;
      }

      if (!body.pluginData) {
        peertubeHelpers.logger.info(
          `[transposer-connector] Main: No pluginData in video ${video.id} update - skipping transposer trigger`
        );
        return;
      }

      if (!body.pluginData["trigger-transposer"]) {
        peertubeHelpers.logger.info(
          `[transposer-connector] Main: trigger-transposer flag is false for video ${video.id} - skipping`
        );
        return;
      }

      peertubeHelpers.logger.info(
        `[transposer-connector] Main: Triggering transposer for video ${video.id} via updated hook`
      );
      body.pluginData["trigger-transposer"] = false;
      triggerTransposer(video);
    },
  });

  async function triggerTransposer(video) {
    peertubeHelpers.logger.info(
      `[transposer-connector] TriggerTransposer: Starting transposer trigger for video ${video.id}`
    );

    const settings = await settingsManager.getSettings([
      "trigger-transcription-and-translation",
    ]);

    if (!settings["trigger-transcription-and-translation"]) {
      peertubeHelpers.logger.warn(
        `[transposer-connector] TriggerTransposer: Missing trigger-transcription-and-translation setting - aborting`
      );
      return;
    }

    peertubeHelpers.logger.info(
      `[transposer-connector] TriggerTransposer: Found trigger-transcription-and-translation setting: ${settings["trigger-transcription-and-translation"]}`
    );

    const webserverUrl = peertubeHelpers.config.getWebserverUrl();
    if (!webserverUrl) {
      peertubeHelpers.logger.error(
        `[transposer-connector] TriggerTransposer: Missing webserver URL configuration - cannot proceed`
      );
      return;
    }

    if (!video) {
      peertubeHelpers.logger.error(
        `[transposer-connector] TriggerTransposer: Video object is missing - cannot proceed`
      );
      return;
    }

    const { language, id } = video;
    peertubeHelpers.logger.info(
      `[transposer-connector] TriggerTransposer: Processing video ${id} with language ${
        language || "undefined"
      }`
    );

    if (!language) {
      peertubeHelpers.logger.warn(
        `[transposer-connector] TriggerTransposer: Video ${id} has no language defined - proceeding without language constraint`
      );
    }

    peertubeHelpers.logger.info(
      `[transposer-connector] TriggerTransposer: Fetching video files for video ${id}`
    );

    const videoFiles = await getVideoFiles(id, peertubeHelpers);
    if (videoFiles.length === 0) {
      peertubeHelpers.logger.warn(
        `[transposer-connector] TriggerTransposer: No video files found for video ${id} - aborting transcription`
      );
      return;
    }

    peertubeHelpers.logger.info(
      `[transposer-connector] TriggerTransposer: Found ${videoFiles.length} video file(s) for video ${id}`
    );

    peertubeHelpers.logger.info(
      `[transposer-connector] TriggerTransposer: Sending transcription request for instance ${webserverUrl} to PeerTube Adapter`
    );

    if (videoFiles && videoFiles.length > 0 && videoFiles[0].url) {
      var url = videoFiles[0].url; // TODO get the lowest video quality
      peertubeHelpers.logger.info(
        `[transposer-connector] TriggerTransposer: Selected video file URL: ${url}`
      );

      //not transcoded
      if (url.includes("web-videos")) {
        peertubeHelpers.logger.info(
          `[transposer-connector] TriggerTransposer: Detected web-videos URL format - triggering transcription and translation`
        );
        triggerTranscriptionAndTranslation(
          video.id,
          url,
          language,
          settings["trigger-transcription-and-translation"],
          webserverUrl
        );
      } else if (url.includes("streaming-playlist")) {
        //transcoded
        peertubeHelpers.logger.info(
          `[transposer-connector] TriggerTransposer: Detected streaming-playlist URL format - triggering transcription and translation`
        );
        triggerTranscriptionAndTranslation(
          video.id,
          url,
          language,
          settings["trigger-transcription-and-translation"],
          webserverUrl
        );
      } else {
        peertubeHelpers.logger.info(
          `[transposer-connector] TriggerTransposer: Unknown URL format detected - proceeding with transcription and translation`
        );
        peertubeHelpers.logger.info(
          `[transposer-connector] TriggerTransposer: URL: ${url}`
        );
        triggerTranscriptionAndTranslation(
          video.id,
          url,
          language,
          settings["trigger-transcription-and-translation"],
          webserverUrl
        );
      }
    } else {
      peertubeHelpers.logger.error(
        `[transposer-connector] TriggerTransposer: Video files array is empty or first file has no valid URL - cannot proceed`
      );
      return;
    }

    peertubeHelpers.logger.info(
      `[transposer-connector] TriggerTransposer: Completed transposer trigger process for video ${video.id}`
    );
  }

  async function getVideoFiles(id, peertubeHelpers) {
    peertubeHelpers.logger.info(
      `[transposer-connector] GetVideoFiles: Starting video file retrieval for video ID ${id}`
    );

    const videoFilesData = await peertubeHelpers.videos.getFiles(id);
    if (!videoFilesData) {
      peertubeHelpers.logger.warn(
        `[transposer-connector] GetVideoFiles: No video files data returned from API for video ID ${id}`
      );
      return [];
    }

    peertubeHelpers.logger.info(
      `[transposer-connector] GetVideoFiles: Successfully retrieved video files data for video ID ${id}`
    );

    const webVideoFiles = videoFilesData.webVideo?.videoFiles || [];
    const hlsVideoFiles = videoFilesData.hls?.videoFiles || [];

    peertubeHelpers.logger.info(
      `[transposer-connector] GetVideoFiles: Found ${webVideoFiles.length} web video files and ${hlsVideoFiles.length} HLS video files for video ID ${id}`
    );

    if (webVideoFiles.length === 0 && hlsVideoFiles.length === 0) {
      peertubeHelpers.logger.warn(
        `[transposer-connector] GetVideoFiles: No video files found in either web or HLS format for video ID ${id}`
      );
      return [];
    }

    const combinedVideoFiles = webVideoFiles.concat(hlsVideoFiles);
    peertubeHelpers.logger.info(
      `[transposer-connector] GetVideoFiles: Combined total of ${combinedVideoFiles.length} video files for video ID ${id}`
    );

    // Sortiere die Dateien nach Größe (aufsteigend)
    const sortedVideoFiles = combinedVideoFiles.sort(
      ({ size: sizeA }, { size: sizeB }) => sizeA - sizeB
    );

    peertubeHelpers.logger.info(
      `[transposer-connector] GetVideoFiles: Sorted video files by size (ascending) for video ID ${id} - smallest file size: ${
        sortedVideoFiles[0]?.size || "unknown"
      } bytes`
    );

    peertubeHelpers.logger.info(
      `[transposer-connector] GetVideoFiles: Successfully completed video file retrieval for video ID ${id} - returning ${sortedVideoFiles.length} files`
    );

    return sortedVideoFiles;
  }

  function triggerTranscriptionAndTranslation(
    videoId,
    downloadUrl,
    language,
    triggerUrl,
    webserverUrl
  ) {
    peertubeHelpers.logger.info(
      `[transposer-connector] TriggerTranscriptionAndTranslation: Starting transcription and translation trigger for video ${videoId}`
    );

    if (!triggerUrl) {
      peertubeHelpers.logger.error(
        `[transposer-connector] TriggerTranscriptionAndTranslation: Missing trigger URL - cannot proceed with video ${videoId}`
      );
      return;
    }

    peertubeHelpers.logger.info(
      `[transposer-connector] TriggerTranscriptionAndTranslation: Using trigger URL: ${triggerUrl}`
    );

    let body = {
      videoId: videoId,
      url: downloadUrl,
      peertubeInstanceBaseDomain: webserverUrl,
    };

    if (language) {
      body.language = language;
      peertubeHelpers.logger.info(
        `[transposer-connector] TriggerTranscriptionAndTranslation: Including language ${language} in request for video ${videoId}`
      );
    } else {
      peertubeHelpers.logger.info(
        `[transposer-connector] TriggerTranscriptionAndTranslation: No language specified for video ${videoId} - proceeding without language constraint`
      );
    }

    peertubeHelpers.logger.info(
      `[transposer-connector] TriggerTranscriptionAndTranslation: Sending POST request to ${triggerUrl}/transcribe-and-translate/video for video ${videoId}`
    );

    peertubeHelpers.logger.info(
      `[transposer-connector] TriggerTranscriptionAndTranslation: Request payload - videoId: ${videoId}, url: ${downloadUrl}, instance: ${webserverUrl}`
    );

    fetch(`${triggerUrl}/transcribe-and-translate/video`, {
      method: "POST",
      headers: {
        Accept: "application/json",
        "Content-Type": "application/json",
      },
      body: JSON.stringify(body),
    })
      .then((response) => {
        peertubeHelpers.logger.info(
          `[transposer-connector] TriggerTranscriptionAndTranslation: Received response with status ${response.status} for video ${videoId}`
        );

        if (!response.ok) {
          peertubeHelpers.logger.error(
            `[transposer-connector] TriggerTranscriptionAndTranslation: HTTP error response ${response.status} for video ${videoId}`
          );

          response
            .json()
            .then((json_error) => {
              peertubeHelpers.logger.error(
                `[transposer-connector] TriggerTranscriptionAndTranslation: Error details for video ${videoId}: ${
                  json_error.message || "No error message provided"
                }`
              );
              throw new Error(
                `HTTP error! status: ${response.status} and message ${json_error.message}`
              );
            })
            .catch((parseError) => {
              peertubeHelpers.logger.error(
                `[transposer-connector] TriggerTranscriptionAndTranslation: Failed to parse error response for video ${videoId}: ${parseError.message}`
              );
              throw new Error(`HTTP error! status: ${response.status}`);
            });
        }

        return response.json();
      })
      .then((data) => {
        const status = data.success ? "successful" : "unsuccessful";
        peertubeHelpers.logger.info(
          `[transposer-connector] TriggerTranscriptionAndTranslation: Transcription and translation request ${status} for video ${videoId}`
        );

        if (data.message) {
          peertubeHelpers.logger.info(
            `[transposer-connector] TriggerTranscriptionAndTranslation: Server response message for video ${videoId}: ${data.message}`
          );
        }

        peertubeHelpers.logger.info(
          `[transposer-connector] TriggerTranscriptionAndTranslation: Successfully completed request for video ${videoId}`
        );
      })
      .catch((error) => {
        peertubeHelpers.logger.error(
          `[transposer-connector] TriggerTranscriptionAndTranslation: Failed to trigger transcription and translation for video ${videoId}: ${error.message}`
        );

        peertubeHelpers.logger.error(
          `[transposer-connector] TriggerTranscriptionAndTranslation: Error details - URL: ${triggerUrl}, Video: ${videoId}, Error: ${
            error.stack || error.message
          }`
        );
      });
  }
}

async function unregister() {
  return;
}

module.exports = {
  register,
  unregister,
};