peertube-plugin-transposer-connector
Version:
Transposer connector is a PeerTube language tool plugin to transcribe and translate with Whisper
335 lines (292 loc) • 9.48 kB
JavaScript
const { initRegisterSettings } = require("./settings.js");
const { LanguageMetadataDAO } = require("./server/dao/LanguageMetadataDAO.js");
const {
initTransposerController,
} = require("./server/controller/transposerController.js");
const {
initLanguageMetadata,
} = require("./server/controller/languageMetadataController.js");
const {
initCaptionMetadata,
} = require("./server/controller/captionMetadataController.js");
async function register({
getRouter,
registerHook,
registerSetting,
settingsManager,
peertubeHelpers,
}) {
// init Settings
initRegisterSettings(registerSetting, peertubeHelpers);
// init DataAccessObjects
var languageMetadataDAO = new LanguageMetadataDAO(peertubeHelpers);
peertubeHelpers.logger.info("Initialized data access objects");
// init Controllers
const router = getRouter();
initTransposerController(router, peertubeHelpers, settingsManager);
initLanguageMetadata(router, languageMetadataDAO, peertubeHelpers);
initCaptionMetadata(router, peertubeHelpers);
// We also update the JSON LD context because we added a new field
// TODO: use https://www.w3.org/TR/json-ld11/#example-71-language-map-expressing-a-property-in-three-languages
// TODO: use https://docs.joinpeertube.org/contribute/plugins#federation
registerHook({
target: "filter:activity-pub.activity.context.build.result",
handler: (jsonld) => {
peertubeHelpers.logger.info(
`filter:activity-pub.activity.context.build.result`
);
var newJsonLd = jsonld.concat([
{ translatedMetadata: "https://schema.org/recordedAt" },
]);
peertubeHelpers.logger.info("newJsonLd context", newJsonLd);
return jsonld;
},
});
//Triggers when a video is updated:
registerHook({
target: "filter:activity-pub.video.json-ld.build.result",
handler: async (jsonld, { video }) => {
const videoId = video.id;
peertubeHelpers.logger.info(
`filter:activity-pub.video.json-ld.build.result with videoId ${videoId}`
);
var languageMetadata = await languageMetadataDAO.findLanguageMetadataById(
video.id
);
Object.assign(jsonld, { translatedMetadata: languageMetadata });
peertubeHelpers.logger.info("metadata Example", JSON.stringify(jsonld));
//TODO: get the videoCaptions
/*
const response = await fetch(
`https://tube.dev.displ.eu/api/v1/videos/${videoId}/captions`
);
const captions = await response.json();
peertubeHelpers.logger.info("captionsList", captions);
*/
return jsonld;
},
});
registerHook({
target: "action:api.video.uploaded",
handler: async ({ video }) => {
if (!video) {
peertubeHelpers.logger.info(
`Video is missing in action:api.video.uploaded`
);
return;
}
peertubeHelpers.logger.info(
`Transposer only triggered when update the video`
);
},
});
registerHook({
target: "action:api.video.updated",
handler: async ({ video, body }) => {
peertubeHelpers.logger.info(
"is filter:activity-pub.video.json-ld.build.result before"
);
if (!video) {
peertubeHelpers.logger.info(
`Video is missing in action:api.video.updated`
);
return;
}
if (!body.pluginData) {
peertubeHelpers.logger.info(
`pluginData is missing in action:api.video.updated`
);
return;
}
if (!body.pluginData["trigger-transposer"]) {
peertubeHelpers.logger.info(`"trigger transposer is false`);
return;
}
body.pluginData["trigger-transposer"] = false;
triggerTransposer(video);
},
});
async function triggerTransposer(video) {
const settings = await settingsManager.getSettings([
"trigger-transcription-and-translation",
]);
if (!settings["trigger-transcription-and-translation"]) {
peertubeHelpers.logger.info(
"trigger-transcription-and-translation setting is missing"
);
return;
}
const webserverUrl = peertubeHelpers.config.getWebserverUrl();
if (!webserverUrl) {
peertubeHelpers.logger.info(
`webserverUrl is missing in action:api.video.uploaded`
);
return;
}
if (!video) {
peertubeHelpers.logger.info(`Video is missing in triggerTransposer`);
return;
}
if (!webserverUrl) {
peertubeHelpers.logger.info(
`webserverUrl is missing in triggerTransposer`
);
return;
}
const { language, id } = video;
peertubeHelpers.logger.info(
`Transposerconnect with ${id}, and language ${language}`
);
if (!language) {
peertubeHelpers.logger.info(
`Video "${id}" doesn't have a language defined.`
);
}
const videoFiles = await getVideoFiles(id, peertubeHelpers);
if (videoFiles.length === 0) {
peertubeHelpers.logger.info(
`No video file found for video "${id}", skipping.`
);
return;
}
peertubeHelpers.logger.info(
`Sending transcription for this instance "${webserverUrl}" to Peertube Adpater`
);
peertubeHelpers.logger.info(
`Sending transcription request with ID "${id}" in "${language}" to Peertube Adpater`
);
if (videoFiles && videoFiles.length > 0 && videoFiles[0].url) {
var url = videoFiles[0].url; // TODO get the lowest video quality
peertubeHelpers.logger.info(`${url} this url is saved in videoFiles[0]`);
//not transcoded
if (url.includes("web-videos")) {
peertubeHelpers.logger.info(
"start trigger transcription and translation with web-videos url"
);
triggerTranscriptionAndTranslation(
video.id,
url,
language,
settings["trigger-transcription-and-translation"],
webserverUrl
);
} else if (url.includes("streaming-playlist")) {
//transcoded
peertubeHelpers.logger.info(
"start trigger transcription and translation with streaming-playlist url"
);
triggerTranscriptionAndTranslation(
video.id,
url,
language,
settings["trigger-transcription-and-translation"],
webserverUrl
);
} else {
peertubeHelpers.logger.info(
"start trigger transcription and translation without url include"
);
peertubeHelpers.logger.info("with url:", url);
triggerTranscriptionAndTranslation(
video.id,
url,
language,
settings["trigger-transcription-and-translation"],
webserverUrl
);
}
} else {
peertubeHelpers.logger.info("videoFiles does not have a valid URL.");
}
}
async function getVideoFiles(id, peertubeHelpers) {
peertubeHelpers.logger.info(`Fetching video files for video ID: ${id}`);
const videoFilesData = await peertubeHelpers.videos.getFiles(id);
if (!videoFilesData) {
peertubeHelpers.logger.info(
`No video files data returned for video ID: ${id}`
);
return [];
}
const webVideoFiles = videoFilesData.webVideo?.videoFiles || [];
const hlsVideoFiles = videoFilesData.hls?.videoFiles || [];
if (webVideoFiles.length === 0 && hlsVideoFiles.length === 0) {
peertubeHelpers.logger.info(`No video files found for video ID: ${id}`);
return [];
}
const combinedVideoFiles = webVideoFiles.concat(hlsVideoFiles);
// Sortiere die Dateien nach Größe (aufsteigend)
const sortedVideoFiles = combinedVideoFiles.sort(
({ size: sizeA }, { size: sizeB }) => sizeA - sizeB
);
peertubeHelpers.logger.info(
`Sorted video files for video ID ${id}:`,
sortedVideoFiles
);
return sortedVideoFiles;
}
/*
//TODO Make somthing when caption is deleted?
registerHook({
target: 'action:api.video-caption.deleted',
handler: videoController.captionDeleted.bind(videoController),
});
*/
function triggerTranscriptionAndTranslation(
videoId,
downloadUrl,
language,
triggerUrl,
webserverUrl
) {
if (!triggerUrl) {
return;
}
peertubeHelpers.logger.info("triggerUrl:", triggerUrl);
let body = {
videoId: videoId,
url: downloadUrl,
peertubeInstanceBaseDomain: webserverUrl,
};
if (language) {
body.language = language;
}
fetch(`${triggerUrl}/transcribe-and-translate/video`, {
method: "POST",
headers: {
Accept: "application/json",
"Content-Type": "application/json",
},
body: JSON.stringify(body),
})
.then((response) => {
if (!response.ok) {
response.json().then((json_error) => {
throw new Error(
`HTTP error! status: ${response.status} and message ${json_error.message}`
);
});
}
return response.json();
})
.then((data) => {
peertubeHelpers.logger.info(
`json result from server ${
data.success ? "successful" : "unsuccessful"
}`
);
})
.catch((error) => {
peertubeHelpers.logger.info(
`Failed to report transcription done (for translation) "${error}", skipping.`
);
});
}
}
async function unregister() {
return;
}
module.exports = {
register,
unregister,
};