akm-cli
Version:
akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cur
240 lines (239 loc) • 9.37 kB
JavaScript
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
import { fetchWithRetry } from "../../core/common.js";
const YOUTUBE_HOSTS = new Set(["www.youtube.com", "youtube.com", "m.youtube.com", "youtu.be"]);
const WATCH_HOST = "https://www.youtube.com/watch?v=";
// Captions: the WEB watch-page caption baseUrls are now Proof-of-Origin-Token
// gated (they carry `exp=xpe` and return an empty body without a `pot` token).
// The ANDROID InnerTube `player` client still returns UNGATED caption baseUrls
// that serve the transcript, so captions are sourced from there. See
// https://github.com/yt-dlp/yt-dlp/issues/13075.
const INNERTUBE_PLAYER_URL = "https://www.youtube.com/youtubei/v1/player";
// Long-stable public InnerTube key; only authorizes the player call (not auth).
const DEFAULT_INNERTUBE_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8";
const ANDROID_INNERTUBE_CLIENT = {
clientName: "ANDROID",
clientVersion: "20.10.38",
androidSdkVersion: 34,
hl: "en",
gl: "US",
};
function extractVideoId(url) {
if (!YOUTUBE_HOSTS.has(url.hostname))
return null;
if (url.hostname === "youtu.be") {
const id = url.pathname.replace(/^\/+/, "").split("/")[0]?.trim();
return id || null;
}
if (url.pathname === "/watch") {
const id = url.searchParams.get("v")?.trim();
return id || null;
}
if (url.pathname.startsWith("/shorts/")) {
const id = url.pathname.slice("/shorts/".length).split("/")[0]?.trim();
return id || null;
}
if (url.pathname.startsWith("/embed/")) {
const id = url.pathname.slice("/embed/".length).split("/")[0]?.trim();
return id || null;
}
return null;
}
function canonicalWatchUrl(videoId) {
return `${WATCH_HOST}${encodeURIComponent(videoId)}`;
}
function decodeEntities(value) {
const safeFromCodePoint = (codePoint, fallback) => {
if (!Number.isFinite(codePoint) || codePoint < 0 || codePoint > 0x10ffff)
return fallback;
try {
return String.fromCodePoint(codePoint);
}
catch {
return fallback;
}
};
return value
.replace(/&#x([0-9a-f]+);/gi, (m, hex) => safeFromCodePoint(Number.parseInt(hex, 16), m))
.replace(/&#([0-9]+);/g, (m, dec) => safeFromCodePoint(Number.parseInt(dec, 10), m))
.replace(/"/g, '"')
.replace(/'|'/g, "'")
.replace(/</g, "<")
.replace(/>/g, ">")
.replace(/&/g, "&");
}
function stripTags(value) {
return value.replace(/<[^>]+>/g, "");
}
function extractJsonObjectAfter(marker, source) {
const markerIndex = source.indexOf(marker);
if (markerIndex === -1)
return null;
const start = source.indexOf("{", markerIndex + marker.length);
if (start === -1)
return null;
let depth = 0;
let inString = false;
let escaped = false;
for (let i = start; i < source.length; i += 1) {
const char = source[i];
if (inString) {
if (escaped) {
escaped = false;
}
else if (char === "\\") {
escaped = true;
}
else if (char === '"') {
inString = false;
}
continue;
}
if (char === '"') {
inString = true;
continue;
}
if (char === "{")
depth += 1;
if (char === "}") {
depth -= 1;
if (depth === 0)
return source.slice(start, i + 1);
}
}
return null;
}
function parsePlayerResponse(html) {
const jsonText = extractJsonObjectAfter("ytInitialPlayerResponse =", html);
if (!jsonText)
return null;
try {
return JSON.parse(jsonText);
}
catch {
return null;
}
}
function chooseCaptionTrack(tracks) {
const normalized = tracks
.map((track) => ({
baseUrl: typeof track.baseUrl === "string" ? track.baseUrl : "",
languageCode: typeof track.languageCode === "string" ? track.languageCode : "",
kind: typeof track.kind === "string" ? track.kind : "",
}))
.filter((track) => track.baseUrl);
if (normalized.length === 0)
return null;
const preferred = normalized.find((track) => track.languageCode === "en" && track.kind !== "asr") ??
normalized.find((track) => track.languageCode.startsWith("en") && track.kind !== "asr") ??
normalized.find((track) => track.languageCode.startsWith("en")) ??
normalized[0];
return preferred.baseUrl;
}
async function fetchText(url, timeoutMs, signal) {
const response = await fetchWithRetry(url, {
headers: {
Accept: "text/html,application/json,application/xml,text/xml,text/plain;q=0.9,*/*;q=0.1",
"User-Agent": "akm-cli youtube fetcher",
},
signal,
}, { timeout: timeoutMs, retries: 1 });
if (!response.ok)
return null;
return response.text();
}
function parseTranscript(xml) {
// Two timedtext shapes: legacy srv1 `<text>` cues, and the format-3
// `<timedtext format="3">` `<p>` cues that the ANDROID InnerTube caption
// URLs return. Strip any nested `<s>` word tags via stripTags.
const matches = [...xml.matchAll(/<text\b[^>]*>([\s\S]*?)<\/text>/g), ...xml.matchAll(/<p\b[^>]*>([\s\S]*?)<\/p>/g)];
const texts = matches.map((match) => decodeEntities(stripTags(match[1] ?? "")).trim()).filter(Boolean);
if (texts.length === 0)
return null;
return texts.join("\n");
}
/**
* Fetch caption tracks via the ANDROID InnerTube `player` endpoint, whose
* caption baseUrls are not POT-gated. Returns [] on any failure so the caller
* can fall back to the (usually empty) watch-page tracks and degrade to a
* description-only snapshot.
*/
async function fetchInnertubeCaptionTracks(videoId, apiKey, timeoutMs, signal) {
try {
const response = await fetchWithRetry(`${INNERTUBE_PLAYER_URL}?key=${encodeURIComponent(apiKey)}`, {
method: "POST",
headers: {
"Content-Type": "application/json",
"User-Agent": "com.google.android.youtube/20.10.38 (Linux; U; Android 14) gzip",
},
body: JSON.stringify({ context: { client: ANDROID_INNERTUBE_CLIENT }, videoId }),
signal,
}, { timeout: timeoutMs, retries: 1 });
if (!response.ok)
return [];
const json = (await response.json());
return json.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
}
catch {
return [];
}
}
/** Extract the page's InnerTube API key, falling back to the stable default. */
function extractInnertubeKey(html) {
return html.match(/"INNERTUBE_API_KEY":"([^"]+)"/)?.[1] ?? DEFAULT_INNERTUBE_KEY;
}
const youtubeFetcher = {
name: "youtube-transcript",
matches(url) {
return extractVideoId(url) !== null;
},
async fetch(url, context) {
const videoId = extractVideoId(url);
if (!videoId)
return null;
const watchUrl = canonicalWatchUrl(videoId);
const watchHtml = await fetchText(watchUrl, context.timeoutMs, context.signal);
if (!watchHtml)
return null;
const playerResponse = parsePlayerResponse(watchHtml);
if (!playerResponse)
return null;
const title = typeof playerResponse.videoDetails?.title === "string" ? playerResponse.videoDetails.title.trim() : "";
const description = typeof playerResponse.videoDetails?.shortDescription === "string"
? playerResponse.videoDetails.shortDescription.trim()
: "";
const sections = [];
let hasTranscript = false;
if (description) {
sections.push("## Description", "", description);
}
// Prefer ungated ANDROID InnerTube caption tracks; fall back to the
// (now usually empty) watch-page tracks so a description-only snapshot
// still works when InnerTube is unavailable.
const apiKey = extractInnertubeKey(watchHtml);
let tracks = await fetchInnertubeCaptionTracks(videoId, apiKey, context.timeoutMs, context.signal);
if (tracks.length === 0) {
tracks = playerResponse.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
}
const captionUrl = chooseCaptionTrack(tracks);
if (captionUrl) {
const transcriptXml = await fetchText(captionUrl, context.timeoutMs, context.signal);
const transcript = transcriptXml ? parseTranscript(transcriptXml) : null;
if (transcript) {
hasTranscript = true;
sections.push("## Transcript", "", transcript);
}
}
if (sections.length === 0)
return null;
return {
url: watchUrl,
title: title || `YouTube ${videoId}`,
markdown: sections.join("\n"),
preferredName: `youtube/${videoId}`,
tags: hasTranscript ? ["youtube", "video", "transcript"] : ["youtube", "video"],
};
},
};
export default youtubeFetcher;