akm-cli/dist/sources/website-url.js

akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cur

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
/**
 * Leaf URL and cache-path derivations shared by website source providers and
 * the website ingest pipeline.
 *
 * Keep this module independent of website-ingest and the snapshot-fetcher
 * registry. Provider registration must be safe to import from low-level source
 * resolution code without pulling the fetcher subgraph into that leaf.
 */
import { createHash } from "node:crypto";
import path from "node:path";
import { ConfigError, UsageError } from "../core/errors.js";
import { classifyNetworkHostname } from "../core/network-policy.js";
import { getRegistryIndexCacheDir } from "../core/paths.js";
import { warnOnce } from "../core/warn.js";
import { assertWebsiteRequestUrl } from "./snapshot-fetchers/host-guard.js";
export function shouldAllowPrivateWebsiteHostsForTests() {
    return process.env.BUN_TEST === "1" || process.env.NODE_ENV === "test";
}
export function shouldAllowPrivateWebsiteUrlForTests(rawUrl) {
    let hostname;
    try {
        hostname = new URL(rawUrl).hostname.toLowerCase();
    }
    catch {
        return false;
    }
    if (classifyNetworkHostname(hostname) === "public")
        return false;
    if (!shouldAllowPrivateWebsiteHostsForTests()) {
        warnOnce(`website-private-host:${hostname}`, `[akm] "${hostname}" is not a publicly routable host, but you added it as a source directly — proceeding. ` +
            "Links discovered elsewhere are still checked.");
    }
    return true;
}
export function getWebsiteCachePaths(siteUrl) {
    const key = createHash("sha256").update(normalizeSiteUrl(siteUrl)).digest("hex").slice(0, 16);
    const rootDir = path.join(getRegistryIndexCacheDir(), `website-${key}`);
    return {
        rootDir,
        stashDir: path.join(rootDir, "stash"),
        manifestPath: path.join(rootDir, "manifest.json"),
    };
}
export function validateWebsiteUrl(rawUrl, options) {
    return validateWebsiteUrlWithError(rawUrl, ConfigError, options);
}
export function validateWebsiteInputUrl(rawUrl, options) {
    return validateWebsiteUrlWithError(rawUrl, UsageError, options);
}
function validateWebsiteUrlWithError(rawUrl, ErrorType, options) {
    if (!rawUrl) {
        throw new ErrorType("Website provider requires a URL");
    }
    let parsed;
    try {
        parsed = new URL(rawUrl);
    }
    catch {
        throw new ErrorType(`Website URL is not valid: "${rawUrl}"`);
    }
    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
        throw new ErrorType(`Website URL must use http:// or https://, got "${parsed.protocol}" in "${rawUrl}"`);
    }
    if (parsed.username || parsed.password) {
        throw new ErrorType("Website URL must not contain embedded credentials");
    }
    assertWebsiteRequestUrl(parsed.toString(), ErrorType, options);
    parsed.hash = "";
    return normalizeSiteUrl(parsed.toString());
}
export function normalizeSiteUrl(rawUrl) {
    const parsed = new URL(rawUrl);
    parsed.hash = "";
    if (parsed.pathname !== "/" && parsed.pathname.endsWith("/")) {
        parsed.pathname = parsed.pathname.replace(/\/+$/, "");
    }
    return parsed.toString();
}