endfield-mcp/dist/data/sync.js

MCP Server for Arknights: Endfield (TypeScript / stdio + Streamable HTTP)

/**
 * GitHub-backed data sync for Endfield-MCP.
 *
 * Mirrors the design of PRTS-MCP's `ts/src/data/sync.ts` — the same proven
 * decision tree, cascade fallback, and atomic-write semantics. The only
 * Endfield-specific parts are the User-Agent and the absence of the
 * `upstream-<sha>` tag convention (our mirror uses plain semver tags, so
 * commitSha === tag verbatim).
 *
 * Contract:
 * - Downloads GitHub Release zip assets only when the release tag changes.
 * - Skips the upstream API call entirely when cached data is fresher than
 *   CACHE_TTL_SECONDS (avoids burning GitHub anonymous quota on every start).
 * - Falls back gracefully to cached data when the network is unavailable,
 *   and to "no data" only when neither cache nor network can provide.
 * - All writes are write-to-tmp-then-rename for crash safety.
 */
import { existsSync, statSync } from "node:fs";
import { mkdir, readFile, rename, unlink, writeFile, } from "node:fs/promises";
import { basename, dirname, isAbsolute, join, relative, resolve, sep, } from "node:path";
import AdmZip from "adm-zip";
import { parseMirrors } from "../config.js";
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
// Version here is intentionally a coarse literal (major.minor), not derived
// from SERVER_VERSION: data/ must not depend on the upper server layer, and
// a precise patch version in a UA adds no value. Bump manually on minor+.
const GITHUB_UA = "Endfield-MCP-Bot/0.3 (Arknights: Endfield fan-creation helper)";
/** Skip the upstream tag check if cached data is fresher than this (seconds). */
const CACHE_TTL_SECONDS = 3600;
// ---------------------------------------------------------------------------
// Internal helpers
// ---------------------------------------------------------------------------
function githubHeaders() {
    const headers = { "User-Agent": GITHUB_UA };
    const token = process.env["GITHUB_TOKEN"];
    if (token)
        headers["Authorization"] = `Bearer ${token}`;
    return headers;
}
/**
 * Return mirror URLs parsed from GITHUB_MIRRORS env var.
 * Delegates to config.ts:parseMirrors to keep parsing in one place.
 * Mirror URL format (ghproxy-style): `<mirror>/<original_url>`.
 */
function mirrorUrls() {
    return parseMirrors(process.env["GITHUB_MIRRORS"] ?? "");
}
/** Return `[url, mirroredUrl1, mirroredUrl2, ...]`. */
function urlCandidates(url) {
    return [url, ...mirrorUrls().map((m) => `${m}/${url}`)];
}
/**
 * fetch() wrapper that cascades through URL candidates on failure.
 *
 * - A fresh AbortSignal.timeout is created per attempt so an earlier timeout
 *   does not consume the budget for later candidates.
 * - HTTP 4xx from the direct URL propagates immediately (resource is
 *   genuinely missing — mirrors won't help).
 * - Network error or HTTP 5xx from any candidate → try the next one.
 */
async function fetchCascading(url, options, timeoutMs) {
    const candidates = urlCandidates(url);
    let lastErr = new Error("All URL candidates failed");
    for (let i = 0; i < candidates.length; i++) {
        try {
            const res = await fetch(candidates[i], {
                ...options,
                signal: AbortSignal.timeout(timeoutMs),
            });
            if (res.ok)
                return res;
            lastErr = new Error(`HTTP ${res.status}`);
            // Direct 4xx → resource does not exist; mirrors cannot help.
            if (i === 0 && res.status >= 400 && res.status < 500)
                break;
        }
        catch (err) {
            lastErr = err;
        }
    }
    throw lastErr;
}
function errorMessage(err) {
    return err instanceof Error ? err.message : String(err);
}
function cacheIsFresh(cache) {
    try {
        const ageMs = Date.now() - new Date(cache.fetchedAt).getTime();
        return ageMs < CACHE_TTL_SECONDS * 1000;
    }
    catch {
        return false;
    }
}
function releaseZipError(spec) {
    if (!existsSync(spec.localZip))
        return "zip file is missing";
    try {
        const missing = spec.validateZip?.(spec.localZip) ?? [];
        if (missing.length === 0)
            return null;
        return missing.slice(0, 10).join("; ");
    }
    catch (err) {
        return `${basename(spec.localZip)} is not a valid zip: ${errorMessage(err)}`;
    }
}
// ---------------------------------------------------------------------------
// Release-based sync
// ---------------------------------------------------------------------------
// We list releases (newest first) rather than hitting /releases/latest.
// The latter returns the single newest Release for the whole repo and assumes
// it carries every asset — which breaks once a repo ships multiple
// independent assets across different Releases (EndFieldGameData: tables zip
// in v0.2.0, story zip in v0.3.0, worldview zip in v0.4.0). Listing lets us
// pick the Release that actually contains the requested asset.
const GITHUB_RELEASES_LIST_URL = "https://api.github.com/repos/{owner}/{repo}/releases?per_page=30";
function releaseCachePath(spec) {
    return join(dirname(spec.localZip), "release_meta.json");
}
async function loadReleaseMeta(spec) {
    try {
        const text = await readFile(releaseCachePath(spec), "utf-8");
        return JSON.parse(text);
    }
    catch {
        return null;
    }
}
async function saveReleaseMeta(spec, meta) {
    const p = releaseCachePath(spec);
    await mkdir(dirname(p), { recursive: true });
    await writeFile(p, JSON.stringify(meta, null, 2), "utf-8");
}
/**
 * Fetch the release tag and asset download URL for the requested asset.
 *
 * Lists the repo's releases (newest first) and returns the first Release
 * that actually contains `spec.assetName`. We deliberately do NOT use the
 * `/releases/latest` endpoint: that returns the single newest Release for the
 * whole repo, which only carries *that* release's assets. Once a repo ships
 * several independent assets across different Releases (tables/story/worldview),
 * `/releases/latest` makes every non-latest asset look "missing" and breaks
 * their syncs. Listing + asset-name matching is robust to that layout.
 *
 * Return / throw contract (load-bearing — `syncRelease` distinguishes the two):
 *   - Returns `{tag, url}` when a Release carrying the asset is found.
 *   - Returns `null` when the API responded but NO scanned Release has the
 *     asset (genuine absence — caller must NOT retry via blind download).
 *   - THROWS on network/API failure (caller treats as "unreachable" and may
 *     fall back to blind download / cached data).
 *
 * This distinction matters because the blind-download fallback in `syncRelease`
 * constructs a `releases/latest/download/<asset>` URL — which only makes sense
 * when we genuinely couldn't reach the API, not when the API confirmed the
 * asset is absent.
 */
export async function checkLatestRelease(spec, timeoutMs = 10_000) {
    const url = GITHUB_RELEASES_LIST_URL.replace("{owner}", spec.owner).replace("{repo}", spec.repo);
    // Network/API failure propagates as a throw (distinguished from "asset not
    // found" → null below). syncRelease catches it to decide fallback behavior.
    const res = await fetchCascading(url, { headers: githubHeaders() }, timeoutMs);
    const data = (await res.json());
    // Releases arrive newest-first; take the first carrying our asset.
    // (An asset is only ever in one Release, so "first match" == "the
    // Release that published it" — which is the freshest by definition.)
    for (const release of data) {
        const asset = release.assets.find((a) => a.name === spec.assetName);
        if (asset) {
            return { tag: release.tag_name, url: asset.browser_download_url };
        }
    }
    // Defensive: if we scanned a full page (per_page releases) and still didn't
    // find the asset, the asset might live in a Release beyond this page. Warn
    // so a misconfigured dataset spec (or a mirror that grew past 30 releases)
    // doesn't fail silently as "asset not found". sync.ts deliberately doesn't
    // depend on the logger module (it returns data; startupSync logs), so this
    // uses console directly — it's a rare, forward-looking guard, not a hot path.
    if (data.length >= 30) {
        console.warn(`[endfield-mcp/sync] Scanned ${data.length} releases (full page) for asset "${spec.assetName}" without a match — if the mirror has >30 releases, the asset may be on a later page. Consider raising per_page or paginating.`);
    }
    return null;
}
/**
 * Download the release asset zip atomically, then write cache metadata.
 * Uses write-to-tmp-then-rename for crash safety.
 */
export async function downloadReleaseAsset(spec, tag, assetUrl, timeoutMs = 120_000) {
    const tmp = spec.localZip + ".tmp";
    await mkdir(dirname(spec.localZip), { recursive: true });
    try {
        const res = await fetchCascading(assetUrl, { headers: githubHeaders(), redirect: "follow" }, timeoutMs).catch((err) => {
            throw new Error(`${errorMessage(err)} downloading ${spec.assetName}`);
        });
        await writeFile(tmp, Buffer.from(await res.arrayBuffer()));
        const missing = spec.validateZip?.(tmp) ?? [];
        if (missing.length > 0) {
            throw new Error(`Downloaded ${spec.assetName} is missing required entries: ${missing.join(", ")}`);
        }
        await rename(tmp, spec.localZip);
        // Our mirror uses plain semver tags (v0.2.0) — commitSha === tag.
        await saveReleaseMeta(spec, {
            repo: `${spec.owner}/${spec.repo}`,
            branch: "releases",
            commitSha: tag,
            fetchedAt: new Date().toISOString(),
            files: [spec.assetName],
        });
    }
    catch (err) {
        try {
            await unlink(tmp);
        }
        catch {
            // best-effort cleanup
        }
        throw err;
    }
}
/**
 * Check latest GitHub Release and download the zip if the tag has changed.
 *
 * Decision tree:
 *   1. Cache fresh AND zip exists → up_to_date (skip API call).
 *   2. Network failure → offline_fallback (zip exists) / no_data (no zip).
 *   3. Tag unchanged AND zip exists → up_to_date (refresh fetchedAt).
 *   4. Tag changed or zip missing → downloadReleaseAsset → updated / fallback.
 */
export async function syncRelease(spec) {
    const dummySpec = {
        owner: spec.owner,
        repo: spec.repo,
        branch: "releases",
        files: [spec.assetName],
        localRoot: dirname(spec.localZip),
    };
    const cache = await loadReleaseMeta(spec);
    const zipError = releaseZipError(spec);
    const zipOk = zipError === null;
    if (cache !== null && zipOk && cacheIsFresh(cache)) {
        return {
            spec: dummySpec,
            status: "up_to_date",
            commitSha: cache.commitSha,
            error: null,
        };
    }
    // checkLatestRelease throws on network/API failure, returns null only when
    // the API confirmed no scanned Release carries the asset. We must treat
    // these differently: a network failure warrants a blind-download fallback
    // (the API might have the asset, we just couldn't reach it), whereas a
    // confirmed "asset not found" does NOT (the blind URL hits the latest
    // release's download path, which is exactly the wrong-release assumption
    // the listing fix removed — retrying it would 404 pointlessly).
    let latest;
    let networkFailed = false;
    try {
        latest = await checkLatestRelease(spec);
    }
    catch {
        latest = null;
        networkFailed = true;
    }
    if (latest === null) {
        if (networkFailed) {
            // Network/API unreachable — fall back to cache or blind download.
            if (zipOk) {
                return {
                    spec: dummySpec,
                    status: "offline_fallback",
                    commitSha: cache?.commitSha ?? null,
                    error: "Network unavailable",
                };
            }
            // No zip and API unreachable — attempt blind download via the
            // releases/latest/download/ shortcut (no GitHub API call; ghproxy and
            // similar mirrors support this URL pattern). This is best-effort: the
            // latest release may not carry this asset (the very case the listing
            // fix handles when the API IS reachable), but with the API down it's
            // our only way to bootstrap a first download through a mirror.
            if (mirrorUrls().length > 0) {
                const blindUrl = `https://github.com/${spec.owner}/${spec.repo}/releases/latest/download/${spec.assetName}`;
                try {
                    await downloadReleaseAsset(spec, "unknown", blindUrl);
                    return {
                        spec: dummySpec,
                        status: "updated",
                        commitSha: "unknown",
                        error: null,
                    };
                }
                catch (err) {
                    return {
                        spec: dummySpec,
                        status: "no_data",
                        commitSha: null,
                        error: errorMessage(err),
                    };
                }
            }
            const error = existsSync(spec.localZip) && zipError
                ? `Network unavailable and no cached zip; cached zip invalid: ${zipError}`
                : "Network unavailable and no cached zip";
            return { spec: dummySpec, status: "no_data", commitSha: null, error };
        }
        // API reachable but no scanned Release carries this asset. Do NOT attempt
        // the blind download (it would hit the latest release's download path —
        // the exact wrong-release assumption the listing fix removes; the API just
        // confirmed the asset is absent). This usually means a misconfigured
        // dataset spec (wrong assetName) or a mirror that hasn't published the
        // asset yet. Fall back to cache if present, else no_data.
        if (zipOk) {
            return {
                spec: dummySpec,
                status: "offline_fallback",
                commitSha: cache?.commitSha ?? null,
                error: `Asset "${spec.assetName}" not found in any scanned Release`,
            };
        }
        return {
            spec: dummySpec,
            status: "no_data",
            commitSha: null,
            error: `Asset "${spec.assetName}" not found in any scanned Release and no cached zip`,
        };
    }
    const commitSha = latest.tag;
    if (cache !== null && cache.commitSha === commitSha && zipOk) {
        await saveReleaseMeta(spec, {
            ...cache,
            fetchedAt: new Date().toISOString(),
        });
        return { spec: dummySpec, status: "up_to_date", commitSha, error: null };
    }
    try {
        await downloadReleaseAsset(spec, latest.tag, latest.url);
        return { spec: dummySpec, status: "updated", commitSha, error: null };
    }
    catch (err) {
        const error = errorMessage(err);
        return zipOk
            ? {
                spec: dummySpec,
                status: "offline_fallback",
                commitSha: cache?.commitSha ?? null,
                error,
            }
            : { spec: dummySpec, status: "no_data", commitSha: null, error };
    }
}
// ---------------------------------------------------------------------------
// Release-archive sync (download + extract)
// ---------------------------------------------------------------------------
function archiveFilesPresent(spec) {
    return spec.requiredFiles.every((f) => {
        const p = join(spec.localRoot, f);
        return existsSync(p) && statSync(p).isFile();
    });
}
function archiveMissingFiles(spec) {
    return spec.requiredFiles.filter((f) => {
        const p = join(spec.localRoot, f);
        return !existsSync(p) || !statSync(p).isFile();
    });
}
function validateArchiveZip(zipPath, requiredFiles) {
    try {
        const zip = new AdmZip(zipPath);
        const entries = new Set(zip
            .getEntries()
            .filter((entry) => !entry.isDirectory)
            .map((entry) => entry.entryName));
        return requiredFiles.filter((file) => !entries.has(file));
    }
    catch (err) {
        return [`${basename(zipPath)} is not a valid zip: ${errorMessage(err)}`];
    }
}
async function safeExtractZip(zipPath, localRoot) {
    const root = resolve(localRoot);
    const zip = new AdmZip(zipPath);
    const tmpPaths = [];
    try {
        for (const entry of zip.getEntries()) {
            if (entry.isDirectory)
                continue;
            const dest = resolve(localRoot, entry.entryName);
            const rel = relative(root, dest);
            if (rel === "" ||
                rel === ".." ||
                rel.startsWith(`..${sep}`) ||
                isAbsolute(rel)) {
                throw new Error(`Unsafe zip member path: ${entry.entryName}`);
            }
            await mkdir(dirname(dest), { recursive: true });
            const tmp = `${dest}.tmp`;
            await writeFile(tmp, entry.getData());
            tmpPaths.push(tmp);
            await rename(tmp, dest);
            tmpPaths.pop();
        }
    }
    catch (err) {
        for (const tmp of tmpPaths) {
            try {
                await unlink(tmp);
            }
            catch {
                // best-effort cleanup
            }
        }
        throw err;
    }
}
/**
 * Download a GitHub Release zip asset and extract it into localRoot.
 *
 * Keeps the data distribution path aligned with PRTS-MCP's pattern
 * (release zip → local extraction → DirectoryStore reads) while preserving
 * whatever on-disk layout the Endfield mirror ships.
 */
export async function syncReleaseArchive(spec) {
    const releaseResult = await syncRelease({
        owner: spec.owner,
        repo: spec.repo,
        assetName: spec.assetName,
        localZip: spec.localZip,
        validateZip: (zipPath) => validateArchiveZip(zipPath, spec.requiredFiles),
    });
    const dummySpec = {
        owner: spec.owner,
        repo: spec.repo,
        branch: "releases",
        files: spec.requiredFiles,
        localRoot: spec.localRoot,
    };
    const filesOk = archiveFilesPresent(spec);
    if (releaseResult.status === "no_data") {
        return filesOk
            ? {
                spec: dummySpec,
                status: "offline_fallback",
                commitSha: releaseResult.commitSha,
                error: releaseResult.error,
            }
            : {
                spec: dummySpec,
                status: "no_data",
                commitSha: releaseResult.commitSha,
                error: releaseResult.error,
            };
    }
    const shouldExtract = releaseResult.status === "updated" || !filesOk;
    if (shouldExtract) {
        try {
            await safeExtractZip(spec.localZip, spec.localRoot);
        }
        catch (err) {
            const error = errorMessage(err);
            return archiveFilesPresent(spec)
                ? {
                    spec: dummySpec,
                    status: "offline_fallback",
                    commitSha: releaseResult.commitSha,
                    error,
                }
                : {
                    spec: dummySpec,
                    status: "no_data",
                    commitSha: releaseResult.commitSha,
                    error,
                };
        }
        const missing = archiveMissingFiles(spec);
        if (missing.length > 0) {
            const error = `Archive extraction missing required files: ${missing.slice(0, 10).join("; ")}`;
            return archiveFilesPresent(spec)
                ? {
                    spec: dummySpec,
                    status: "offline_fallback",
                    commitSha: releaseResult.commitSha,
                    error,
                }
                : {
                    spec: dummySpec,
                    status: "no_data",
                    commitSha: releaseResult.commitSha,
                    error,
                };
        }
    }
    return {
        spec: dummySpec,
        status: releaseResult.status,
        commitSha: releaseResult.commitSha,
        error: releaseResult.error,
    };
}
//# sourceMappingURL=sync.js.map