mcp-wayback-machine
Version:
MCP server and CLI tool for interacting with the Wayback Machine without API keys
272 lines (270 loc) • 8.88 kB
JavaScript
/**
* HTTP request caching with in-memory and pluggable persistent backends.
* Stores serialised responses (status, headers, body) with TTL-based expiry.
* Supports per-endpoint TTL via URL pattern matching.
*
* The default disk cache lives under the user's cache directory (XDG_CACHE_HOME
* or ~/.cache on Linux/macOS, %LOCALAPPDATA% on Windows) with 0700 / 0600
* permissions so it cannot be poisoned by other users on a shared host.
*
* Alternative backends (e.g. Cloudflare KV) implement the CacheBackend interface
* and are passed via the CachingFetcher constructor.
*/
import { join } from "node:path";
import { homedir } from "node:os";
import * as z from "zod";
import { fetchWithTimeout } from "./http.js";
export const CachedResponse = z.object({
status: z.number(),
statusText: z.string().trim(),
headers: z.record(z.string(), z.string().trim()),
body: z.string().trim(),
expiry: z.number(),
});
/**
* TTL durations in milliseconds
*/
export const TTL = {
/**
* Archived snapshot content — immutable once captured
*/
SNAPSHOT: 24 * 60 * 60 * 1000,
/**
* Availability API — snapshots don't change often
*/
AVAILABILITY: 60 * 60 * 1000,
/**
* CDX search — snapshot list grows but never mutates
*/
CDX_SEARCH: 60 * 60 * 1000,
/**
* Sparkline capture statistics — grows, never mutates
*/
SPARKLINE: 60 * 60 * 1000,
/**
* Save (POST) — idempotent per URL
*/
SAVE: 30 * 60 * 1000,
/**
* Save status polling — changes during active jobs
*/
SAVE_STATUS: 30 * 1000,
};
/**
* Default cache directory — per-user, not a shared world-readable tmp path.
* Honours $XDG_CACHE_HOME on Linux when set; falls back to ~/.cache otherwise.
* On Windows, %LOCALAPPDATA% is preferred when present.
*/
function defaultCacheDir() {
const xdg = process.env.XDG_CACHE_HOME;
if (xdg !== undefined && xdg !== "") {
return join(xdg, "mcp-wayback-machine");
}
const localAppData = process.env.LOCALAPPDATA;
if (localAppData !== undefined && localAppData !== "") {
return join(localAppData, "mcp-wayback-machine", "cache");
}
return join(homedir(), ".cache", "mcp-wayback-machine");
}
/**
* Disk-backed cache backend using the filesystem.
* Uses restrictive permissions (0700 dir, 0600 files) to prevent cache poisoning.
*/
export class DiskCacheBackend {
dir;
constructor(dir) {
this.dir = dir;
}
async get(key) {
try {
const { readFile } = await import("node:fs/promises");
const { join } = await import("node:path");
const filePath = join(this.dir, `${key}.json`);
const data = await readFile(filePath, "utf-8");
const parsed = JSON.parse(data);
return CachedResponse.parse(parsed);
}
catch {
return undefined;
}
}
async set(key, entry) {
try {
const { mkdir, writeFile } = await import("node:fs/promises");
const { join } = await import("node:path");
await mkdir(this.dir, { recursive: true, mode: 0o700 });
const filePath = join(this.dir, `${key}.json`);
await writeFile(filePath, JSON.stringify(entry), {
encoding: "utf-8",
mode: 0o600,
});
}
catch {
// Disk cache write failure is non-fatal
}
}
async delete(key) {
try {
const { unlink } = await import("node:fs/promises");
const { join } = await import("node:path");
await unlink(join(this.dir, `${key}.json`));
}
catch {
// File may not exist
}
}
async clear() {
try {
const { readdir, unlink } = await import("node:fs/promises");
const { join } = await import("node:path");
const files = await readdir(this.dir);
await Promise.all(files.map((file) => unlink(join(this.dir, file)).catch(() => undefined)));
}
catch {
// Directory may not exist
}
}
}
const DEFAULT_CONFIG = {
ttl: TTL.AVAILABILITY,
backend: new DiskCacheBackend(defaultCacheDir()),
};
/**
* Match a URL to the appropriate TTL based on endpoint patterns.
* Returns undefined for URLs that should not be cached.
*/
function resolveTtl(url) {
// Snapshot content — any URL replay with /web/{timestamp}.../{original}
if (/\/web\/\d{4,14}(id_|im_|js_|cs_)?\//.test(url)) {
return TTL.SNAPSHOT;
}
// Screenshot access — /screenshot/{url}
if (url.includes("web.archive.org/screenshot/")) {
return TTL.SNAPSHOT;
}
// Availability API
if (url.includes("archive.org/wayback/available")) {
return TTL.AVAILABILITY;
}
// CDX search API
if (url.includes("web.archive.org/cdx/search/cdx")) {
return TTL.CDX_SEARCH;
}
// Sparkline API
if (url.includes("web.archive.org/__wb/sparkline")) {
return TTL.SPARKLINE;
}
// Save endpoint (GET or POST)
if (url.includes("web.archive.org/save")) {
// Save status polling uses different URL pattern
if (url.includes("/save/status/")) {
return TTL.SAVE_STATUS;
}
return TTL.SAVE;
}
// Unknown endpoint — use default TTL
return undefined;
}
async function hashKey(url) {
const data = new TextEncoder().encode(url);
const buffer = await crypto.subtle.digest("SHA-256", data);
const bytes = new Uint8Array(buffer);
return Array.from(bytes)
.map((b) => b.toString(16).padStart(2, "0"))
.join("");
}
function serialiseHeaders(response) {
const headers = {};
response.headers.forEach((value, key) => {
headers[key] = value;
});
return headers;
}
function toResponse(cached) {
return new Response(cached.body, {
status: cached.status,
statusText: cached.statusText,
headers: cached.headers,
});
}
export class CachingFetcher {
config;
memoryCache = new Map();
constructor(config = {}) {
this.config = { ...DEFAULT_CONFIG, ...config };
}
/**
* The persistent backend in use (disk, KV, etc.).
*/
get backend() {
return this.config.backend;
}
/**
* Fetch with caching. Returns cached response if fresh,
* otherwise fetches from network and populates both caches.
*
* TTL is resolved automatically from the URL pattern.
* Pass cacheTtl to override, or false to bypass cache entirely.
*/
async fetch(url, options = {}, cacheTtl) {
// Only cache GET requests
if (options.method !== undefined && options.method !== "GET") {
return fetchWithTimeout(url, options);
}
if (cacheTtl === false) {
return fetchWithTimeout(url, options);
}
const key = await hashKey(url);
const resolvedTtl = resolveTtl(url);
const ttl = cacheTtl ?? resolvedTtl ?? this.config.ttl;
// Check memory cache
const memoryHit = this.memoryCache.get(key);
if (memoryHit !== undefined && memoryHit.expiry > Date.now()) {
return toResponse(memoryHit);
}
// Check persistent backend
const backendHit = await this.config.backend.get(key);
if (backendHit !== undefined && backendHit.expiry > Date.now()) {
this.memoryCache.set(key, backendHit);
return toResponse(backendHit);
}
// Cache miss — fetch from network
const response = await fetchWithTimeout(url, options);
const body = await response.text();
const entry = {
status: response.status,
statusText: response.statusText,
headers: serialiseHeaders(response),
body,
expiry: Date.now() + (typeof ttl === "number" ? ttl : this.config.ttl),
};
this.memoryCache.set(key, entry);
await this.config.backend.set(key, entry);
return toResponse(entry);
}
async clear() {
this.memoryCache.clear();
await this.config.backend.clear();
}
getStats() {
return { memoryEntries: this.memoryCache.size };
}
prune() {
const now = Date.now();
for (const [key, entry] of this.memoryCache) {
if (entry.expiry <= now) {
this.memoryCache.delete(key);
}
}
return Promise.resolve();
}
}
/**
* Disk-backed cache backend using the filesystem.
* Uses restrictive permissions (0700 dir, 0600 files) to prevent cache poisoning.
*/
/**
* Shared instance for use across tools in stdio mode.
*/
export const cachingFetcher = new CachingFetcher();
//# sourceMappingURL=cache.js.map