@struktoai/mirage-core/dist/commands/builtin/utils/http.js

A Unified Virtual Filesystem For AI Agents

// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ========= Copyright 2026 @ Strukto.AI All Rights Reserved. =========
// This layer reports what the server said and never decides whether that is an
// error: curl treats a 404 as a successful transfer (exit 0, body on stdout)
// while wget treats it as exit 8, so the status has to reach the command. An
// earlier version threw on !resp.ok here, which made both tools fail on any
// 4xx and leaked 'fetch failed' onto stderr for a refused connection.
import { callerSignal } from "../../../context/session_context.js";
import { primeStream } from "../../../io/stream.js";
import { makeAbortError, mergeSignals } from "../../../utils/abort.js";
import { HttpConnectError, HttpTimeoutError } from "../errors.js";
import { HTTP_TIMEOUT_SECONDS } from "./constants.js";
import { cookieValue, sameOrigin } from "./cookie.js";
export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (compatible; mirage/1.0)';
const DEFAULT_PORTS = { 'http:': 80, 'https:': 443 };
// A chain past this many hops reads as a loop. undici's own limit, and the
// connect failure is what its native following reported at it.
const MAX_REDIRECTS = 20;
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
// The Fetch standard's rule, which curl shares by default: 303 answers with
// a GET for anything but HEAD, 301 and 302 turn a POST into a GET, and 307
// and 308 keep the method and its body.
function redirectMethod(status, method) {
    if (status === 303 && method !== 'HEAD')
        return 'GET';
    if ((status === 301 || status === 302) && method === 'POST')
        return 'GET';
    return method;
}
function nextHop(location, current) {
    try {
        const from = new URL(current);
        const to = new URL(location, from);
        return { url: to.href, sameOrigin: to.origin === from.origin };
    }
    catch {
        return null;
    }
}
function hop(resp, url, method, body) {
    return {
        status: resp.status,
        reason: resp.statusText,
        body,
        url,
        method,
        headers: [...resp.headers.entries()],
        history: [],
    };
}
export function isHttpError(resp) {
    return resp.status >= 400;
}
function withDefaultUa(headers) {
    // Header names are case-insensitive, so a `user-agent` the caller set
    // replaces the default rather than riding beside it.
    const given = headers ?? {};
    return Object.keys(given).some((name) => name.toLowerCase() === 'user-agent')
        ? { ...given }
        : { 'User-Agent': DEFAULT_USER_AGENT, ...given };
}
function endpoint(url) {
    try {
        const parsed = new URL(url);
        const port = parsed.port !== '' ? Number(parsed.port) : (DEFAULT_PORTS[parsed.protocol] ?? 80);
        return { host: parsed.hostname, port };
    }
    catch {
        return { host: url, port: 80 };
    }
}
// A fetch that does not verify the server's certificate, which curl's -k
// asks for. fetch itself takes no TLS options, so this runtime-agnostic core
// cannot build one; a host that can registers it (the node package: undici
// over an agent with `rejectUnauthorized: false`). The browser registers
// none, since the platform always verifies, so there -k still verifies.
let insecureFetch = null;
export function registerInsecureFetch(fn) {
    insecureFetch = fn;
}
// What a failed transfer is reported as. Past a canceled invocation, the only
// abort is the deadline, which curl answers with its own code (28), so it is
// told apart from a connection that never opened.
function failure(err, caller, url, started) {
    if (caller?.aborted === true)
        return makeAbortError(caller);
    const { host, port } = endpoint(url);
    if (err instanceof DOMException && err.name === 'AbortError') {
        return new HttpTimeoutError(host, port, Date.now() - started);
    }
    return new HttpConnectError(host, port);
}
// The body as it arrives, under the same deadline and with the same failures
// as the request; the deadline ends with it, and letting it go cancels the
// rest of the transfer.
async function* streamed(resp, url, caller, started, timer) {
    const reader = resp.body?.getReader();
    try {
        if (reader === undefined)
            return;
        for (;;) {
            let step;
            try {
                step = await reader.read();
            }
            catch (err) {
                throw failure(err, caller, url, started);
            }
            if (step.done)
                return;
            yield step.value;
        }
    }
    finally {
        if (timer !== null)
            clearTimeout(timer);
        await reader?.cancel().catch((err) => {
            console.debug('body cancel failed:', err);
        });
    }
}
async function doFetch(url, options) {
    const timeoutMs = options.timeoutMs === undefined ? HTTP_TIMEOUT_SECONDS * 1000 : options.timeoutMs;
    const send = options.verify === false && insecureFetch !== null ? insecureFetch : fetch;
    const started = Date.now();
    // What the invocation answers to: its own signal and the call's (a line's
    // stop, a job's kill or limit), so a stopped CLI that reaches here (git)
    // ends its request as Python's cancelled task does.
    const caller = mergeSignals(options.signal, callerSignal());
    const controller = new AbortController();
    const timer = timeoutMs === null
        ? null
        : setTimeout(() => {
            controller.abort();
        }, timeoutMs);
    // A streamed body takes the deadline with it.
    let handed = false;
    try {
        let headers = withDefaultUa(options.headers);
        const follow = options.followRedirects !== false;
        // Redirects are followed by hand so every hop stays observable, the way
        // httpx keeps `response.history` for the python twin; one deadline
        // spans the whole chain, as curl's --max-time does.
        const history = [];
        let current = url;
        let method = options.method ?? 'GET';
        let body = options.body;
        const typed = Object.entries(headers).find(([name]) => name.toLowerCase() === 'cookie')?.[1];
        for (;;) {
            let sent = headers;
            if (options.cookies !== undefined) {
                const value = cookieValue(options.cookies.header(current), sameOrigin(current, url) ? (typed ?? null) : null);
                sent = Object.fromEntries(Object.entries(headers).filter(([name]) => name.toLowerCase() !== 'cookie'));
                if (value !== null)
                    sent.Cookie = value;
            }
            const init = {
                method,
                headers: sent,
                signal: mergeSignals(controller.signal, caller) ?? controller.signal,
                redirect: 'manual',
            };
            if (body !== undefined) {
                init.body = body;
            }
            let resp;
            try {
                resp = await send(current, init);
                if (follow && resp.type === 'opaqueredirect') {
                    // A browser answers a manual redirect with an opaque one: status
                    // 0, no headers, no Location, so the hops cannot be walked here.
                    // The platform follows them instead and the history stays empty,
                    // a deliberate divergence from curl, whose -iL shows every hop.
                    resp = await send(current, { ...init, redirect: 'follow' });
                }
                // The cookies count once the headers are in, as curl keeps them: a
                // body that then fails still leaves them in the jar.
                options.cookies?.store(current, [...resp.headers.entries()]
                    .filter(([name]) => name === 'set-cookie')
                    .map(([, value]) => value));
            }
            catch (err) {
                throw failure(err, caller, current, started);
            }
            const location = resp.headers.get('location');
            const next = follow && REDIRECT_STATUSES.has(resp.status) && location !== null
                ? nextHop(location, current)
                : null;
            if (next === null && options.stream === true) {
                const body = await primeStream(streamed(resp, current, caller, started, timer));
                handed = true;
                return { ...hop(resp, current, method, body), history };
            }
            let buf;
            try {
                // The deadline can fire while the body is still arriving, so the
                // body is read under the same failures, as httpx reads it inside
                // the request the python twin wraps.
                buf = await resp.arrayBuffer();
            }
            catch (err) {
                throw failure(err, caller, current, started);
            }
            const answer = hop(resp, current, method, new Uint8Array(buf));
            if (next === null) {
                return { ...answer, history };
            }
            if (history.length >= MAX_REDIRECTS) {
                const { host, port } = endpoint(current);
                throw new HttpConnectError(host, port);
            }
            history.push(answer);
            if (!next.sameOrigin) {
                // Credentials stay with the host they were typed for, as the native
                // follow and curl (without --location-trusted) both keep them: the
                // Authorization and the Cookie header a line typed.
                headers = Object.fromEntries(Object.entries(headers).filter(([name]) => !['authorization', 'cookie'].includes(name.toLowerCase())));
            }
            const switched = redirectMethod(resp.status, method);
            if (switched !== method)
                body = undefined;
            method = switched;
            current = next.url;
        }
    }
    finally {
        if (!handed && timer !== null)
            clearTimeout(timer);
    }
}
export function httpRequest(url, options = {}) {
    const method = options.method ?? 'GET';
    return doFetch(url, { ...options, method });
}
export function httpFormRequest(url, opts = {}) {
    const method = opts.method ?? 'POST';
    const form = new URLSearchParams();
    for (const [k, v] of Object.entries(opts.formData ?? {})) {
        form.append(k, v);
    }
    const headers = {
        'Content-Type': 'application/x-www-form-urlencoded',
        ...(opts.headers ?? {}),
    };
    return doFetch(url, {
        method,
        headers,
        body: new TextEncoder().encode(form.toString()),
        ...(opts.timeoutMs !== undefined ? { timeoutMs: opts.timeoutMs } : {}),
        ...(opts.followRedirects !== undefined ? { followRedirects: opts.followRedirects } : {}),
        ...(opts.verify !== undefined ? { verify: opts.verify } : {}),
        ...(opts.signal !== undefined ? { signal: opts.signal } : {}),
        ...(opts.cookies !== undefined ? { cookies: opts.cookies } : {}),
        ...(opts.stream !== undefined ? { stream: opts.stream } : {}),
    });
}
export function httpGet(url, opts = {}) {
    const options = { method: 'GET' };
    if (opts.headers !== undefined)
        options.headers = opts.headers;
    if (opts.timeoutMs !== undefined)
        options.timeoutMs = opts.timeoutMs;
    if (opts.signal !== undefined)
        options.signal = opts.signal;
    if (opts.stream !== undefined)
        options.stream = opts.stream;
    return httpRequest(url, options);
}
//# sourceMappingURL=http.js.map