@sema-agent/core/dist/tools/web.d.ts

Stateless, task-oriented AI agent core

import type { AgentTool } from "../internal/harness-types.js";
import type { Brain, ToolSpec } from "../core/types.js";
import type { Model } from "../internal/llm.js";
export interface WebFetchConfig {
    /** If set, ONLY these hostnames may be fetched (exact host match) — and the allowlist is enforced on
     *  **every redirect hop**, not just the initial URL (a redirect to a non-allowlisted host is refused
     *  before the request is made). Omitted ⇒ no host restriction (the deployment's tool-policy gate is then
     *  the only control — wire one for a multi-tenant deployment). */
    allowHosts?: string[];
    /** Injectable fetch (tests / a proxy). Defaults to the global `fetch`. */
    fetchImpl?: typeof fetch;
    /** Max response bytes to read (default 2 MB). */
    maxBytes?: number;
    /** Wall-clock budget for the whole fetch (connect + redirects + body read), in ms. Default 30 000.
     *  Without a local timeout, an unreachable host in a no-egress environment hangs until the OS-level
     *  TCP timeout (60-120 s+) — an agent that retries then burns wall-clock on nothing. On expiry the
     *  tool returns a routable error string, so the model can move on. */
    timeoutMs?: number;
    /** Optional sub-model extraction: given the (markdown) content + the caller's prompt, return a focused
     *  answer. Its output is still derived from untrusted content, so it too is returned fenced.
     *
     *  Pre-2.13 review F1 (additive): the object return form lets a summarizer report that its own OUTPUT
     *  was cut (`truncated: true` — e.g. the sub-model hit max_tokens) as structured fact. The tool then
     *  renders the disclosure OUTSIDE the untrusted fence (a page must not be able to self-declare it) and stamps
     *  `details.truncated`. A plain-string return still works and claims a complete summary.
     *
     *  ruled 2026-08-05 (grounding contract): the object form additionally carries `inputTruncated` — the
     *  summarizer did not SEE the whole page, because the content exceeded what it could feed its model.
     *  That is a different fact from `truncated` (the answer was cut) and used to be invisible: a summarizer
     *  silently clipping its input returned a plain string, and the tool then presented an answer derived
     *  from a prefix as an answer about the page. `inputChars`/`usedChars` size the gap for the disclosure. */
    summarize?: (content: string, prompt: string, signal?: AbortSignal) => Promise<string | {
        text: string;
        truncated?: boolean;
        inputTruncated?: boolean;
        inputChars?: number;
        usedChars?: number;
    }>;
    /** User-Agent header sent with every hop (批② P2-3). Default {@link DEFAULT_WEBFETCH_USER_AGENT};
     *  a deployment can brand it (CC sends `Claude-User (<cc ua>; +https://support.anthropic.com/)`). */
    userAgent?: string;
}
/**
 * Grounding floor for a fetched page (ruled 2026-08-05, WebFetch grounding contract).
 *
 * A page that returns almost no EXTRACTABLE TEXT — the client-rendered shell whose whole body is a
 * script tag plus an empty mount div is the canonical shape — still produces a perfectly well-formed
 * summary when it is handed to an extraction sub-model, because "there is nothing here" is not an answer
 * the model is asked for. The summary then travels inside the untrusted fence with no marker separating
 * "derived from the page" from "invented to fill the shape of the question", and the caller has no way
 * to tell the two apart. The bytes are the discriminator: below this floor the tool stops presenting the
 * summary as a source and says what it actually got.
 *
 * 200 characters is deliberately low — high enough that a JS shell (typically under 100 chars of text,
 * often zero) is always caught, low enough that a real but terse page (a short API response, a plain-text
 * status endpoint) is the only false positive. A terse page is NOT accused of being unsupportable: the
 * disclosure states the measurement and reproduces the source, which is true and useful either way. The
 * tool genuinely cannot distinguish "this URL serves fourteen characters and they are the answer" from
 * "this URL serves a shell", so it says what it measured instead of ruling on which one happened.
 */
export declare const WEBFETCH_GROUNDING_MIN_TEXT_CHARS = 200;
/** Structured grounding verdict for one fetch — rides `details.grounding` (additive) so a deployment can
 *  gate on the same fact the model-facing disclosure is built from, instead of parsing prose. */
export interface WebFetchGrounding {
    /** `"low"` ⇒ the extracted text is below {@link WEBFETCH_GROUNDING_MIN_TEXT_CHARS}. */
    level: "ok" | "low";
    /** Extracted text length after markup removal, whitespace-collapsed. */
    textChars: number;
    /** Size of the body this verdict was measured against — the same value the details card reports as
     *  `bytes`, i.e. wire bytes when the response was read as bytes, characters on a `text()`-only injected
     *  fetch, and the KEPT prefix (not the original object) whenever the byte cap or a cut applied. The
     *  denominator of {@link textRatio}. */
    bytes: number;
    /** `textChars / bytes`, rounded to 3 decimals; 0 when nothing was retrieved. A ratio near 0 with a
     *  large `bytes` is the markup/script-dominated shape; near 1 is plain text. */
    textRatio: number;
}
export declare function htmlToText(html: string): string;
export declare function webFetchToolSpec(config?: WebFetchConfig): ToolSpec;
/** Defined (harness) form of web_fetch — back-compat for direct execution / tests. For `spec.tools`, use the raw
 *  {@link webFetchToolSpec} (prepare-task defineTool-wraps spec.tools entries).
 *
 *  ⚠️ CTX-DEPENDENT BEHAVIOR: this factory takes no `enrichCtx`, so the product it returns is sealed
 *  around the pre-RB-409 minimal `{toolCallId, signal}` ctx. Fed into `TaskSpec.tools`, a finished
 *  product is mounted AS-IS (prepare-task's brand branch — there is no ctx seat on
 *  `AgentTool.execute` to inject through), so every `ctx.*` read inside {@link webFetchToolSpec}
 *  reads absent — including `reminderMark`/`reminderDisclosureCounts`, i.e. the design/319 exact-mark
 *  DEFUSE and its observation counters silently do not run for that mount. Two ways to keep them:
 *  hand `spec.tools` the raw {@link webFetchToolSpec} (the Runner then wraps it with its own trusted
 *  ctx builder), or call `defineTool(webFetchToolSpec(config), { enrichCtx })` yourself. */
export declare function createWebFetchTool(config?: WebFetchConfig): AgentTool;
/** {@link createWebFetchSummarizer}'s truncation bound — CC's `MAX_MARKDOWN_LENGTH`
 *  (WebFetchTool/utils.ts:128). Exported so a deployment wiring the reference summarizer doesn't need
 *  its own copy of this number to reason about (a downstream deployment asked for this explicitly,
 *  to avoid a value-copy of the truncation threshold drifting from this one). */
export declare const WEBFETCH_SUMMARY_MAX_CONTENT = 100000;
/** {@link createWebFetchSummarizer}'s fixed guidelines block — CC's `makeSecondaryModelPrompt`
 *  (WebFetchTool/prompt.ts, non-preapproved-domain arm, guidelines verbatim). Exported alongside
 *  {@link WEBFETCH_SUMMARY_MAX_CONTENT} for the same reason: a deployment composing
 *  its OWN summarizer (not this reference one) that still wants CC-parity guidance text doesn't need its
 *  own copy of this string. Not runtime-parameterized — the page content and the caller's `prompt` are
 *  interpolated AROUND this block, never inside it. */
export declare const WEBFETCH_SUMMARY_GUIDELINES: string;
/**
 * sema's own grounding clause for the reference summarizer (ruled 2026-08-05), kept OUT of
 * {@link WEBFETCH_SUMMARY_GUIDELINES} so that block stays the byte-verbatim CC anchor it is documented to
 * be. Interpolated BEFORE it, so the CC block remains the prompt's fixed tail.
 *
 * Two failure modes observed in the field, both of which produce a confident, well-formed, wrong answer:
 *  · a page with no substantive content still gets a specific answer written for it (the code-side
 *    grounding floor catches the extreme case, but the model is the only one that can decline mid-band);
 *  · a listing of resources is judged relevant or irrelevant from the NAMES in it — "none of these files
 *    mention X, so this source has nothing on X" — which reports a property of a naming convention as a
 *    property of the data, and sends the caller away from a source that did hold the answer.
 * Both are instructions to report the absence rather than to resolve it, which is why they belong in the
 * prompt rather than in a post-hoc check.
 */
export declare const WEBFETCH_SUMMARY_GROUNDING_CLAUSE: string;
/**
 * Characters of page content the reference summarizer will feed its model, derived from that model's own
 * declared window (ruled 2026-08-05). Previously the only bound was the flat CC-parity
 * {@link WEBFETCH_SUMMARY_MAX_CONTENT}, which is a property of the PAGE side and says nothing about the
 * model on the other end: a small-window extraction model — the usual choice, since this is the "small
 * fast model" leg — was handed ~100 000 characters and the whole call failed, so the caller got the
 * "summarization failed" fallback (a raw dump that is then truncated anyway) instead of an answer.
 *
 * `headroom` exists because the character→token estimate is a floor, not a bound: {@link Model.charsPerToken}
 * defaults to 4, and dense content (CJK, JSON, code, long URLs) tokenizes well below that, so a budget
 * computed at face value overshoots exactly on the pages large enough to matter. The result is still
 * clamped by the CC constant, so no deployment's input grows because of this.
 */
export declare const WEBFETCH_SUMMARY_INPUT_HEADROOM = 0.8;
/**
 * Floor for the derived budget. A model whose declared dimensions leave no room after the output
 * reservation still gets a small excerpt rather than a computation that lands at or below zero and feeds
 * the summarizer an empty page — a call that cannot possibly answer is strictly worse than one that might.
 *
 * Honest scope (falsification review round 1): this is a FLOOR, not a proof of fit. With the output
 * reservation bounded at window/4 (see `resolveSummaryInputChars`), the floor now fires only on genuinely
 * tiny windows (roughly under 3k tokens at the default chars-per-token) — a model whose declared
 * dimensions leave no room gets a small excerpt whose attempt often succeeds anyway, and a failure is
 * already disclosed by the tool's "summarization failed" head-note rather than swallowed.
 * Refusing at construction instead was considered and declined: `maxTokens` is commonly declared equal to
 * the window by providers of small models, so refusing would reject deployments that work today.
 */
export declare const WEBFETCH_SUMMARY_MIN_CONTENT = 4000;
/**
 * Resolve the page-content budget for {@link createWebFetchSummarizer} against `model`'s window.
 * An explicit `override` wins outright (a deployment that measured its own model beats an estimate).
 *
 * Fail-loud on every non-finite input, same posture as {@link resolveWebMaxBytes} (falsification review
 * round 1): a `NaN` anywhere in the model's declared dimensions propagates through the arithmetic and out
 * through `Math.min`/`Math.max` unchanged, and a `NaN` budget makes `content.length > budget` false — so
 * the clip silently stops happening and the whole oversize page is fed to the model, which is the failure
 * this function exists to prevent, restored by a different door. An `override` below 1 is refused for the
 * same reason in reverse: it floors to 0 and feeds the model an empty page.
 */
export declare function resolveSummaryInputChars(model: Model, override?: number): number;
/** Options for {@link createWebFetchSummarizer}. */
export interface WebFetchSummarizerOptions {
    /** Override the derived page-content budget (see {@link resolveSummaryInputChars}). */
    maxContentChars?: number;
    /**
     * Per-model auth — MIRRORS {@link TaskSpec.getApiKeyAndHeaders} (same signature, resolved per
     * call against the summarizer's model, the side-query seat's form). Without this seat the
     * summarizer called the brain with `{ signal }` only, so a summarizer model on a
     * per-model-credential route fell to the brain's construction-time credential — which the
     * key↔URL pairing gate refuses off the deployment root. Absent ⇒ the options object is
     * byte-identical to the pre-seat shape (no keys added).
     */
    getApiKeyAndHeaders?: (model: Model) => Promise<{
        apiKey?: string;
        headers?: Record<string, string>;
    } | undefined> | {
        apiKey?: string;
        headers?: Record<string, string>;
    } | undefined;
}
/**
 * A reference {@link WebFetchConfig.summarize} implementation: reuses a
 * caller-supplied `brain` (rather than requiring a dedicated second model) to extract a focused answer
 * from fetched content per the caller's `prompt`, instead of the tool dumping the whole (fenced, still
 * untrusted) page. CC ALWAYS routes fetched content through a secondary model this way
 * (WebFetchTool/utils.ts:484-530); a deployment that omits `summarize` entirely gets `webFetchToolSpec`'s
 * own honest "summarization unavailable in this deployment" head-note + raw dump instead — never a
 * silent one. Prompt shape matches CC's non-preapproved-domain secondary-model prompt (WebFetchTool/
 * prompt.ts `makeSecondaryModelPrompt`, guidelines block verbatim).
 *
 * Not the only valid implementation — a deployment with a genuinely separate summarization model/service
 * can supply its own function of the same shape to `WebFetchConfig.summarize` instead. This one exists so
 * "just reuse whatever brain/model I already have" doesn't have to be reinvented per deployment; it was
 * previously a private copy inside this package's own `sema-tb` CLI entry point, promoted here so other
 * deployments (a server assembling `WebFetchConfig` for its own scenarios, for one) can reuse it directly
 * rather than re-deriving it from source.
 */
export declare function createWebFetchSummarizer(brain: Brain, model: Model, options?: WebFetchSummarizerOptions): (content: string, prompt: string, signal?: AbortSignal) => Promise<string | {
    text: string;
    truncated?: boolean;
    inputTruncated?: boolean;
    inputChars?: number;
    usedChars?: number;
}>;
/**
 * design/102 (K-8 CC full-body) — WebSearch (BRAIN / model-service leg). The search BACKEND is deployment-injected
 * (provider-native search / a search API / an MCP-backed service); core ships NONE — the same boundary as
 * {@link WebFetchConfig.summarize} and the model gateway. Absent backend ⇒ the tool is not assembled
 * ({@link assembleCodeTools} only pushes it when `search` is wired). Results are UNTRUSTED external data —
 * {@link delimitUntrusted}-fenced like web_fetch. 🔴 `effect:"read"`, NO `egress` field: a search is
 * idempotent/retryable; declaring `egress:true` would throw at prep (`config.egress_requires_write_effect`) AND
 * break resume-safety (a read reconciles as safe-to-retry, a write/egress one does not).
 */
export interface WebSearchConfig {
    /**
     * Deployment-injected search backend. core ships none. The optional `opts` carries the model's per-call
     * `allowed_domains`/`blocked_domains` HINTS so a backend that supports native domain filtering (e.g. a
     * `site:` query) can apply them up front. A backend MAY ignore `opts` — core re-enforces the domain
     * constraint on the returned results as a floor (see `createWebSearchTool`), so honoring it is an
     * optimization, not a correctness requirement. Back-compat: existing 2-arg backends keep working.
     */
    search: (query: string, signal?: AbortSignal, opts?: {
        allowedDomains?: string[];
        blockedDomains?: string[];
    }) => Promise<Array<{
        title: string;
        url: string;
        snippet: string;
    }>>;
    /** Max results returned to the model (default 10). */
    maxResults?: number;
    /** Wall-clock budget for one `search` call, in ms. Default 30 000 (RB-231). The backend is
     *  deployment-injected and may ignore the AbortSignal entirely, so this is enforced with BOTH an abort
     *  and a race — a hung backend must not hang the turn, and without this the deployment had no interface
     *  to bound it. */
    timeoutMs?: number;
}
/** Clip by CODE POINT (RB-122/RB-142/RB-147 family: a UTF-16 `.slice` can split a surrogate pair). Clips
 *  RAW field text; `delimitUntrusted` owns the neutralization afterwards — clipping with `inlineUntrusted`
 *  here would fold newlines and double-neutralize. Exported for the RB-142 D-1 enumerative clipper guard
 *  (defectscan-truncation-invariant-surface) — not on the npm face (src/index.ts does not re-export it). */
export declare function clipCodePoints(s: string, max: number): string;
export declare function createWebSearchTool(config: WebSearchConfig): ToolSpec;
/** Options for {@link createSearxngSearchBackend}. */
export interface SearxngBackendOptions {
    /** Injected fetch (tests / exotic runtimes). Defaults to `globalThis.fetch`. */
    fetchImpl?: typeof fetch;
    /** Per-request wall clock in ms before the request is aborted. Defaults to 10 000 — well inside
     *  WebSearchConfig's own 30 s budget so the tool-level race never fires first on a healthy instance. */
    timeoutMs?: number;
    /** Extra query parameters appended to every /search call (e.g. `{ language: "zh-CN" }`). */
    extraParams?: Record<string, string>;
}
/**
 * design/162 (core half) — a {@link WebSearchConfig.search} backend speaking SearXNG's JSON API
 * (`GET <baseUrl>/search?q=…&format=json`). The zero-key rung of the search fallback chain: the shell
 * installs and health-checks a local SearXNG instance; a deployment then assembles this backend from
 * nothing but the base URL. core owns NO process management — a dead instance surfaces as the thrown
 * fetch/HTTP error, which `createWebSearchTool` already fences and bounds (SEARCH_ERROR_EXCERPT_CHARS).
 *
 * The model's `allowed_domains` hint maps to a `site:` query prefix (SearXNG forwards it to engines
 * that understand it) — an OPTIMIZATION only: the tool re-enforces the domain constraint on returned
 * results either way, so a backend ignoring the hint is correct, just wasteful. `blocked_domains` has
 * no reliable engine-side spelling and is left entirely to the tool's own floor.
 *
 * Result mapping keeps the strict {title, url, snippet} shape: SearXNG's `content` field is the
 * snippet; rows missing a url are dropped (an engine-side info box, not a citable result).
 */
export declare function createSearxngSearchBackend(baseUrl: string, options?: SearxngBackendOptions): WebSearchConfig["search"];
/**
 * One live probe of a search backend (design/162 onboarding convenience): calls it with a fixed
 * cheap query and classifies the outcome. Never throws — the classification IS the answer.
 */
export declare function probeSearchBackend(search: WebSearchConfig["search"], options?: {
    timeoutMs?: number;
}): Promise<{
    ok: true;
    results: number;
} | {
    ok: false;
    error: string;
}>;