@vigilbase/cli/dist/local/cloudflare-checkup/collector.js

Vigilbase CLI — security scanners, agent tools, and Vigilbase One from the terminal.

/**
 * Collecting a Cloudflare account through cf (spec §F5, §D3).
 *
 * Every read goes through the frozen allowlist and the shared scheduler (four
 * workers, the request budget, auth pauses and retries). Each response is
 * projected immediately by the vendored contract and the raw text discarded.
 * Lists are paged with `--page N` until an empty page, because cf strips
 * `result_info`: a final page and a server-clamped page look the same.
 */
import { ACCOUNT_KINDS, CloudflareCheckupSnapshotV1Schema, NOT_COLLECTABLE_ZONE_KINDS, NOT_COLLECTED_RESULT, PROJECTORS, SNAPSHOT_LIMITS, ZONE_SETTING_IDS, ZONE_SINGLE_KINDS, assembleSnapshot, collectionFailedResult, httpErrorResult, projectAccount, projectZones, projectionToResult, serializeSnapshot, } from './contract.js';
import { ENV_TOKEN_PERMISSION_HINT, LocalCheckupError } from './errors.js';
import { snapshotParseError } from './snapshot-file.js';
import { TEMPLATES, buildArgv } from './cf/allowlist.js';
import { authRequiredError } from './cf/auth-lifecycle.js';
import { RULESET_STDOUT_CAP_BYTES, parseCfJson } from './cf/run-cf.js';
export const WORKER_COUNT = 4;
/** Assumed ruleset details per zone for the up-front request estimate. */
const ESTIMATED_RULESETS_PER_ZONE = 6;
const failed = (detail, failure) => ({ ok: false, result: collectionFailedResult(detail), ...(failure ? { failure } : {}) });
function failureResult(failure) {
    switch (failure.kind) {
        case 'http':
            return failure.status >= 400 && failure.status <= 599
                ? { ok: false, result: httpErrorResult(failure.status), failure }
                : failed('cf_error', failure);
        case 'timeout': return failed('timeout', failure);
        case 'output_too_large': return failed('output_too_large', failure);
        case 'command_unavailable': return failed('command_unavailable', failure);
        default: return failed('cf_error', failure);
    }
}
function availabilityKey(id, settingId) {
    return id === 'zone_setting' && settingId ? `zone_setting:${settingId}` : id;
}
/** Fetch one read: a single call, a one-call list, or every page of a paged list. */
async function fetchRaw(env, id, ids, options = {}) {
    if (env.unavailable.has(availabilityKey(id, ids.settingId)))
        return failed('command_unavailable');
    const spec = TEMPLATES[id];
    const callOptions = {
        ...(ids.accountId !== undefined && id !== 'account_selection' ? { accountId: ids.accountId } : {}),
        ...(id === 'zone_ruleset' ? { stdoutCapBytes: RULESET_STDOUT_CAP_BYTES } : {}),
    };
    const once = async (page) => {
        const outcome = await env.scheduler.call(buildArgv(id, page === undefined ? ids : { ...ids, page }, env.auth), callOptions);
        if (!outcome.ok)
            return failureResult(outcome.failure);
        const parsed = parseCfJson(outcome.stdout, spec.output);
        return parsed.ok ? { ok: true, raw: parsed.value } : failed('invalid_output');
    };
    if (spec.paging.kind === 'single')
        return once();
    if (spec.paging.kind === 'one_call') {
        const outcome = await once();
        // The cursor is not printed, so a full page may hide more rulesets.
        if (outcome.ok && outcome.raw.length >= SNAPSHOT_LIMITS.rulesetListTruncationAt)
            return failed('possibly_truncated');
        return outcome;
    }
    const items = [];
    let previousFirst;
    for (let page = 1;; page++) {
        const outcome = await once(page);
        if (!outcome.ok)
            return outcome;
        const pageItems = outcome.raw;
        if (pageItems.length === 0)
            return { ok: true, raw: items };
        if (page > spec.paging.maxPages)
            return failed('page_limit');
        const first = JSON.stringify(pageItems[0]);
        // cf ignoring --page would repeat the first page forever.
        if (page > 1 && first === previousFirst)
            return failed('invalid_output');
        previousFirst = first;
        items.push(...pageItems);
        if (options.maxItems !== undefined && items.length > options.maxItems) {
            return { ok: false, result: collectionFailedResult('output_too_large'), tooManyItems: true };
        }
    }
}
function isAuthFailure(failure) {
    return failure?.kind === 'auth' || (failure?.kind === 'http' && failure.status === 401);
}
/** Abort errors for reads that must succeed (account selection, account, zones). */
function criticalError(env, outcome, what) {
    if (isAuthFailure(outcome.failure))
        return authRequiredError(env.auth.authMode);
    if (outcome.result.reason === 'collection_failed' && outcome.result.detail === 'command_unavailable') {
        return new LocalCheckupError('CF_ALLOWLIST_MISMATCH', `This cf release cannot run the ${what} command the checkup needs.`);
    }
    if (outcome.failure?.kind === 'http' && outcome.failure.status === 403) {
        return new LocalCheckupError('CF_PERMISSION_DENIED', `Cloudflare denied the ${what} read (HTTP 403).`, { status: 403, ...(env.auth.authMode === 'env' ? { hint: ENV_TOKEN_PERMISSION_HINT } : {}) });
    }
    const status = outcome.failure?.kind === 'http' ? outcome.failure.status : undefined;
    return new LocalCheckupError('CF_ERROR', `cf could not read the ${what}${status ? ` (HTTP ${status})` : outcome.result.reason === 'collection_failed' ? ` (${outcome.result.detail})` : ''}.`, status ? { status } : {});
}
/** `accounts list --per-page 50 --page N` until an empty page. */
export async function listAccounts(env) {
    const outcome = await fetchRaw(env, 'account_selection', {});
    if (!outcome.ok)
        throw criticalError(env, outcome, 'account list');
    const accounts = [];
    for (const item of outcome.raw) {
        if (!item || typeof item !== 'object')
            continue;
        const { id, name } = item;
        if (typeof id !== 'string' || !/^[0-9a-f]{32}$/i.test(id))
            continue;
        accounts.push({ id: id.toLowerCase(), name: typeof name === 'string' ? name.slice(0, 256) : '' });
    }
    return accounts;
}
/** Requests for N zones: about 19 + extra pages + N × (33 + rulesets per zone). */
export function estimateRequests(zoneCount, rulesetsPerZone = ESTIMATED_RULESETS_PER_ZONE) {
    return 19 + zoneCount * (33 + rulesetsPerZone);
}
/** A worker pool whose tasks may enqueue more tasks (ruleset details). */
async function runTasks(seed, concurrency, onError) {
    const queue = [];
    let active = 0;
    let error = null;
    let notify = () => undefined;
    let changed = new Promise((resolve) => { notify = resolve; });
    const signal = () => {
        const resolve = notify;
        changed = new Promise((next) => { notify = next; });
        resolve();
    };
    const enqueue = (task, options) => {
        if (options?.front)
            queue.unshift(task);
        else
            queue.push(task);
        signal();
    };
    seed(enqueue);
    const worker = async () => {
        for (;;) {
            if (error)
                return;
            const task = queue.shift();
            if (!task) {
                if (active === 0)
                    return;
                await changed;
                continue;
            }
            active++;
            try {
                await task(enqueue);
            }
            catch (caught) {
                if (!error) {
                    error = caught;
                    onError(caught);
                }
            }
            finally {
                active--;
                signal();
            }
        }
    };
    await Promise.all(Array.from({ length: concurrency }, () => worker()));
    if (error)
        throw error;
}
/**
 * Collect every snapshot entry for one account. Aborts (no file) when the
 * account read fails other than 400/403/404, when the zone list fails, when it
 * holds more than 100 zones, or when the account is invisible and has no zones.
 */
export async function collectAccount(env, accountId, hooks = {}) {
    const entries = [];
    const ids = { accountId };
    const account = await fetchRaw(env, 'account', ids);
    if (account.ok) {
        const projected = projectAccount(account.raw);
        if (!projected.ok)
            throw new LocalCheckupError('CF_ERROR', 'cf returned account details the checkup cannot read.');
        if (projected.data.id !== accountId)
            throw new LocalCheckupError('CF_ERROR', 'cf returned a different Cloudflare account than requested.');
        entries.push({ kind: 'account', result: { ok: true, data: projected.data } });
    }
    else if (account.result.reason === 'http_error' && [400, 403, 404].includes(account.result.status)) {
        // The scanner tolerates these (runInitPhase); 401 is never recorded.
        entries.push({ kind: 'account', result: account.result });
    }
    else {
        throw criticalError(env, account, 'account');
    }
    const zonesRaw = await fetchRaw(env, 'zones', ids, { maxItems: SNAPSHOT_LIMITS.maxZones });
    if (!zonesRaw.ok) {
        if (zonesRaw.tooManyItems) {
            throw new LocalCheckupError('ACCOUNT_TOO_LARGE', `This account has more than ${SNAPSHOT_LIMITS.maxZones} zones; use the browser token mode.`);
        }
        throw criticalError(env, zonesRaw, 'zone list');
    }
    const zonesProjected = projectZones(zonesRaw.raw);
    if (!zonesProjected.ok)
        throw new LocalCheckupError('CF_ERROR', 'cf returned a zone list the checkup cannot read.');
    const zones = zonesProjected.data;
    if (zones.some((zone) => zone.account.id !== accountId))
        throw new LocalCheckupError('CF_ERROR', 'cf returned zones from a different account.');
    if (!entries[0].result.ok && zones.length === 0) {
        throw new LocalCheckupError('ACCOUNT_NOT_VISIBLE', 'Cloudflare does not show this account or any of its zones to the signed-in cf user.');
    }
    entries.push({ kind: 'zones', result: { ok: true, data: zones } });
    hooks.onZones?.(zones);
    const zoneTasksLeft = new Map(zones.map((zone) => [zone.id, 0]));
    let zonesDone = 0;
    const report = () => env.onProgress?.({ zonesDone, zonesTotal: zones.length, requests: env.scheduler.invocations });
    const finishZoneTask = (zoneId) => {
        const left = (zoneTasksLeft.get(zoneId) ?? 1) - 1;
        zoneTasksLeft.set(zoneId, left);
        if (left === 0)
            zonesDone++;
        report();
    };
    const projectInto = async (kind, taskIds) => {
        const outcome = await fetchRaw(env, kind, taskIds);
        if (!outcome.ok)
            return outcome.result;
        const project = PROJECTORS[kind];
        return projectionToResult(project(outcome.raw));
    };
    await runTasks((enqueue) => {
        for (const kind of ACCOUNT_KINDS) {
            if (kind === 'account' || kind === 'zones')
                continue;
            enqueue(async () => {
                entries.push({ kind, result: await projectInto(kind, ids) });
                report();
            });
        }
        for (const zone of zones) {
            const zoneIds = { accountId, zoneId: zone.id };
            zoneTasksLeft.set(zone.id, ZONE_SETTING_IDS.length + ZONE_SINGLE_KINDS.length);
            for (const settingId of ZONE_SETTING_IDS) {
                enqueue(async () => {
                    entries.push({ kind: 'zone_setting', zoneId: zone.id, settingId, result: await projectInto('zone_setting', { ...zoneIds, settingId }) });
                    finishZoneTask(zone.id);
                });
            }
            for (const kind of ZONE_SINGLE_KINDS) {
                enqueue(async (more) => {
                    const result = await projectInto(kind, zoneIds);
                    entries.push({ kind, zoneId: zone.id, result });
                    if (kind === 'zone_rulesets' && result.ok) {
                        const rulesetIds = [...new Set(result.data.map((head) => head.id))];
                        zoneTasksLeft.set(zone.id, (zoneTasksLeft.get(zone.id) ?? 0) + rulesetIds.length);
                        for (const rulesetId of rulesetIds) {
                            more(async () => {
                                entries.push(await collectRulesetDetail(env, zoneIds, rulesetId));
                                finishZoneTask(zone.id);
                            }, { front: true });
                        }
                    }
                    finishZoneTask(zone.id);
                });
            }
            for (const kind of NOT_COLLECTABLE_ZONE_KINDS)
                entries.push({ kind, zoneId: zone.id, result: NOT_COLLECTED_RESULT });
        }
    }, env.concurrency ?? WORKER_COUNT, (error) => env.scheduler.abort(error instanceof Error ? error : new Error(String(error))));
    env.scheduler.throwIfAborted();
    return entries;
}
/** A ruleset detail keeps its projected rules; `assembleSnapshot` applies the failed-detail collector rule. */
async function collectRulesetDetail(env, ids, rulesetId) {
    const outcome = await fetchRaw(env, 'zone_ruleset', { ...ids, rulesetId });
    const base = { kind: 'zone_ruleset', zoneId: ids.zoneId, rulesetId };
    if (!outcome.ok)
        return { ...base, result: outcome.result };
    const projected = PROJECTORS.zone_ruleset(outcome.raw);
    if (!projected.ok)
        return { ...base, result: collectionFailedResult(projected.detail) };
    // A detail whose ID differs from the listed head cannot be attributed.
    if (projected.data.id !== rulesetId)
        return { ...base, result: collectionFailedResult('invalid_output') };
    return { ...base, result: { ok: true, data: projected.data } };
}
/** Assemble, validate and serialise; the returned text is exactly what is written and uploaded. */
export function buildSnapshot(accountId, collector, entries) {
    const assembled = assembleSnapshot({ accountId, collector, entries });
    const parsed = CloudflareCheckupSnapshotV1Schema.safeParse(assembled);
    if (!parsed.success)
        throw snapshotParseError(parsed.error.issues);
    const text = serializeSnapshot(parsed.data);
    if (Buffer.byteLength(text) > SNAPSHOT_LIMITS.maxBytes) {
        throw new LocalCheckupError('SNAPSHOT_TOO_LARGE', 'The collected snapshot is larger than the 8 MiB contract limit; no file was written.');
    }
    return { snapshot: parsed.data, text };
}