@sema-agent/core/dist/tools/artifact/conformance.js

Stateless, task-oriented AI agent core

import { randomUUID } from "node:crypto";
import { thrownText } from "../../core/thrown-value.js";
import { readArtifactHostCapabilities, } from "../../core/artifact-host.js";
class Violation extends Error {
}
const fail = (detail) => {
    throw new Violation(detail);
};
const check = (cond, detail) => {
    if (!cond)
        fail(detail);
};
const enc = new TextEncoder();
const FOREIGN_URL = "https://conformance.invalid/artifact/none";
const HTML_MEDIA = "text/html";
const ctxOf = (h, attemptId) => ({ taskId: `conformance-${h.runId}`, runId: h.runId, attemptId: attemptId ?? `${h.runId}-${++h.seq}` });
const fixtureBudget = (h) => Math.max(1, Math.min(h.caps.maxFileBytes, Math.floor(h.caps.maxTotalBytes / 2)));
const fit = (h, text) => enc.encode(text).slice(0, fixtureBudget(h));
const el = (tag, inner) => `<${tag}>${inner}</${tag}>`;
const page = (h, n, extraFiles = []) => [{ path: "index.html", content: fit(h, `${n}${el("title", `C${n}`)}${el("p", String(n))}`), mediaType: HTML_MEDIA }, ...extraFiles];
const script = (h, n) => ({ path: "assets/app.js", content: fit(h, `${n};console.log(${n})`).slice(0, Math.floor(h.caps.maxTotalBytes / 2)), mediaType: "text/javascript" });
const req = (files, over = {}) => ({ files, publishContext: "plain", ...over });
const same = (a, b) => a.byteLength === b.byteLength && a.every((x, i) => x === b[i]);
async function publish(h, r, attemptId) {
    try {
        return await h.host.publish(r, ctxOf(h, attemptId));
    }
    catch (err) {
        return fail(`publish threw: ${thrownText(err)}`);
    }
}
async function publishOk(h, r, what, attemptId) {
    const out = await publish(h, r, attemptId);
    if (!out.ok)
        return fail(`${what}: expected ok, got ${out.code}${"message" in out ? ` (${out.message})` : ""}`);
    check(typeof out.ref.url === "string" && out.ref.url.length > 0 && typeof out.ref.slug === "string" && typeof out.ref.version === "string", `${what}: the ref must carry url, slug and version strings`);
    return out.ref;
}
async function listMine(h) {
    const out = await h.host.list({ scope: "mine", limit: 1000 }, ctxOf(h));
    if (!out.ok)
        return fail(`list mine refused: ${out.message}`);
    return out.rows;
}
const vectorTable = [];
const define = (v, run) => void vectorTable.push({ ...v, run });
define({ id: "publish.fresh_mints_distinct_pages", describe: "two publishes without `url` mint two pages with distinct urls (fresh never touches an existing page)", requiresActions: ["publish"] }, async (h) => {
    const a = await publishOk(h, req(page(h, 1)), "first fresh");
    const b = await publishOk(h, req(page(h, 2)), "second fresh");
    check(a.url !== b.url, `both fresh publishes answered the same url ${a.url}`);
    return `two pages: ${a.url}, ${b.url}`;
});
define({ id: "publish.update_by_url_advances_same_page", describe: "a publish with `url` and the current `baseVersion` advances that page's version on the same url and slug (update never mints a page); where read / read_file are declared, the NEW bytes are what reads back", requiresActions: ["publish"] }, async (h) => {
    const a = await publishOk(h, req(page(h, 1)), "fresh");
    const next = page(h, 2);
    const b = await publishOk(h, req(next, { url: a.url, baseVersion: a.version }), "update");
    check(b.url === a.url && b.slug === a.slug, `the update answered url ${b.url} / slug ${b.slug}, expected ${a.url} / ${a.slug}`);
    check(b.version !== a.version, `the update left the version at ${a.version}`);
    const persisted = [];
    if (h.caps.actions.includes("read")) {
        const r = await h.host.read({ url: a.url }, ctxOf(h));
        check(r.ok && r.content === new TextDecoder().decode(next[0].content) && r.ref.version === b.version, "read after the update returns the PREVIOUS content or version: the update was acknowledged but not persisted");
        persisted.push("read");
    }
    if (h.caps.actions.includes("read_file")) {
        const f = await h.host.readFile({ url: a.url, path: next[0].path }, ctxOf(h));
        check(f.ok && same(f.file.content, next[0].content), "read_file after the update returns the PREVIOUS bytes: the update was acknowledged but not persisted");
        persisted.push("read_file");
    }
    return `version ${a.version} → ${b.version} on ${a.url}${persisted.length > 0 ? `; new bytes read back via ${persisted.join(", ")}` : ""}`;
});
define({ id: "publish.no_base_version_is_last_write_wins", describe: "`url` present and `baseVersion` absent ⇒ the write applies (last-write-wins is a defined behaviour, never a conflict)", requiresActions: ["publish"] }, async (h) => {
    const a = await publishOk(h, req(page(h, 1)), "fresh");
    const b = await publishOk(h, req(page(h, 2), { url: a.url }), "first update without baseVersion");
    const c = await publishOk(h, req(page(h, 3), { url: a.url }), "second update without baseVersion");
    check(b.version !== a.version && c.version !== b.version, `versions did not advance: ${a.version}, ${b.version}, ${c.version}`);
    return `three versions on ${a.url}`;
});
define({ id: "publish.stale_base_conflicts", describe: "a stale `baseVersion` ⇒ `conflict` carrying the CURRENT version and either a full snapshot or the explicit `\"reread\"` word; nothing is written", requiresActions: ["publish"] }, async (h) => {
    const a = await publishOk(h, req(page(h, 1)), "fresh");
    const b = await publishOk(h, req(page(h, 2), { url: a.url, baseVersion: a.version }), "update to v2");
    const out = await publish(h, req(page(h, 3), { url: a.url, baseVersion: a.version }));
    check(!out.ok && out.code === "conflict", `a stale base answered ${out.ok ? "ok" : out.code}, expected conflict`);
    if (out.ok || out.code !== "conflict")
        return "";
    check(out.currentVersion === b.version, `conflict names currentVersion ${out.currentVersion}, expected ${b.version}`);
    check(out.current === "reread" || (typeof out.current === "object" && Array.isArray(out.current.files)), "conflict.current must be a snapshot object or the string \"reread\"");
    const after = await publishOk(h, req(page(h, 4), { url: a.url, baseVersion: b.version }), "update on the true current version after the conflict");
    check(after.version !== b.version, "the conflict wrote a version: the base the host itself named was stale one call later");
    return `conflict at ${b.version}, current: ${out.current === "reread" ? "reread" : "snapshot"}`;
});
for (const action of ["read", "read_file"]) {
    define({ id: `publish.conflict_preserves_${action}`, describe: `a stale-base rejection leaves readable bytes unchanged through ${action}, even if the host preserves the version number and returns a correct pre-write snapshot`, requiresActions: ["publish", action] }, async (h) => {
        const a = await publishOk(h, req(page(h, 1)), "fresh");
        const b = await publishOk(h, req(page(h, 2), { url: a.url, baseVersion: a.version }), "update");
        const observe = async () => {
            if (action === "read") {
                const out = await h.host.read({ url: a.url }, ctxOf(h));
                if (!out.ok)
                    return fail(`read refused: ${out.code}`);
                return JSON.stringify([out.ref.url, out.ref.slug, out.ref.version, out.ref.title, out.content, out.isolatedSummary]);
            }
            const out = await h.host.readFile({ url: a.url, path: "index.html" }, ctxOf(h));
            if (!out.ok)
                return fail(`read_file refused: ${out.code}`);
            return JSON.stringify([out.file.path, out.file.mediaType, Array.from(out.file.content)]);
        };
        const before = await observe();
        const rejected = await publish(h, req(page(h, 3), { url: a.url, baseVersion: a.version }));
        check(!rejected.ok && rejected.code === "conflict" && rejected.currentVersion === b.version, "a stale base must conflict at the current version");
        check(await observe() === before, `${action} changed after a rejected publish`);
        return `${action} unchanged after conflict`;
    });
}
define({ id: "publish.conflict_snapshot_is_complete", describe: "when the conflict hands back a snapshot it is the WHOLE current version: every published path with its bytes and the media type sent, plus the metadata", requiresActions: ["publish"] }, async (h) => {
    const extra = h.caps.multiFile ? [script(h, 1)] : [];
    const a = await publishOk(h, req(page(h, 1, extra), { title: "Conformance title" }), "fresh");
    const current = page(h, 2, extra);
    const b = await publishOk(h, req(current, { url: a.url, baseVersion: a.version, title: "Conformance title" }), "update to v2");
    const out = await publish(h, req(page(h, 3, extra), { url: a.url, baseVersion: a.version }));
    check(!out.ok && out.code === "conflict", "a stale base must conflict");
    if (out.ok || out.code !== "conflict")
        return "";
    if (out.current === "reread")
        return `host answers "reread" (explicit; snapshot completeness not claimed) at ${b.version}`;
    for (const f of current) {
        const got = out.current.files.find((x) => x.path === f.path);
        check(got !== undefined, `snapshot lacks published path ${f.path}`);
        check(got !== undefined && same(got.content, f.content), `snapshot bytes differ for ${f.path}`);
        check(got !== undefined && got.mediaType === f.mediaType, `snapshot media type for ${f.path} is ${got?.mediaType}, expected ${f.mediaType}`);
    }
    check(out.current.files.length === current.length, `snapshot carries ${out.current.files.length} files, the current version has ${current.length}`);
    check(out.current.title === "Conformance title", `snapshot title is ${out.current.title}, expected the current title`);
    return `snapshot complete: ${current.length} file(s) + metadata`;
});
define({ id: "publish.force_overrides_stale_base", describe: "`force: true` over a stale `baseVersion` applies (a plain published version is overwritable)", requiresActions: ["publish"] }, async (h) => {
    const a = await publishOk(h, req(page(h, 1)), "fresh");
    const b = await publishOk(h, req(page(h, 2), { url: a.url, baseVersion: a.version }), "update to v2");
    const forced = await publishOk(h, req(page(h, 3), { url: a.url, baseVersion: a.version, force: true }), "forced publish over a stale base");
    check(forced.version !== b.version, "force answered ok but the version did not advance");
    return `forced ${b.version} → ${forced.version}`;
});
define({ id: "publish.force_refused_over_page_saved_version", describe: "`force: true` over a version saved from inside the page ⇒ `force_refused` (the one version force must not discard)", requiresActions: ["publish"], requiresHook: "pageSavedVersion" }, async (h) => {
    const a = await publishOk(h, req(page(h, 1)), "fresh");
    await h.hooks.pageSavedVersion(a);
    const out = await publish(h, req(page(h, 2), { url: a.url, baseVersion: a.version, force: true }));
    check(!out.ok && out.code === "force_refused", `force over a page-saved version answered ${out.ok ? "ok" : out.code}, expected force_refused`);
    return "force_refused";
});
define({ id: "publish.attempt_replay_is_idempotent", describe: "the same `attemptId` replays the first outcome: a fresh publish never mints a second page, an update never advances twice", requiresActions: ["publish"] }, async (h) => {
    const before = h.caps.actions.includes("list") ? (await listMine(h)).length : undefined;
    const key = `${h.runId}-replay-fresh`;
    const a = await publishOk(h, req(page(h, 1), { title: "first" }), "fresh under key", key);
    const again = await publishOk(h, req(page(h, 9), { title: "changed" }), "replay under the same key", key);
    check(again.url === a.url && again.version === a.version, `the replay answered ${again.url} @ ${again.version}, expected ${a.url} @ ${a.version}`);
    if (before !== undefined) {
        const after = (await listMine(h)).length;
        check(after === before + 1, `two fresh calls under one key left ${after - before} new page(s) in the listing, expected 1`);
    }
    const key2 = `${h.runId}-replay-update`;
    const u1 = await publishOk(h, req(page(h, 2), { url: a.url, baseVersion: a.version }), "update under key2", key2);
    const u2 = await publishOk(h, req(page(h, 3), { url: a.url, baseVersion: a.version }), "replayed update under key2", key2);
    check(u2.version === u1.version, `the replayed update answered version ${u2.version}, expected ${u1.version}`);
    await publishOk(h, req(page(h, 4), { url: a.url, baseVersion: u1.version }), "an update on the replayed version (proves the version did not advance twice)");
    return `fresh and update each replayed once on ${a.url}`;
});
define({ id: "publish.attempt_replay_of_refusal", describe: "a typed refusal is replayed under the same `attemptId` too: a later call under that key with a request that would otherwise SUCCEED still gets the first refusal (the host does not re-decide)", requiresActions: ["publish"] }, async (h) => {
    const key = `${h.runId}-replay-refusal`;
    const a = await publish(h, req(page(h, 1), { url: FOREIGN_URL }), key);
    check(!a.ok && a.code === "url_foreign", `a foreign url answered ${a.ok ? "ok" : a.code}, expected url_foreign`);
    const b = await publish(h, req(page(h, 2)), key);
    check(!b.ok && b.code === "url_foreign", `the replay re-decided: a fresh request under the refused key answered ${b.ok ? "ok" : b.code}, expected the replayed url_foreign`);
    return "the refusal replayed over a request that would otherwise succeed";
});
define({ id: "publish.url_foreign_refused", describe: "a `url` this host never minted ⇒ `url_foreign` (a typed refusal, never a throw, never a page)", requiresActions: ["publish"] }, async (h) => {
    const out = await publish(h, req(page(h, 1), { url: FOREIGN_URL }));
    check(!out.ok && out.code === "url_foreign", `a foreign url answered ${out.ok ? "ok" : out.code}, expected url_foreign`);
    return "url_foreign";
});
define({ id: "publish.atomic_under_precommit_fault", describe: "a failure injected BEFORE the commit point leaves no observable change: no phantom version (the same base still applies) and, when list is declared, no new page", requiresActions: ["publish"], requiresHook: "precommitFault" }, async (h) => {
    const a = await publishOk(h, req(page(h, 1)), "fresh");
    const before = h.caps.actions.includes("list") ? (await listMine(h)).length : undefined;
    const contentBefore = h.caps.actions.includes("read") ? await h.host.read({ url: a.url }, ctxOf(h)) : undefined;
    await h.hooks.precommitFault.arm();
    let faulted;
    try {
        const out = await h.host.publish(req(page(h, 2), { url: a.url, baseVersion: a.version }), ctxOf(h));
        faulted = out.ok ? "ok" : "typed";
    }
    catch {
        faulted = "threw";
    }
    finally {
        await h.hooks.precommitFault.disarm();
    }
    check(faulted !== "ok", "the armed fault did not fail the publish — the hook is not reaching the host's write path");
    if (contentBefore !== undefined) {
        const contentAfter = await h.host.read({ url: a.url }, ctxOf(h));
        check(contentBefore.ok && contentAfter.ok && contentAfter.content === contentBefore.content && contentAfter.ref.version === contentBefore.ref.version, "the faulted publish changed the readable content or version before any commit");
    }
    const after = await publishOk(h, req(page(h, 3), { url: a.url, baseVersion: a.version }), "update on the pre-fault base");
    check(after.version !== a.version, "the update after the fault did not advance");
    if (before !== undefined)
        check((await listMine(h)).length === before, "the faulted publish left a new page in the listing");
    return `fault ${faulted}; base ${a.version} still applied → ${after.version}`;
});
define({ id: "publish.byte_limits_enforced", describe: "one file over `maxFileBytes` ⇒ `payload_too_large`; with multiFile, files each within the per-file limit but summing past `maxTotalBytes` ⇒ `payload_too_large`", requiresActions: ["publish"] }, async (h) => {
    const big = await publish(h, req([{ path: "index.html", content: new Uint8Array(h.caps.maxFileBytes + 1), mediaType: HTML_MEDIA }]));
    check(!big.ok && big.code === "payload_too_large", `a file of maxFileBytes+1 answered ${big.ok ? "ok" : big.code}`);
    if (!h.caps.multiFile)
        return "per-file limit enforced (single-file host)";
    const files = [];
    let left = h.caps.maxTotalBytes + 1;
    for (let i = 0; left > 0; i++) {
        const size = Math.min(left, h.caps.maxFileBytes);
        files.push({ path: i === 0 ? "index.html" : `part${i}.bin`, content: new Uint8Array(size), mediaType: i === 0 ? HTML_MEDIA : "application/octet-stream" });
        left -= size;
    }
    const total = await publish(h, req(files));
    check(!total.ok && total.code === "payload_too_large", `${files.length} files summing to maxTotalBytes+1 answered ${total.ok ? "ok" : total.code}`);
    return `per-file and total limits enforced (${files.length} files)`;
});
define({ id: "publish.at_limit_accepted", describe: "a file of exactly `maxFileBytes` (within the total) is accepted; with multiFile, files summing to exactly `maxTotalBytes` are accepted", requiresActions: ["publish"], requiresLargePayloads: true }, async (h) => {
    const fileSize = Math.min(h.caps.maxFileBytes, h.caps.maxTotalBytes);
    await publishOk(h, req([{ path: "index.html", content: new Uint8Array(fileSize), mediaType: HTML_MEDIA }]), `a file of exactly ${fileSize} bytes`);
    if (!h.caps.multiFile)
        return `${fileSize} bytes accepted`;
    const files = [];
    let left = h.caps.maxTotalBytes;
    for (let i = 0; left > 0; i++) {
        const size = Math.min(left, h.caps.maxFileBytes);
        files.push({ path: i === 0 ? "index.html" : `part${i}.bin`, content: new Uint8Array(size), mediaType: i === 0 ? HTML_MEDIA : "application/octet-stream" });
        left -= size;
    }
    await publishOk(h, req(files), `${files.length} files summing to exactly maxTotalBytes`);
    return `at-limit file and at-limit total accepted`;
});
define({ id: "list.scope_words", describe: "`mine` lists only the caller's pages (including the ones just published), `shared` only others', `all` both; `truncated` is a boolean on every answer", requiresActions: ["publish", "list"] }, async (h) => {
    const a = await publishOk(h, req(page(h, 1)), "fresh");
    const b = await publishOk(h, req(page(h, 2)), "fresh");
    const mine = await h.host.list({ scope: "mine", limit: 1000 }, ctxOf(h));
    const shared = await h.host.list({ scope: "shared", limit: 1000 }, ctxOf(h));
    const all = await h.host.list({ scope: "all", limit: 1000 }, ctxOf(h));
    for (const [scope, out] of [["mine", mine], ["shared", shared], ["all", all]]) {
        check(out.ok, `list ${scope} refused`);
        check(out.ok && typeof out.truncated === "boolean", `list ${scope}: truncated is not a boolean`);
    }
    if (!mine.ok || !shared.ok || !all.ok)
        return "";
    check(mine.rows.every((r) => r.owner === "mine"), "a `mine` row is not owned by the caller");
    check(shared.rows.every((r) => r.owner === "shared"), "a `shared` row is owned by the caller");
    check([a.url, b.url].every((u) => mine.rows.some((r) => r.url === u)), "the pages just published are missing from `mine`");
    check([a.url, b.url].every((u) => all.rows.some((r) => r.url === u)), "the pages just published are missing from `all`");
    check(all.rows.length >= mine.rows.length, "`all` lists fewer rows than `mine`");
    return `mine ${mine.rows.length}, shared ${shared.rows.length}, all ${all.rows.length}`;
});
define({ id: "list.limit_and_truncated", describe: "`limit` bounds the rows and `truncated` says so when more exist", requiresActions: ["publish", "list"] }, async (h) => {
    await publishOk(h, req(page(h, 1)), "fresh");
    await publishOk(h, req(page(h, 2)), "fresh");
    const out = await h.host.list({ scope: "mine", limit: 1 }, ctxOf(h));
    check(out.ok, "list refused");
    if (!out.ok)
        return "";
    check(out.rows.length <= 1, `limit 1 answered ${out.rows.length} rows`);
    check(out.truncated === true, "two pages exist but `truncated` is not true at limit 1");
    return "limit honoured, truncated true";
});
define({ id: "read.own_page_full_content", describe: "`read` of the caller's own single-file page returns its full text with `isolatedSummary: false` and the page's ref", requiresActions: ["publish", "read"] }, async (h) => {
    const files = page(h, 1);
    const a = await publishOk(h, req(files), "fresh");
    const out = await h.host.read({ url: a.url }, ctxOf(h));
    check(out.ok, `read refused: ${out.ok ? "" : out.code}`);
    if (!out.ok)
        return "";
    check(out.isolatedSummary === false, "the caller's own page came back as an isolated summary");
    check(out.content === new TextDecoder().decode(files[0].content), "the content differs from the published text");
    check(out.ref.url === a.url, `read.ref.url is ${out.ref.url}, expected ${a.url}`);
    return "full text, own page";
});
define({ id: "read.unknown_url_typed", describe: "`read` of a url this host never minted ⇒ a typed `not_found` / `forbidden`, never a throw", requiresActions: ["read"] }, async (h) => {
    let out;
    try {
        out = await h.host.read({ url: FOREIGN_URL }, ctxOf(h));
    }
    catch (err) {
        return fail(`read threw: ${thrownText(err)}`);
    }
    check(!out.ok && (out.code === "not_found" || out.code === "forbidden"), `an unknown url answered ${out.ok ? "ok" : out.code}`);
    return out.ok ? "" : out.code;
});
define({ id: "files.list_and_read_file", describe: "`list_files` names every published path with its byte count; `read_file` returns each file's bytes and media type; an unknown path ⇒ `not_found`", requiresActions: ["publish", "list_files", "read_file"] }, async (h) => {
    const files = page(h, 1, h.caps.multiFile ? [script(h, 2)] : []);
    const a = await publishOk(h, req(files), "fresh");
    const listed = await h.host.listFiles({ url: a.url }, ctxOf(h));
    check(listed.ok, `list_files refused: ${listed.ok ? "" : listed.code}`);
    if (!listed.ok)
        return "";
    for (const f of files) {
        const e = listed.entries.find((x) => x.path === f.path);
        check(e !== undefined && e.bytes === f.content.byteLength, `list_files lacks ${f.path} at ${f.content.byteLength} bytes`);
        const got = await h.host.readFile({ url: a.url, path: f.path }, ctxOf(h));
        check(got.ok, `read_file ${f.path} refused: ${got.ok ? "" : got.code}`);
        if (!got.ok)
            return "";
        check(same(got.file.content, f.content) && got.file.mediaType === f.mediaType, `read_file ${f.path}: bytes or media type differ`);
    }
    const missing = await h.host.readFile({ url: a.url, path: "no/such/file.txt" }, ctxOf(h));
    check(!missing.ok && missing.code === "not_found", `an unknown path answered ${missing.ok ? "ok" : missing.code}`);
    return `${files.length} file(s) listed and read`;
});
export const ARTIFACT_CONFORMANCE_VECTORS = vectorTable.map(({ run: _run, ...v }) => Object.freeze(v));
const applies = (v, caps) => v.requiresActions.every((a) => caps.actions.includes(a)) && (v.requiresMultiFile !== true || caps.multiFile);
export function artifactConformanceVectors(host) {
    const verdict = readArtifactHostCapabilities(host);
    if (!verdict.ok)
        throw new Error(`artifactConformanceVectors: the host's declaration is invalid — ${verdict.problem}`);
    return ARTIFACT_CONFORMANCE_VECTORS.filter((v) => applies(v, verdict.capabilities));
}
export async function runArtifactConformance(host, opts = {}) {
    const verdict = readArtifactHostCapabilities(host);
    if (!verdict.ok)
        throw new Error(`runArtifactConformance: the host's declaration is invalid — ${verdict.problem}`);
    const h = { host, caps: verdict.capabilities, runId: opts.runId ?? randomUUID(), hooks: opts.hooks ?? {}, largePayloads: opts.largePayloads === true, seq: 0 };
    host.onMount?.(h.runId);
    const results = [];
    for (const v of vectorTable) {
        if (!applies(v, h.caps)) {
            results.push({ id: v.id, status: "not_applicable", detail: `requires ${[...v.requiresActions, ...(v.requiresMultiFile ? ["multiFile"] : [])].join(" + ")}; not declared` });
            continue;
        }
        if (v.requiresHook !== undefined && h.hooks[v.requiresHook] === undefined) {
            results.push({ id: v.id, status: "skipped", detail: `hook ${v.requiresHook} not supplied` });
            continue;
        }
        if (v.requiresLargePayloads === true && !h.largePayloads) {
            results.push({ id: v.id, status: "skipped", detail: "largePayloads not enabled" });
            continue;
        }
        let row;
        try {
            row = { id: v.id, status: "pass", detail: await v.run(h) };
        }
        catch (err) {
            row = { id: v.id, status: "fail", detail: err instanceof Violation ? err.message : `threw: ${thrownText(err)}` };
        }
        results.push(row);
    }
    const count = (s) => results.filter((r) => r.status === s).length;
    return { capabilities: h.caps, runId: h.runId, results, passed: count("pass"), failed: count("fail"), notApplicable: count("not_applicable"), skipped: count("skipped") };
}