@sema-agent/core/dist/core/redaction-passes.js

Stateless, task-oriented AI agent core

export function summarizeRedactions(findings) {
    if (findings.length === 0)
        return "no redactions";
    const counts = new Map();
    for (const f of findings) {
        const label = `${f.kind} (${f.confidence})`;
        counts.set(label, (counts.get(label) ?? 0) + 1);
    }
    const parts = [...counts].map(([label, n]) => `${label} x${n}`);
    return `${findings.length} redaction${findings.length === 1 ? "" : "s"}: ${parts.join(", ")}`;
}
function passZeroMapper(batches) {
    const cursors = batches.map(() => ({ i: 0, delta: 0 }));
    return (pos) => {
        let p = pos;
        for (let b = batches.length - 1; b >= 0; b--) {
            const edits = batches[b];
            const c = cursors[b];
            let mapped;
            while (mapped === undefined && c.i < edits.length) {
                const e = edits[c.i];
                const outStart = e.at + c.delta;
                if (p < outStart)
                    mapped = p - c.delta;
                else if (p < outStart + e.insertedLen)
                    mapped = e.at;
                else {
                    c.delta += e.insertedLen - e.removedLen;
                    c.i++;
                }
            }
            p = mapped ?? p - c.delta;
        }
        return p;
    };
}
const PREEXISTING_MARKER_RE = /\[redacted(?:-[a-z]+)?\]/g;
const FORMAT_CHAR_RE = /\p{Cf}/u;
const MAX_FORMAT_SLOT_SCANS = 64;
const REDACTED_WHOLE = "[redacted]";
function finding(pass, span, family) {
    const f = { kind: pass.kind, confidence: pass.confidence, span, marker: pass.marker, source: pass.source };
    if (family !== undefined)
        f.family = family;
    return f;
}
function insertedFor(pass, match, groups) {
    return typeof pass.replace === "string"
        ? pass.replace.replace(/\$(\d)/g, (_m, d) => groups[Number(d) - 1] ?? "")
        : pass.replace(match, ...groups);
}
function removedRegion(pass, match, groups, inserted) {
    if (typeof pass.replace === "string") {
        const parts = pass.replace.split(/(\$\d)/).filter((p) => p !== "");
        const isRef = (p) => /^\$\d$/.test(p);
        const refText = (p) => groups[Number(p[1]) - 1] ?? "";
        let head = "";
        for (const p of parts) {
            if (!isRef(p))
                break;
            head += refText(p);
        }
        let tail = "";
        for (let i = parts.length - 1; i >= 0 && isRef(parts[i]); i--)
            tail = refText(parts[i]) + tail;
        if (parts.every(isRef))
            return [0, match.length];
        if (!match.startsWith(head))
            head = "";
        if (!match.endsWith(tail) || head.length + tail.length > match.length)
            tail = "";
        return [head.length, match.length - tail.length];
    }
    const markerAt = pass.marker === "" ? -1 : inserted.lastIndexOf(pass.marker);
    const headStop = markerAt < 0 ? inserted.length : markerAt;
    const tailStop = markerAt < 0 ? inserted.length : inserted.length - markerAt - pass.marker.length;
    let head = 0;
    while (head < match.length && head < headStop && match[head] === inserted[head])
        head++;
    let tail = 0;
    while (tail < match.length - head && tail < inserted.length - head && tail < tailStop && match[match.length - 1 - tail] === inserted[inserted.length - 1 - tail])
        tail++;
    return [head, match.length - tail];
}
const SOLO_SCAN_WORK_PER_CHAR = 8;
const SOLO_SCAN_WORK_FLOOR = 1 << 17;
function soloHits(input, passes) {
    const hits = [];
    passes.forEach((pass, i) => {
        const re = new RegExp(pass.re.source, pass.re.flags.includes("g") ? pass.re.flags : `${pass.re.flags}g`);
        const rescans = pass.family !== undefined;
        let work = Math.max(SOLO_SCAN_WORK_PER_CHAR * input.length, SOLO_SCAN_WORK_FLOOR);
        for (let m = re.exec(input); m !== null; m = re.exec(input)) {
            const at = m.index;
            const familyAt = () => (typeof pass.family === "function" ? pass.family(input, at) : pass.family);
            if (rescans) {
                work -= m[0].length;
                if (work < 0) {
                    hits.push({ start: at, end: input.length, marker: pass.marker === "" ? REDACTED_WHOLE : pass.marker, pass: i, family: familyAt(), matchStart: at, matchEnd: input.length });
                    break;
                }
                re.lastIndex = at + 1;
            }
            else
                re.lastIndex = at + Math.max(m[0].length, 1);
            const inserted = insertedFor(pass, m[0], m.slice(1));
            if (inserted === m[0])
                continue;
            const [a, b] = removedRegion(pass, m[0], m.slice(1), inserted);
            if (b <= a)
                continue;
            const family = familyAt();
            hits.push({ start: at + a, end: at + b, marker: inserted.slice(a, inserted.length - (m[0].length - b)), pass: i, family, matchStart: at, matchEnd: at + m[0].length });
        }
    });
    return hits;
}
function unremovedHits(hits, removals) {
    const gone = [];
    for (const r of [...removals].filter((x) => x.end > x.start).sort((x, y) => x.start - y.start || y.end - x.end)) {
        const last = gone[gone.length - 1];
        if (last !== undefined && r.start <= last[1])
            last[1] = Math.max(last[1], r.end);
        else
            gone.push([r.start, r.end]);
    }
    return hits.filter((h) => {
        let lo = 0;
        let hi = gone.length - 1;
        let at = -1;
        while (lo <= hi) {
            const mid = (lo + hi) >> 1;
            if (gone[mid][0] <= h.start) {
                at = mid;
                lo = mid + 1;
            }
            else
                hi = mid - 1;
        }
        return at === -1 || gone[at][1] < h.end;
    });
}
function sequentialRemovals(batches) {
    const removals = [];
    for (let j = 0; j < batches.length; j++) {
        const toPassZero = passZeroMapper(batches.slice(0, j));
        for (const e of batches[j]) {
            const start = toPassZero(e.at + e.head);
            const end = toPassZero(e.at + e.removedLen - e.tail);
            removals.push({ start, end: Math.max(start, end), marker: e.inserted.slice(e.head, e.insertedLen - e.tail), pass: e.pass, family: e.family });
        }
    }
    return removals;
}
function renderRemovals(input, removals) {
    const sorted = removals.filter((r) => r.end > r.start || r.marker !== "").sort((x, y) => x.start - y.start || y.end - x.end);
    const edits = [];
    let out = "";
    let cursor = 0;
    for (let i = 0; i < sorted.length;) {
        const start = sorted[i].start;
        let end = sorted[i].end;
        let widest;
        for (; i < sorted.length && (sorted[i].start < end || sorted[i].start === start || (sorted[i].marker === "" && sorted[i].start === end)); i++) {
            const r = sorted[i];
            if (r.end > end)
                end = r.end;
            if (r.marker !== "" && (widest === undefined || r.end - r.start > widest.end - widest.start))
                widest = r;
        }
        const marker = widest?.marker ?? REDACTED_WHOLE;
        out += input.slice(cursor, start) + marker;
        cursor = end;
        edits.push({ at: start, removedLen: end - start, insertedLen: marker.length, inserted: marker, family: widest?.family, head: 0, tail: 0, pass: widest?.pass ?? sorted[i - 1].pass });
    }
    return { out: out + input.slice(cursor), edits };
}
function runPasses(input, passes, findings, toOriginal) {
    const batches = [];
    let cur = input;
    passes.forEach((pass, passIndex) => {
        const edits = [];
        const passInput = cur;
        const toPassZero = passZeroMapper(batches.slice());
        cur = cur.replace(pass.re, (...args) => {
            const match = args[0];
            const offset = args[args.length - 2];
            const groups = args.slice(1, -2);
            const inserted = insertedFor(pass, match, groups);
            if (inserted === match)
                return inserted;
            const family = typeof pass.family === "function" ? pass.family(passInput, offset) : pass.family;
            if (findings !== undefined)
                findings.push({ pass: passIndex, f: finding(pass, [toOriginal(toPassZero(offset)), toOriginal(toPassZero(offset + match.length))], family) });
            const [a, b] = removedRegion(pass, match, groups, inserted);
            edits.push({ at: offset, removedLen: match.length, insertedLen: inserted.length, inserted, family, head: a, tail: match.length - b, pass: passIndex });
            return inserted;
        });
        batches.push(edits);
    });
    return { out: cur, batches };
}
function applyPasses(input, passes, report, toOriginal, disjoint = false) {
    const found = report === undefined ? undefined : [];
    const plain = runPasses(input, passes, found, toOriginal);
    const solo = soloHits(input, passes);
    const sequential = solo.length === 0 ? [] : sequentialRemovals(plain.batches);
    const outer = unremovedHits(solo, sequential);
    const mappable = !disjoint || replacedSpans(plain.batches) !== undefined;
    const whole = outer.length === 0 && mappable ? undefined : renderRemovals(input, [...sequential, ...outer]);
    if (whole === undefined || (whole.out === plain.out && mappable)) {
        if (report !== undefined)
            report.findings.push(...found.map((x) => x.f));
        return plain;
    }
    if (report !== undefined) {
        const seen = new Set(found.map((x) => `${x.pass}:${x.f.span[0]}:${x.f.span[1]}`));
        for (const h of outer) {
            const span = [toOriginal(h.matchStart), toOriginal(h.matchEnd)];
            const key = `${h.pass}:${span[0]}:${span[1]}`;
            if (seen.has(key))
                continue;
            seen.add(key);
            found.push({ pass: h.pass, f: finding(passes[h.pass], span, h.family) });
        }
        found.sort((x, y) => x.pass - y.pass || x.f.span[0] - y.f.span[0]);
        report.findings.push(...found.map((x) => x.f));
    }
    return { out: whole.out, batches: [whole.edits] };
}
function replacedSpans(batches) {
    const spans = [];
    for (let j = 0; j < batches.length; j++) {
        const toPassZero = passZeroMapper(batches.slice(0, j));
        for (const e of batches[j]) {
            const s0 = toPassZero(e.at);
            const e0 = toPassZero(e.at + e.removedLen);
            if (e0 - s0 !== e.removedLen)
                return undefined;
            spans.push({ start: s0, end: e0, inserted: e.inserted, pass: e.pass, family: e.family });
        }
    }
    spans.sort((a, b) => a.start - b.start);
    for (let i = 1; i < spans.length; i++)
        if (spans[i].start < spans[i - 1].end)
            return undefined;
    return spans;
}
export function runRedactionPasses(input, passes, report) {
    if (report !== undefined && report.preexistingMarkers === undefined) {
        report.preexistingMarkers = input.match(PREEXISTING_MARKER_RE)?.length ?? 0;
    }
    if (!FORMAT_CHAR_RE.test(input))
        return applyPasses(input, passes, report, (pos) => pos).out;
    const viewChars = [];
    const viewToOriginal = [];
    const formatAtSlot = [];
    const charStartOffsets = [];
    {
        let orig = 0;
        let viewOff = 0;
        let slotBuf = "";
        for (const ch of input) {
            if (FORMAT_CHAR_RE.test(ch)) {
                slotBuf += ch;
            }
            else {
                formatAtSlot.push(slotBuf);
                slotBuf = "";
                viewToOriginal.push(orig);
                charStartOffsets.push(viewOff);
                viewChars.push(ch);
                viewOff += ch.length;
            }
            orig += ch.length;
        }
        formatAtSlot.push(slotBuf);
        viewToOriginal.push(input.length);
        charStartOffsets.push(viewOff);
    }
    const view = viewChars.join("");
    const offsetToCharIndex = new Map();
    charStartOffsets.forEach((off, idx) => offsetToCharIndex.set(off, idx));
    const toOriginal = (pos) => {
        const idx = offsetToCharIndex.get(pos);
        return idx === undefined ? input.length : viewToOriginal[idx];
    };
    const { batches } = applyPasses(view, passes, report, toOriginal, true);
    const viewSpans = replacedSpans(batches);
    if (viewSpans === undefined)
        return REDACTED_WHOLE;
    const spans = viewSpans.map((sp) => {
        const startIdx = offsetToCharIndex.get(sp.start) ?? viewChars.length;
        const endIdx = offsetToCharIndex.get(sp.end) ?? viewChars.length;
        const lastIdx = endIdx - 1;
        const endOrig = lastIdx >= startIdx && lastIdx < viewChars.length ? viewToOriginal[lastIdx] + viewChars[lastIdx].length : viewToOriginal[startIdx];
        return { start: viewToOriginal[startIdx], end: endOrig, inserted: sp.inserted };
    });
    const viewOffsetToOriginal = (viewStart, viewEnd) => {
        const startIdx = offsetToCharIndex.get(viewStart) ?? viewChars.length;
        const endIdx = offsetToCharIndex.get(viewEnd) ?? viewChars.length;
        const lastIdx = endIdx - 1;
        const endOrig = lastIdx >= startIdx && lastIdx < viewChars.length ? viewToOriginal[lastIdx] + viewChars[lastIdx].length : viewToOriginal[startIdx];
        return { start: viewToOriginal[startIdx], end: endOrig };
    };
    const cutOffsets = [];
    for (let k = 1; k < viewChars.length; k++)
        if (formatAtSlot[k] !== "")
            cutOffsets.push(charStartOffsets[k]);
    if (cutOffsets.length > MAX_FORMAT_SLOT_SCANS)
        return REDACTED_WHOLE;
    const extra = [];
    const rawSpans = replacedSpans(applyPasses(input, passes, undefined, (pos) => pos, true).batches);
    if (rawSpans !== undefined)
        extra.push(...rawSpans);
    for (const cut of cutOffsets) {
        const cutSpans = replacedSpans(applyPasses(view.slice(cut), passes, undefined, (pos) => pos, true).batches);
        if (cutSpans === undefined)
            continue;
        for (const c of cutSpans) {
            const o = viewOffsetToOriginal(c.start + cut, c.end + cut);
            extra.push({ ...c, start: o.start, end: o.end });
        }
    }
    extra.sort((a, b) => a.start - b.start);
    const firstEndingAfter = (pos) => {
        let lo = 0;
        let hi = spans.length;
        while (lo < hi) {
            const mid = (lo + hi) >> 1;
            if (spans[mid].end > pos)
                hi = mid;
            else
                lo = mid + 1;
        }
        return lo;
    };
    for (const r of extra) {
        const at = firstEndingAfter(r.start);
        const hit = spans[at];
        if (hit !== undefined && hit.start < r.end) {
            if (r.start < hit.start)
                hit.start = r.start;
            if (r.end > hit.end)
                hit.end = r.end;
            let next = at + 1;
            while (next < spans.length && spans[next].start < hit.end)
                hit.end = Math.max(hit.end, spans[next++].end);
            if (next > at + 1)
                spans.splice(at + 1, next - at - 1);
            continue;
        }
        spans.splice(at, 0, { start: r.start, end: r.end, inserted: r.inserted });
        if (report !== undefined)
            report.findings.push(finding(passes[r.pass], [r.start, r.end], r.family));
    }
    let result = "";
    let cursor = 0;
    for (const sp of spans) {
        result += input.slice(cursor, sp.start) + sp.inserted;
        cursor = sp.end;
    }
    result += input.slice(cursor);
    return result;
}
const BRACKET_MARKER = { "url-credentials": "[redacted-credentials]" };
const bracketMarker = (kind) => BRACKET_MARKER[kind] ?? "[redacted]";
const capturesLabel = (re) => new RegExp(`${re.source}|`).exec("").length > 1;
export function egressPasses(groups, replacers = {}) {
    return groups.map((rows) => {
        const head = rows[0];
        const marker = bracketMarker(head.kind);
        const re = rows.length === 1 ? head.match : new RegExp(rows.map((r) => `(?:${r.match.source})`).join("|"), head.match.flags);
        const sticky = rows.map((r) => ({ id: r.id, re: new RegExp(r.match.source, r.match.flags.replace("g", "") + "y") }));
        const family = rows.length === 1 ? head.id : (input, at) => sticky.find((s) => ((s.re.lastIndex = at), s.re.test(input)))?.id ?? head.id;
        return { kind: head.kind, confidence: head.confidence, marker, source: head.source, re, replace: replacers[head.kind] ?? (capturesLabel(head.match) ? `$1${marker}` : marker), family };
    });
}