scai
Version:
> **A local-first AI CLI for understanding, querying, and iterating on large codebases.** > **100% local • No token costs • No cloud • No prompt injection • Private by design**
649 lines (648 loc) • 28.3 kB
JavaScript
// File: src/modules/evidenceVerifierStep.ts
import fs from "fs";
import path from "path";
import { logInputOutput } from "../utils/promptLogHelper.js";
const STOPWORDS = new Set([
"the", "and", "for", "with", "from", "that", "this", "are", "was", "were",
"has", "have", "had", "not", "but", "can", "could", "should", "would", "into",
"onto", "about", "above", "below", "under", "over", "then", "else", "when",
"where", "what", "which", "while", "return", "const", "let", "var", "true",
"false", "null", "undefined", "new", "set", "get", "in", "to", "of", "on",
"at", "by", "how", "does", "here", "work", "works", "or", "and", "not"
]);
const WEAK_TOKENS = new Set([
"file", "line", "move", "update", "change", "modify", "readme", "fix", "code"
]);
const GENERIC_TOKENS = new Set([
"defined", "define", "location", "where", "find", "code"
]);
const FILE_EXT_REGEX = /\.(ts|tsx|js|jsx|mjs|cjs|md)$/i;
const MAX_SENTENCE_TARGETS = 8;
const MAX_SENTENCE_MATCHES_PER_TARGET = 2;
const MAX_SENTENCE_EVIDENCE_PER_FILE = 16;
const MAX_EVIDENCE_ITEMS_PER_FILE = 24;
function clamp(value, min = 0, max = 1) {
return Math.max(min, Math.min(max, value));
}
function escapeRegex(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
function uniq(values) {
return Array.from(new Set(values));
}
function normalizeToken(token) {
return token.toLowerCase();
}
function stemToken(token) {
if (token.endsWith("ies") && token.length > 4) {
return `${token.slice(0, -3)}y`;
}
if (token.endsWith("es") && token.length > 4) {
return token.slice(0, -2);
}
if (token.endsWith("s") && token.length > 3) {
return token.slice(0, -1);
}
return token;
}
function tokenizeText(text) {
return (text.match(/\b[a-zA-Z_][a-zA-Z0-9_]{1,}\b/g) ?? []).map(normalizeToken);
}
function expandToken(token) {
const normalized = normalizeToken(token);
const stemmed = stemToken(normalized);
const variants = new Set([normalized, stemmed]);
if (normalized === "database" || stemmed === "database") {
variants.add("db");
variants.add("sqlite");
variants.add("sql");
variants.add("schema");
}
if (normalized === "query" || normalized === "queries" || stemmed === "query") {
variants.add("sql");
variants.add("select");
variants.add("insert");
variants.add("update");
variants.add("delete");
variants.add("template");
variants.add("templates");
}
if (normalized === "defined" || stemmed === "define") {
variants.add("template");
variants.add("schema");
}
return Array.from(variants).filter((value) => value.length >= 2);
}
function buildSnippet(lines, lineIndex) {
return lines
.slice(Math.max(0, lineIndex - 1), Math.min(lines.length, lineIndex + 2))
.join("\n");
}
function isCommentLikeOrStringOnly(line) {
const trimmed = line.trim();
if (!trimmed)
return false;
if (trimmed.startsWith("//") ||
trimmed.startsWith("/*") ||
trimmed.startsWith("*") ||
trimmed.startsWith("#")) {
return true;
}
if (/^['"`].*['"`][,;]?$/.test(trimmed)) {
return true;
}
return false;
}
function extractTargets(query, expansionTerms = []) {
const isLikelySentenceTarget = (target) => {
const trimmed = target.trim();
if (trimmed.length < 8)
return false;
if (!/\s/.test(trimmed))
return false;
const tokens = tokenizeText(trimmed);
if (tokens.length === 0)
return false;
const informativeTokens = tokens.filter((token) => !STOPWORDS.has(token) &&
!GENERIC_TOKENS.has(token) &&
!WEAK_TOKENS.has(token));
return informativeTokens.length > 0;
};
const quotedSentenceTargets = [];
const quotedRegex = /['"`](.+?)['"`]/g;
let quoteMatch;
while ((quoteMatch = quotedRegex.exec(query)) !== null) {
const target = quoteMatch[1].trim();
if (isLikelySentenceTarget(target)) {
quotedSentenceTargets.push(target);
}
}
const heuristicSentenceTargets = [];
if (!quotedSentenceTargets.length) {
query
.split(/[\.\n]/)
.map((segment) => segment.trim())
.filter((segment) => segment.length > 12)
.filter((segment) => !/^[\s{[]*["'`][^"'`]+["'`]\s*:/.test(segment))
.filter((segment) => isLikelySentenceTarget(segment))
.forEach((segment) => heuristicSentenceTargets.push(segment));
}
const sentenceTargets = uniq([...quotedSentenceTargets, ...heuristicSentenceTargets]).slice(0, MAX_SENTENCE_TARGETS);
const filenameTargets = uniq(query
.split(/\s+/)
.map((word) => word.replace(/^['"`]|[,'"`)]+$/g, "").trim())
.filter((word) => FILE_EXT_REGEX.test(word)));
const explicitPathTargets = uniq(filenameTargets
.filter((target) => target.includes("/") || target.includes("\\"))
.map((target) => target.replace(/\\/g, "/")));
const baseNameTargets = uniq(filenameTargets.map((name) => path.basename(name.replace(FILE_EXT_REGEX, ""))));
const tokens = tokenizeText(query).filter((token) => token.length >= 3);
const normalizedExpansionTerms = expansionTerms
.map((token) => normalizeToken(token))
.filter((token) => token.length >= 3);
const mergedTokens = uniq([...tokens, ...normalizedExpansionTerms]);
const expandedTokens = uniq(mergedTokens.flatMap((token) => expandToken(token)));
const symbolTargets = uniq(mergedTokens.filter((token) => {
const normalized = token;
if (STOPWORDS.has(normalized))
return false;
if (quotedSentenceTargets.some((sentence) => sentence.toLowerCase().includes(normalized))) {
return false;
}
if (filenameTargets.includes(token))
return false;
if (baseNameTargets.includes(token))
return false;
return token.length >= 3;
}));
const queryTokenSet = new Set(expandedTokens
.filter((token) => !STOPWORDS.has(token)));
const pathSignalTokenSet = new Set(expandedTokens.filter((token) => !STOPWORDS.has(token) &&
!WEAK_TOKENS.has(token) &&
!GENERIC_TOKENS.has(token) &&
token.length >= 2));
return {
sentenceTargets,
filenameTargets,
explicitPathTargets,
baseNameTargets,
symbolTargets,
queryTokenSet,
pathSignalTokenSet
};
}
function extractIdentifierTargets(query) {
const identifiers = new Set();
const rawTokens = query.match(/[a-zA-Z_][a-zA-Z0-9_]*/g) ?? [];
for (const token of rawTokens) {
if (token.length < 3)
continue;
if (FILE_EXT_REGEX.test(token))
continue;
if (!/[A-Z_]/.test(token))
continue;
identifiers.add(normalizeToken(token));
}
return identifiers;
}
function computeSymbolConfidence(symbol, sentenceTargets) {
const normalized = normalizeToken(symbol);
if (sentenceTargets.some((target) => target.toLowerCase().includes(normalized))) {
return 0.95;
}
if (WEAK_TOKENS.has(normalized)) {
return 0.45;
}
return 0.85;
}
function dedupeEvidence(items) {
const bestByKey = new Map();
for (const item of items) {
const ev = item.evidence;
const spanKey = `${ev.span?.startLine ?? 0}:${ev.span?.endLine ?? 0}`;
const key = `${ev.type}|${ev.claim.toLowerCase()}|${spanKey}`;
const existing = bestByKey.get(key);
if (!existing || (ev.confidence ?? 0) > (existing.evidence.confidence ?? 0)) {
bestByKey.set(key, item);
}
}
return Array.from(bestByKey.values());
}
function capEvidence(items) {
const typePriority = {
filename: 5,
structural: 4,
symbol: 3,
sentence: 2,
"keyword-cluster": 1
};
return [...items]
.sort((a, b) => {
const priorityDelta = typePriority[b.evidence.type] - typePriority[a.evidence.type];
if (priorityDelta !== 0)
return priorityDelta;
const confidenceDelta = (b.evidence.confidence ?? 0) - (a.evidence.confidence ?? 0);
if (confidenceDelta !== 0)
return confidenceDelta;
return a.line - b.line;
})
.slice(0, MAX_EVIDENCE_ITEMS_PER_FILE);
}
function computeProximityAdjustment(lines) {
if (lines.length < 2)
return 0;
const sorted = uniq(lines.map((line) => String(line))).map(Number).sort((a, b) => a - b);
let minGap = Number.POSITIVE_INFINITY;
for (let i = 1; i < sorted.length; i++) {
minGap = Math.min(minGap, sorted[i] - sorted[i - 1]);
}
let adjustment = 0;
if (minGap <= 2)
adjustment += 0.08;
else if (minGap <= 5)
adjustment += 0.05;
else if (minGap <= 10)
adjustment += 0.02;
else if (minGap > 30)
adjustment -= 0.06;
const spread = sorted[sorted.length - 1] - sorted[0];
if (sorted.length >= 3 && spread > 120) {
adjustment -= 0.04;
}
return adjustment;
}
function computeFileConfidence(evidenceWithMeta, symbolTargets) {
const weights = {
sentence: 1.0,
filename: 0.9,
symbol: 0.7,
structural: 0.75,
"keyword-cluster": 0.4
};
const caps = {
sentence: 1.3,
filename: 0.9,
symbol: 1.1,
structural: 1.1,
"keyword-cluster": 0.5
};
const grouped = new Map();
for (const item of evidenceWithMeta) {
const type = item.evidence.type;
if (!grouped.has(type))
grouped.set(type, []);
grouped.get(type)?.push(item);
}
let baseSum = 0;
for (const [type, items] of grouped.entries()) {
const sorted = [...items].sort((a, b) => (b.evidence.confidence ?? 0) - (a.evidence.confidence ?? 0));
let typeScore = 0;
sorted.forEach((item, index) => {
const diminishing = index === 0 ? 1 : 1 / (1 + index * 1.4);
const quality = clamp(item.evidence.confidence ?? 0.8);
typeScore += weights[type] * quality * diminishing;
});
baseSum += Math.min(typeScore, caps[type]);
}
const normalizedBase = clamp(1 - Math.exp(-baseSum / 1.8));
const distinctTypeCount = new Set(evidenceWithMeta.map((item) => item.evidence.type)).size;
const coverageBoost = distinctTypeCount <= 1
? 0
: distinctTypeCount === 2
? 0.05
: distinctTypeCount === 3
? 0.1
: 0.14;
const matchedSymbols = new Set(evidenceWithMeta
.filter((item) => (item.evidence.type === "symbol" || item.evidence.type === "structural") && item.token)
.map((item) => normalizeToken(item.token)));
const tokenCoverageBoost = symbolTargets.length > 0
? 0.12 * (matchedSymbols.size / symbolTargets.length)
: 0;
const symbolLikeEvidence = evidenceWithMeta.filter((item) => item.evidence.type === "symbol" || item.evidence.type === "structural");
const weakEvidenceCount = symbolLikeEvidence.filter((item) => item.weakToken).length;
const genericPenalty = symbolLikeEvidence.length > 0
? 0.2 * (weakEvidenceCount / symbolLikeEvidence.length)
: 0;
const commentLikeSymbolCount = evidenceWithMeta.filter((item) => item.evidence.type === "symbol" && item.commentLike).length;
const commentPenalty = symbolLikeEvidence.length > 0
? 0.15 * (commentLikeSymbolCount / symbolLikeEvidence.length)
: 0;
const proximityLines = evidenceWithMeta
.map((item) => item.line)
.filter((line) => line > 0);
const proximityAdjustment = computeProximityAdjustment(proximityLines);
const final = clamp(normalizedBase +
coverageBoost +
tokenCoverageBoost +
proximityAdjustment -
genericPenalty -
commentPenalty);
return {
baseSum,
normalizedBase,
coverageBoost,
tokenCoverageBoost,
proximityAdjustment,
genericPenalty,
commentPenalty,
final
};
}
function normalizeBm25Score(score, minScore, maxScore) {
if (!Number.isFinite(score) || !Number.isFinite(minScore) || !Number.isFinite(maxScore)) {
return 0.5;
}
if (maxScore === minScore) {
return 0.5;
}
// SQLite BM25 lower is better. Normalize to [0,1] with 1 = best score.
return clamp((maxScore - score) / (maxScore - minScore), 0, 1);
}
/**
* Deterministic evidence verification:
* - Scans candidate files for concrete sentence/symbol/filename/structural matches.
* - Uses identifier-boundary matching to reduce substring false positives.
* - Deduplicates evidence globally per file.
* - Computes confidence with bounded weighted scoring + proximity/coverage/penalties.
*/
export const evidenceVerifierStep = {
name: "evidenceVerifier",
description: "Deterministic evidence-first scan over candidate files to populate fileAnalysis, with calibrated confidence.",
groups: ["analysis"],
run: async (input) => {
var _a, _b, _c, _d;
const query = input.query ?? "";
const context = input.context;
if (!context?.analysis) {
throw new Error("[evidenceVerifier] context.analysis is required.");
}
(_a = context.analysis).fileAnalysis ?? (_a.fileAnalysis = {});
(_b = context.analysis).verify ?? (_b.verify = { byFile: {} });
(_c = context.analysis.verify).byFile ?? (_c.byFile = {});
const uniquePaths = uniq([...(context.initContext?.relatedFiles ?? [])]);
const relatedFileScores = context.initContext?.relatedFileScores ?? {};
const scorePool = uniquePaths
.map((filePath) => relatedFileScores[filePath])
.filter((value) => typeof value === "number" && Number.isFinite(value));
const bm25Min = scorePool.length ? Math.min(...scorePool) : NaN;
const bm25Max = scorePool.length ? Math.max(...scorePool) : NaN;
if (!uniquePaths.length) {
console.warn("[evidenceVerifier] No candidate files to scan.");
return { query, data: {} };
}
const { sentenceTargets, filenameTargets, explicitPathTargets, baseNameTargets, symbolTargets, queryTokenSet, pathSignalTokenSet } = extractTargets(query, context.initContext?.queryExpansionTerms ?? []);
const identifierTargets = extractIdentifierTargets(query);
for (const filePath of uniquePaths) {
let code = "";
try {
code = fs.readFileSync(filePath, "utf-8");
}
catch (err) {
console.warn(`[evidenceVerifier] Failed to read ${filePath}: ${err.message}`);
}
const lines = code ? code.split("\n") : [];
const rawEvidence = [];
const addEvidence = (evidence, line, token, commentLike) => {
rawEvidence.push({
evidence,
line,
token,
weakToken: token ? WEAK_TOKENS.has(normalizeToken(token)) : false,
commentLike
});
};
const loweredSentenceTargets = sentenceTargets.map((target) => target.toLowerCase());
const sentenceTargetMatchCounts = new Map();
let sentenceEvidenceCount = 0;
lines.forEach((line, idx) => {
const loweredLine = line.toLowerCase();
loweredSentenceTargets.forEach((target, targetIndex) => {
if (sentenceEvidenceCount >= MAX_SENTENCE_EVIDENCE_PER_FILE) {
return;
}
const currentCount = sentenceTargetMatchCounts.get(target) ?? 0;
if (currentCount >= MAX_SENTENCE_MATCHES_PER_TARGET) {
return;
}
if (target.length >= 3 && loweredLine.includes(target)) {
const originalTarget = sentenceTargets[targetIndex];
addEvidence({
claim: `Sentence match: "${originalTarget}"`,
type: "sentence",
excerpt: buildSnippet(lines, idx),
span: { startLine: idx + 1, endLine: idx + 1 },
confidence: 1
}, idx + 1);
sentenceTargetMatchCounts.set(target, currentCount + 1);
sentenceEvidenceCount += 1;
}
});
});
for (const symbol of symbolTargets) {
const regex = new RegExp(`\\b${escapeRegex(symbol)}\\b`, "i");
for (let idx = 0; idx < lines.length; idx++) {
const line = lines[idx];
if (!regex.test(line))
continue;
addEvidence({
claim: `Symbol reference found: "${symbol}"`,
type: "symbol",
excerpt: buildSnippet(lines, idx),
span: { startLine: idx + 1, endLine: idx + 1 },
confidence: computeSymbolConfidence(symbol, sentenceTargets)
}, idx + 1, symbol, isCommentLikeOrStringOnly(line));
break;
}
}
const normalizedFilePath = filePath.replace(/\\/g, "/");
const fullFileName = path.basename(filePath);
const baseFileName = fullFileName.replace(FILE_EXT_REGEX, "");
const fileNameTargetNames = new Set(filenameTargets.map((target) => path.basename(target)));
const exactPathMatch = explicitPathTargets.some((target) => normalizedFilePath === target ||
normalizedFilePath.endsWith(`/${target}`));
const fileNameMatch = exactPathMatch ||
fileNameTargetNames.has(fullFileName) ||
baseNameTargets.includes(baseFileName);
if (fileNameMatch) {
addEvidence({
claim: exactPathMatch
? `File path exactly matches query target: "${fullFileName}"`
: `Filename matches query target: "${fullFileName}"`,
type: "filename",
excerpt: `Path: ${filePath}`,
span: { startLine: 0, endLine: 0 },
confidence: 1
}, 0, fullFileName);
}
const filePathTokens = new Set(tokenizeText(filePath).flatMap((token) => expandToken(token)));
const matchingPathTokens = Array.from(pathSignalTokenSet).filter((token) => filePathTokens.has(token));
if (matchingPathTokens.length > 0) {
const pathConfidence = clamp(0.5 + matchingPathTokens.length * 0.08, 0.5, 0.86);
addEvidence({
claim: `Path tokens align with query intent: ${matchingPathTokens.slice(0, 5).join(", ")}`,
type: "keyword-cluster",
excerpt: `Path: ${filePath}`,
span: { startLine: 0, endLine: 0 },
confidence: pathConfidence
}, 0, matchingPathTokens[0]);
}
const struct = context.analysis.fileAnalysis[filePath]?.structural;
const structuralEvidence = [];
if (struct) {
(struct.functions ?? []).forEach((fn) => {
if (!fn.name)
return;
const normalized = normalizeToken(fn.name);
if (!queryTokenSet.has(normalized))
return;
const ev = {
claim: `Function name matches query: "${fn.name}"`,
type: "structural",
excerpt: fn.name,
span: { startLine: fn.start ?? 0, endLine: fn.end ?? 0 },
confidence: WEAK_TOKENS.has(normalized) ? 0.6 : 0.85
};
structuralEvidence.push(ev);
addEvidence(ev, fn.start ?? 0, fn.name);
});
(struct.classes ?? []).forEach((cls) => {
if (!cls.name)
return;
const normalized = normalizeToken(cls.name);
if (!queryTokenSet.has(normalized))
return;
const ev = {
claim: `Class name matches query: "${cls.name}"`,
type: "structural",
excerpt: cls.name,
span: { startLine: cls.start ?? 0, endLine: cls.end ?? 0 },
confidence: WEAK_TOKENS.has(normalized) ? 0.6 : 0.85
};
structuralEvidence.push(ev);
addEvidence(ev, cls.start ?? 0, cls.name);
});
[...(struct.imports ?? []), ...(struct.exports ?? [])].forEach((symbol) => {
if (!symbol)
return;
const normalized = normalizeToken(symbol);
if (!queryTokenSet.has(normalized))
return;
const ev = {
claim: `Import/Export matches query: "${symbol}"`,
type: "structural",
excerpt: symbol,
span: { startLine: 0, endLine: 0 },
confidence: WEAK_TOKENS.has(normalized) ? 0.55 : 0.8
};
structuralEvidence.push(ev);
addEvidence(ev, 0, symbol);
});
if (structuralEvidence.length > 0) {
logInputOutput("evidenceVerifier", "output", {
file: filePath,
count: structuralEvidence.length,
examples: structuralEvidence.slice(0, 5).map((ev) => ({
claim: ev.claim,
excerpt: ev.excerpt,
confidence: ev.confidence
}))
});
}
}
const dedupedEvidence = capEvidence(dedupeEvidence(rawEvidence));
const evidenceItems = dedupedEvidence.map((item) => item.evidence);
const score = computeFileConfidence(dedupedEvidence, symbolTargets);
const hasFilenameHit = evidenceItems.some((ev) => ev.type === "filename");
const hasExactIdentifierHit = identifierTargets.size > 0 &&
dedupedEvidence.some((item) => {
if (item.evidence.type !== "symbol" && item.evidence.type !== "structural") {
return false;
}
const rawToken = item.token ?? item.evidence.excerpt ?? "";
return identifierTargets.has(normalizeToken(rawToken));
});
const bm25Raw = relatedFileScores[filePath];
const bm25Normalized = typeof bm25Raw === "number"
? normalizeBm25Score(bm25Raw, bm25Min, bm25Max)
: 0.5;
// Keep BM25 as a weak retrieval prior only.
const bm25Prior = typeof bm25Raw === "number"
? (bm25Normalized - 0.5) * 0.2
: 0;
const fileConfidence = (hasFilenameHit || hasExactIdentifierHit)
? 1
: clamp(score.final + bm25Prior);
const matchedLines = uniq(dedupedEvidence
.map((item) => (item.line > 0 ? lines[item.line - 1] : ""))
.filter(Boolean));
const isFocusFile = context.analysis.focus?.selectedFiles?.includes(filePath) ?? false;
const hasEvidence = evidenceItems.length > 0;
const isRelevantByConfidence = fileConfidence >= 0.25;
const isRelevant = isFocusFile || isRelevantByConfidence;
if (isRelevant && hasEvidence) {
const confidenceLabel = fileConfidence.toFixed(2);
const verifyRationale = `[confidence:${confidenceLabel}] ${evidenceItems.length} evidence item(s) match the query${isFocusFile ? " (focus file already selected)" : ""}; components base=${score.normalizedBase.toFixed(2)}, coverage=${(score.coverageBoost + score.tokenCoverageBoost).toFixed(2)}, proximity=${score.proximityAdjustment.toFixed(2)}, penalties=${(score.genericPenalty + score.commentPenalty).toFixed(2)}, bm25Prior=${bm25Prior.toFixed(2)}`;
context.analysis.verify.byFile[filePath] = {
fileConfidence,
role: "primary",
isRelevant: true,
rationale: verifyRationale,
evidence: evidenceItems,
searchScore: {
bm25Raw,
bm25Normalized,
bm25Prior,
}
};
context.analysis.fileAnalysis[filePath] = {
...context.analysis.fileAnalysis[filePath],
intent: "relevant",
relevanceExplanation: verifyRationale,
role: "primary",
action: {
isRelevant: true,
shouldModify: hasEvidence
},
proposedChanges: hasEvidence
? {
summary: "Evidence found in file",
scope: "minor",
targets: matchedLines.length ? matchedLines : undefined,
rationale: `calibrated-confidence=${confidenceLabel}`
}
: {
summary: "No evidence found",
scope: "none"
},
semanticAnalyzed: false,
risks: hasEvidence
? []
: ["No concrete evidence found; modification not permitted"],
};
}
else {
context.analysis.verify.byFile[filePath] = {
fileConfidence,
role: "contextual",
isRelevant: false,
rationale: "Insufficient verify evidence for relevance.",
evidence: evidenceItems,
searchScore: {
bm25Raw,
bm25Normalized,
bm25Prior,
}
};
(_d = context.analysis.fileAnalysis)[filePath] || (_d[filePath] = {
intent: "irrelevant",
action: { isRelevant: false, shouldModify: false },
proposedChanges: {
summary: "No evidence found",
scope: "none"
},
semanticAnalyzed: false,
});
}
}
const output = {
query,
data: { fileAnalysis: context.analysis.fileAnalysis }
};
const logSummary = Object.entries(context.analysis.verify.byFile ?? {}).map(([filePath, verify]) => {
const evidenceCount = verify.evidence?.length ?? 0;
const confidenceMatch = verify.rationale?.match(/\[confidence:(\d+\.\d+)\]/);
const confidence = confidenceMatch?.[1] ?? "0.00";
return {
file: filePath,
confidence,
bm25Raw: verify.searchScore?.bm25Raw,
bm25Prior: verify.searchScore?.bm25Prior?.toFixed(2),
evidenceCount,
isRelevant: verify.isRelevant ?? false
};
});
logInputOutput("evidenceVerifier", "output", logSummary);
return output;
}
};