UNPKG

open-agreements

Version:

Open-source legal template filling CLI and library

833 lines • 40.2 kB
import AdmZip from 'adm-zip'; import { DOMParser } from '@xmldom/xmldom'; import { getParagraphText } from '@usejunior/docx-core'; import { cleanConfigRemovesBodyContent } from '../metadata.js'; import { enumerateTextParts, getAllTextPartNames, getGeneralTextPartNames } from './ooxml-parts.js'; import { isParagraphContentEmpty } from './cleaner.js'; import { parseReplacementKey } from './replacement-keys.js'; import { getTableRowContext, normalizeQuotes } from './patcher.js'; import { DOUBLE_PERCENT_PATTERN } from '../fill-utils.js'; const W_NS = 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'; /** * Normalize text for value comparison: * - Convert non-breaking spaces to regular spaces * - Normalize smart quotes to straight quotes * - Collapse runs of spaces/tabs to single space (preserve newlines) * - Trim */ export function normalizeText(text) { return text // Non-breaking spaces .replace(/[\u00A0\u2007\u202F]/g, ' ') // Smart single quotes → straight .replace(/[\u2018\u2019\u2039\u203A]/g, "'") // Smart double quotes → straight .replace(/[\u201C\u201D\u201A\u201E\u00AB\u00BB]/g, '"') // Collapse horizontal whitespace (spaces/tabs) to single space, preserve newlines .replace(/[^\S\n]+/g, ' ') .trim(); } /** * Verify a filled fieldSelector output DOCX: * - All context values appear in the document text * - No unrendered {template_tags} remain * - No leftover [bracketed placeholders] from the replacement map remain * - No doubled currency or percent sigils from a value that already carried the * sign the source supplies * - Footnotes removed (if clean config specified) * - Drafting note paragraphs removed (if clean config specified) */ export async function verifyOutput(outputPath, values, replacements, cleanConfig, cleanedSourcePath) { const checks = []; const rawFullText = extractAllText(outputPath); // Footnotes are outside extractAllText()'s scope. They feed the sigil-doubling // checks only: those read rather than rewrite, and every other check here // keeps the part scope the patcher and cleaner actually operate on. const footnoteText = extractFootnoteText(outputPath); const sigilText = footnoteText ? `${rawFullText}\n${footnoteText}` : rawFullText; const normalizedFullText = normalizeText(rawFullText); const xml = extractDocumentXml(outputPath); // Check 0: Word cross-reference fields are structurally complete and point // at bookmarks that actually exist. Cached field-result text can make a // broken REF look correct until Word or LibreOffice refreshes fields, at // which point it becomes "Error: Reference source not found." Validate the // underlying OOXML instead of relying on renderer logs or cached display // text. This check is fatal because the artifact is not safe to deliver. const refDiagnostics = validateWordFields(outputPath); checks.push({ name: 'Word REF fields resolve', passed: refDiagnostics.length === 0, details: refDiagnostics.length > 0 ? refDiagnostics.join('; ') : undefined, fatal: true, }); // Check 1: All context values present (with normalization) const missingValues = []; for (const [key, value] of Object.entries(values)) { if (typeof value !== 'string' || !value.trim()) continue; // only verify non-empty text values const normalizedValue = normalizeText(value); if (!normalizedFullText.includes(normalizedValue)) { missingValues.push(`${key}="${value}"`); } } checks.push({ name: 'Context values present', passed: missingValues.length === 0, details: missingValues.length > 0 ? `Missing: ${missingValues.join(', ')}` : undefined, }); // Check 3b: Generic signer-facing corruption that is not tied to a recipe's // replacement map. Keep the patterns deliberately narrow; every diagnostic // includes the OOXML story and paragraph so a failure is reproducible. const outputArtifactFindings = findRenderedTextArtifacts(outputPath); const artifactFindings = cleanedSourcePath ? subtractBaselinedArtifacts(outputArtifactFindings, findRenderedTextArtifacts(cleanedSourcePath)) : outputArtifactFindings; checks.push({ name: 'No rendered text artifacts', passed: artifactFindings.length === 0, details: artifactFindings.length > 0 ? `Found: ${artifactFindings.slice(0, 8).join('; ')}` : undefined, }); // Check 2: No unrendered {template_tags} const unrenderedTags = rawFullText.match(/\{[a-z_][a-z0-9_]*\}/gi) ?? []; checks.push({ name: 'No unrendered template tags', passed: unrenderedTags.length === 0, details: unrenderedTags.length > 0 ? `Found: ${unrenderedTags.join(', ')}` : undefined, }); // Check 3: No leftover [bracketed placeholders] from replacement map. // Context-qualified keys ("context > placeholder") are verified at their // qualified location — mirroring the patcher's deterministic anchoring — so an // intentional occurrence of the same bare token in an unrelated context is not // reported as a failed mapped replacement. Simple keys keep whole-document // search (a bare placeholder should not survive anywhere). const leftoverBrackets = findLeftoverPlaceholders(outputPath, replacements, cleanedSourcePath); checks.push({ name: 'Leftover source placeholders', passed: leftoverBrackets.length === 0, details: leftoverBrackets.length > 0 ? `Found: ${leftoverBrackets.join(', ')}` : undefined, }); // Check 4: No footnote references (if removeFootnotes was set) if (cleanConfig?.removeFootnotes) { const footnoteRefs = (xml.match(/(?:footnote|endnote)Reference/g) ?? []).length; checks.push({ name: 'Footnotes removed', passed: footnoteRefs === 0, details: footnoteRefs > 0 ? `${footnoteRefs} note reference(s) remain` : undefined, }); } // Check 5: No double dollar signs ($$ or $ $) // This catches cases where the template already has a $ before a placeholder // and the user also included a $ in their value (e.g. "$1,000,000") const doubleDollarPattern = /\$[\s\u00A0\t]*\$/; const doubleDollarLines = sigilText.split('\n').filter((line) => doubleDollarPattern.test(line)); checks.push({ name: 'No double dollar signs', passed: doubleDollarLines.length === 0, details: doubleDollarLines.length > 0 ? `Found ${doubleDollarLines.length} occurrence(s): "${doubleDollarLines[0].trim().slice(0, 80)}"` : undefined, }); // Check 5b: No double percent signs (%% or % %) // The trailing-sigil twin of Check 5. Recipes disagree about which side of the // seam owns the sign — the ROFR & co-sale replacement key "[specify percentage]%" // absorbs the source %, while the certificate of incorporation and stock // purchase agreement leave it in place — so a sign-carrying value that is // correct for one template doubles the sign in another (issue #719). const doublePercentLines = sigilText .split('\n') .filter((line) => DOUBLE_PERCENT_PATTERN.test(line)); checks.push({ name: 'No double percent signs', passed: doublePercentLines.length === 0, details: doublePercentLines.length > 0 ? `Found ${doublePercentLines.length} occurrence(s): "${doublePercentLines[0].trim().slice(0, 80)}"` : undefined, }); // Check 6: No drafting note paragraphs (if patterns were set) if (cleanConfig?.removeParagraphPatterns && cleanConfig.removeParagraphPatterns.length > 0) { const regexes = cleanConfig.removeParagraphPatterns.map((p) => new RegExp(p, 'i')); const lines = rawFullText.split('\n'); const matchingLines = lines.filter((line) => regexes.some((r) => r.test(line.trim()))); checks.push({ name: 'Drafting notes removed', passed: matchingLines.length === 0, details: matchingLines.length > 0 ? `Found: ${matchingLines[0].trim().slice(0, 80)}...` : undefined, }); } // Check 7: No range-deleted sections (if ranges were set) if (cleanConfig?.removeRanges && cleanConfig.removeRanges.length > 0) { const rangeStartPatterns = cleanConfig.removeRanges.map((r) => new RegExp(r.start, 'i')); const lines = rawFullText.split('\n'); const matchingLines = lines.filter((line) => rangeStartPatterns.some((r) => r.test(line.trim()))); checks.push({ name: 'Range-deleted sections removed', passed: matchingLines.length === 0, details: matchingLines.length > 0 ? `Found: ${matchingLines[0].trim().slice(0, 80)}...` : undefined, }); } // Check 8: No single-character underlined runs adjacent to non-underlined runs // This is a hallmark of quota-based text redistribution corrupting formatting. // When cleanedSourcePath is provided, only flag NEW anomalies introduced by fill/patch. const outputAnomalyCount = countFormattingAnomalies(outputPath); const baselineAnomalyCount = cleanedSourcePath ? countFormattingAnomalies(cleanedSourcePath) : 0; const newAnomalyCount = Math.max(0, outputAnomalyCount - baselineAnomalyCount); checks.push({ name: 'No formatting anomalies', passed: newAnomalyCount === 0, details: newAnomalyCount > 0 ? `Found ${newAnomalyCount} new single-char underlined run(s) adjacent to non-underlined runs (${outputAnomalyCount} total, ${baselineAnomalyCount} in source)` : baselineAnomalyCount > 0 ? `${baselineAnomalyCount} pre-existing anomaly(ies) in source (baselined)` : undefined, }); // Check 9: First body paragraph has content (only when the clean config // removes body content — a removed range/pattern/cover-page can strand an // empty structural paragraph, e.g. one holding a <w:sectPr>, which renders // as a blank first page; see issue #605 / legal-explainer#1800) if (cleanConfig && cleanConfigRemovesBodyContent(cleanConfig)) { const firstParagraphEmpty = hasLeadingBlankPageRisk(outputPath); checks.push({ name: 'First body paragraph has content', passed: !firstParagraphEmpty, details: firstParagraphEmpty ? 'First body paragraph is textless — a cleaning artifact may leave a blank first page/section; consider "removeEmptyLeadingParagraphs": true in clean.json' : undefined, }); } return { passed: checks.every((c) => c.passed), checks, }; } function fieldInstruction(el) { return el.getAttributeNS(W_NS, 'instr') || el.getAttribute('w:instr') || el.getAttribute('instr') || ''; } function refTarget(instruction) { const match = instruction.match(/^\s*REF\s+(?:"([^"]+)"|([^\s\\]+))/i); return match?.[1] ?? match?.[2]; } function describeInstruction(instruction) { const compact = instruction.replace(/\s+/g, ' ').trim(); return compact || '(empty instruction)'; } /** * Return actionable diagnostics for malformed complex Word fields and REF * instructions whose bookmark target is absent. Both atomic `w:fldSimple` * fields and complex begin/instrText/separate/end fields are supported; field * instructions may be split across any number of runs. */ export function validateWordFields(docxPath) { const zip = new AdmZip(docxPath); const parser = new DOMParser(); const partNames = getGeneralTextPartNames(enumerateTextParts(zip)); const bookmarks = new Set(); const docs = []; for (const partName of partNames) { const entry = zip.getEntry(partName); if (!entry) continue; const root = parser.parseFromString(entry.getData().toString('utf-8'), 'text/xml'); docs.push({ partName, root }); const starts = root.getElementsByTagNameNS(W_NS, 'bookmarkStart'); for (let i = 0; i < starts.length; i++) { const name = starts[i].getAttributeNS(W_NS, 'name') || starts[i].getAttribute('w:name'); if (name) bookmarks.add(name); } } const diagnostics = []; const inspectRef = (partName, instruction, malformed) => { const target = refTarget(instruction); if (!target) return; const suffix = malformed ? `; invalid field triplet: ${malformed}` : ''; if (malformed || !bookmarks.has(target)) { diagnostics.push(`${partName}: REF target "${target}" ${malformed ? 'has an invalid field structure' : 'has no matching bookmark'} ` + `(instruction: "${describeInstruction(instruction)}"${suffix})`); } }; for (const { partName, root } of docs) { const simpleFields = root.getElementsByTagNameNS(W_NS, 'fldSimple'); for (let i = 0; i < simpleFields.length; i++) { inspectRef(partName, fieldInstruction(simpleFields[i])); } const stack = []; const visit = (node) => { if (node.nodeType === 1) { const el = node; if (el.namespaceURI === W_NS && el.localName === 'fldChar') { const type = el.getAttributeNS(W_NS, 'fldCharType') || el.getAttribute('w:fldCharType'); if (type === 'begin') { stack.push({ instruction: [], hasSeparate: false, partName }); } else if (type === 'separate') { const frame = stack[stack.length - 1]; if (!frame) diagnostics.push(`${partName}: invalid field triplet: orphan separate marker`); else if (frame.hasSeparate) diagnostics.push(`${partName}: invalid field triplet: duplicate separate marker`); else frame.hasSeparate = true; } else if (type === 'end') { const frame = stack.pop(); if (!frame) { diagnostics.push(`${partName}: invalid field triplet: orphan end marker`); } else { inspectRef(partName, frame.instruction.join(''), frame.hasSeparate ? undefined : 'missing separate marker'); } } return; } if (el.namespaceURI === W_NS && el.localName === 'instrText' && stack.length > 0) { const frame = stack[stack.length - 1]; if (!frame.hasSeparate) frame.instruction.push(el.textContent ?? ''); return; } } for (let child = node.firstChild; child; child = child.nextSibling) visit(child); }; visit(root); for (const frame of stack) { const instruction = frame.instruction.join(''); const target = refTarget(instruction); if (target) inspectRef(partName, instruction, 'missing end marker'); else diagnostics.push(`${partName}: invalid field triplet: missing end marker (instruction: "${describeInstruction(instruction)}")`); } } return [...new Set(diagnostics)]; } /** * Whether the first block-level element of <w:body> is a paragraph with no * visible content (no text and no drawing/picture/object). */ export function isFirstBodyParagraphEmpty(docxPath) { const zip = new AdmZip(docxPath); const entry = zip.getEntry('word/document.xml'); if (!entry) return false; const doc = new DOMParser().parseFromString(entry.getData().toString('utf-8'), 'text/xml'); const body = doc.getElementsByTagNameNS(W_NS, 'body')[0]; if (!body) return false; for (let node = body.firstChild; node; node = node.nextSibling) { if (node.nodeType !== 1) continue; const el = node; if (el.localName === 'p' && el.namespaceURI === W_NS) { return isParagraphContentEmpty(el); } return false; // first block element is a table/sdt/... — has content } return false; } /** * A textless leading paragraph is common Word scaffolding and is not itself a * blank page. Warn only when it also carries explicit page-advancing structure. */ export function hasLeadingBlankPageRisk(docxPath) { const zip = new AdmZip(docxPath); const entry = zip.getEntry('word/document.xml'); if (!entry) return false; const doc = new DOMParser().parseFromString(entry.getData().toString('utf-8'), 'text/xml'); const body = doc.getElementsByTagNameNS(W_NS, 'body')[0]; if (!body) return false; for (let node = body.firstChild; node; node = node.nextSibling) { if (node.nodeType !== 1) continue; const para = node; if (para.localName !== 'p' || para.namespaceURI !== W_NS || !isParagraphContentEmpty(para)) return false; if (para.getElementsByTagNameNS(W_NS, 'pageBreakBefore').length > 0) return true; const breaks = para.getElementsByTagNameNS(W_NS, 'br'); for (let i = 0; i < breaks.length; i++) { const type = breaks[i].getAttributeNS(W_NS, 'type') || breaks[i].getAttribute('w:type'); if (type === 'page') return true; } const sectPr = para.getElementsByTagNameNS(W_NS, 'sectPr')[0]; if (!sectPr) return false; const type = sectPr.getElementsByTagNameNS(W_NS, 'type')[0]; const value = type && (type.getAttributeNS(W_NS, 'val') || type.getAttribute('w:val')); return value === 'nextPage' || value === 'evenPage' || value === 'oddPage'; } return false; } export function findRenderedTextArtifacts(docxPath) { const zip = new AdmZip(docxPath); const parser = new DOMParser(); const findings = []; for (const partName of getAllTextPartNames(enumerateTextParts(zip))) { const entry = zip.getEntry(partName); if (!entry) continue; const doc = parser.parseFromString(entry.getData().toString('utf-8'), 'text/xml'); const paras = doc.getElementsByTagNameNS(W_NS, 'p'); for (let i = 0; i < paras.length; i++) { const text = getParagraphText(paras[i]); if (!text.trim()) continue; const prefix = `${partName}:paragraph ${i + 1}`; const report = (kind, match) => { findings.push(`${prefix} ${kind} ${JSON.stringify(match.slice(0, 60))}`); }; // Unbalanced closing brackets include the observed `]`, `]/`, and `];` // corruption while accepting ordinary balanced citations/placeholders. let depth = 0; for (const char of text) { if (char === '[') depth++; else if (char === ']' && depth > 0) depth--; else if (char === ']') { report('has orphan closing bracket', ']'); break; } } const spacedPunctuation = text.match(/[ \t\u00a0]+[,;:](?=\s|$)/); if (spacedPunctuation) report('has whitespace before punctuation', spacedPunctuation[0]); const doubledPunctuation = text.match(/([,;:])\s*\1/); if (doubledPunctuation) report('has duplicated punctuation', doubledPunctuation[0]); const words = [...text.matchAll(/\b[\p{L}\p{N}][\p{L}\p{N}.-]*\b/gu)]; const allowedSingleRepeats = new Set(['had', 'that', 'very']); for (let w = 1; w < words.length; w++) { if (words[w][0].toLocaleLowerCase() === words[w - 1][0].toLocaleLowerCase() && text.slice((words[w - 1].index ?? 0) + words[w - 1][0].length, words[w].index).trim() === '' && !allowedSingleRepeats.has(words[w][0].toLocaleLowerCase())) { report('has duplicated word', `${words[w - 1][0]} ${words[w][0]}`); break; } } // Repeated two-or-more-token phrases catch duplicated headings such as // "Form S-1 Form S-1" without treating a recurring single legal term as noise. for (let size = 2; size <= 5 && size * 2 <= words.length; size++) { let found = false; for (let w = 0; w + size * 2 <= words.length; w++) { const a = words.slice(w, w + size).map((m) => m[0].toLocaleLowerCase()).join(' '); const b = words.slice(w + size, w + size * 2).map((m) => m[0].toLocaleLowerCase()).join(' '); const boundary = text.slice((words[w + size - 1].index ?? 0) + words[w + size - 1][0].length, words[w + size].index); if (a === b && boundary.trim() === '' && a !== 'time to time') { report('has duplicated phrase', words.slice(w, w + size * 2).map((m) => m[0]).join(' ')); found = true; break; } } if (found) break; } const malformedReference = text.match(/\b(?:under|pursuant to|in accordance with|set forth in|specified in|subject to|see)\s+(?:Sections?|Articles?|Exhibits?|Schedules?)\s+(?=[,.;:)\]]|$|(?:and|or|of|to)\b)/i); if (malformedReference) report('has reference without a target', malformedReference[0]); const missingPercent = text.match(/\b(?:at least|more than|less than|not less than|not more than)\s+\d+(?:\.\d+)?\s+of\b/i); if (missingPercent) report('has percentage threshold without percent sign', missingPercent[0]); } } return findings; } /** Remove source-carried findings as a multiset while retaining output locations. */ function subtractBaselinedArtifacts(output, source) { const signature = (finding) => finding.replace(/^.*?:paragraph \d+ /, ''); const sourceCounts = new Map(); for (const finding of source) { const key = signature(finding); sourceCounts.set(key, (sourceCounts.get(key) ?? 0) + 1); } return output.filter((finding) => { const key = signature(finding); const remaining = sourceCounts.get(key) ?? 0; if (remaining === 0) return true; sourceCounts.set(key, remaining - 1); return false; }); } /** Character ranges in paragraph text carried by complex Word-field results. */ function getComplexFieldResultRanges(para) { const states = []; const ranges = []; let visibleOffset = 0; const visit = (node) => { if (node.nodeType === 1) { const el = node; if (el.namespaceURI === W_NS && el.localName === 'fldChar') { const type = el.getAttributeNS(W_NS, 'fldCharType') || el.getAttribute('w:fldCharType'); if (type === 'begin') states.push('instruction'); else if (type === 'separate' && states.length > 0) states[states.length - 1] = 'result'; else if (type === 'end' && states.length > 0) states.pop(); return; } if (el.namespaceURI === W_NS && el.localName === 't') { const text = el.textContent ?? ''; if (states.includes('result') && text.length > 0) { ranges.push({ start: visibleOffset, end: visibleOffset + text.length }); } visibleOffset += text.length; return; } } for (let child = node.firstChild; child; child = child.nextSibling) visit(child); }; visit(para); return ranges; } /** * Extract each paragraph's normalized text and table-row label context from a * DOCX, using the same normalization (quotes only) the patcher matches on. */ function extractParagraphInfos(docxPath) { const zip = new AdmZip(docxPath); const parser = new DOMParser(); const partNames = getGeneralTextPartNames(enumerateTextParts(zip)); const infos = []; for (const partName of partNames) { const entry = zip.getEntry(partName); if (!entry) continue; const doc = parser.parseFromString(entry.getData().toString('utf-8'), 'text/xml'); const paras = doc.getElementsByTagNameNS(W_NS, 'p'); for (let i = 0; i < paras.length; i++) { const para = paras[i]; const paraText = getParagraphText(para); if (!paraText) continue; const rowContext = getTableRowContext(para); infos.push({ text: normalizeQuotes(paraText), rowContext: rowContext !== null ? normalizeQuotes(rowContext) : null, fieldResultRanges: getComplexFieldResultRanges(para), }); } } return infos; } /** Count non-overlapping occurrences of `needle` in `haystack`. */ function countOccurrences(haystack, needle) { if (needle.length === 0) return 0; let count = 0; let from = 0; let pos; while ((pos = haystack.indexOf(needle, from)) !== -1) { count++; from = pos + needle.length; } return count; } /** * Whether replacement search text is itself a signer-facing source artifact. * * A replacement map can also carry ordinary prose in order to parameterize a * value that may render back to the same visible text (for example, the IRA's * repeated `this Section 2.8` cross-references). Treating every replacement * key as a placeholder makes that valid source-carried prose a false positive. * Keep this check deliberately syntactic: brackets, blank lines, and explicit * merge-token delimiters are artifacts; ordinary legal prose is not. */ function isSourcePlaceholderText(text) { return (/\[[^\]]*\]/.test(text) || /_{2,}/.test(text) || /\{[a-z_][a-z0-9_]*\}/i.test(text) || /<<[^<>]+>>/.test(text) || /«[^»]+»/.test(text)); } /** * Whether a context key has an unfilled placeholder at its qualified location, * mirroring how the patcher targets them: * - Table-row context: a `searchText` survives in a paragraph whose row label * cell contains the context. * - Paragraph context: a `searchText` appears AFTER the first occurrence of the * context in the paragraph (the patcher fills the first placeholder after the * context). * * Used only when no source baseline is available (see findLeftoverPlaceholders). */ function hasQualifiedLeftover(paragraphs, context, searchText) { if (searchText.length === 0) return false; for (const { text, rowContext } of paragraphs) { if (rowContext !== null) { if (rowContext.includes(context) && text.includes(searchText)) return true; } else { const ctxPos = text.indexOf(context); if (ctxPos === -1) continue; if (text.indexOf(searchText, ctxPos + context.length) !== -1) return true; } } return false; } /** Whether the qualified token is still the cached result of a live Word field. */ function hasQualifiedFieldBackedOccurrence(paragraphs, context, searchText) { return paragraphs.some(({ text, rowContext, fieldResultRanges }) => { const contextPos = rowContext !== null ? (rowContext.includes(context) ? 0 : -1) : text.indexOf(context); if (contextPos === -1) return false; const searchPos = text.indexOf(searchText, rowContext !== null ? 0 : contextPos + context.length); if (searchPos === -1) return false; const searchEnd = searchPos + searchText.length; return fieldResultRanges.some(({ start, end }) => start < searchEnd && end > searchPos); }); } /** * Find leftover source placeholders from the replacement map that survive in the * filled output. * * Known limitation (deliberate tradeoff): the count baseline treats ANY * reduction of a context key's token as success, so it cannot distinguish an * occurrence a selector deliberately retains (e.g. the two documented `[___]` in * series_designation) from an unexpected extra occurrence of the same token that * failed to fill — the two are textually identical. Fully distinguishing them * would require machine-readable retained-occurrence metadata in the field * contract (fields/<id>.json), which is out of scope here. Two existing * mechanisms cover the realistic paths this check cannot: (a) an unexpected * extra token means the source changed, which the source_sha256 pin / source * drift canary flags before fill; (b) a DECLARED selector occurrence that fails * to resolve surfaces an `unresolved` selector warning. * * Two key shapes are handled differently, matching how the patcher targets them: * * - Simple keys ("[Company Name]") replace every occurrence, so any surviving * occurrence anywhere in the document is reported. * * - Context-qualified keys ("context > placeholder") target one placeholder per * context, so a bare token may legitimately remain elsewhere (an unrelated * context) or be deliberately retained (a selector field that fills only its * declared occurrences). These are handled by comparing the placeholder's * count against the cleaned source: * - With a source baseline, a context key is reported only when the fill * reduced NOTHING (output count >= source count) — i.e. the mapping was * entirely unhandled (a genuine defect, e.g. a caption the patcher/ * selector never touched), as opposed to a partial reduction that leaves * deliberately-retained or selector-verified occurrences behind. This is * the same deterministic baseline-against-source technique used for * formatting anomalies. Counting is done on the flattened text (the same * text the patcher's replacements ultimately land in) so a context that * is unmatchable due to intra-paragraph line breaks — the #607 caption — * is still caught. * - Without a baseline, the key is reported only if an unfilled placeholder * survives at its qualified location — so an intentional occurrence of the * same token in an unrelated context (e.g. "[s]" in "Management Rights * Letter[s]" while the mapped "Closing > [s]" was filled) is not reported. * * Returns the offending key labels (simple keys as their search text; context * keys as their full "context > placeholder" label). */ export function findLeftoverPlaceholders(docxPath, replacements, cleanedSourcePath) { const simpleSearchTexts = new Set(); const contextKeys = []; for (const key of Object.keys(replacements)) { const parsed = parseReplacementKey(key, ''); if (parsed.type === 'context') { contextKeys.push({ // Match the patcher's quote normalization so the checks anchor on // exactly what the patcher would have targeted. context: normalizeQuotes(parsed.context), searchText: normalizeQuotes(parsed.searchText), label: key, }); } else { simpleSearchTexts.add(parsed.searchText); } } const leftovers = new Set(); const outputFullText = normalizeQuotes(extractAllText(docxPath)); // Simple keys: whole-document search for actual placeholder-shaped source // artifacts. Ordinary prose replacement keys are not placeholders and can // legitimately render back to the same visible text. for (const text of simpleSearchTexts) { if (isSourcePlaceholderText(text) && outputFullText.includes(normalizeQuotes(text))) leftovers.add(text); } // Context keys: count baseline against the cleaned source, else qualified location. if (contextKeys.length > 0) { if (cleanedSourcePath) { const sourceFullText = normalizeQuotes(extractAllText(cleanedSourcePath)); const sourceParagraphs = extractParagraphInfos(cleanedSourcePath); const outputParagraphs = extractParagraphInfos(docxPath); for (const ck of contextKeys) { // An ordinary cross-reference can be a legitimate source-carried value. // Preserve structural checking for Word REF results, which really must // transition to ordinary text when declared for replacement. if (!isSourcePlaceholderText(ck.searchText) && !hasQualifiedFieldBackedOccurrence(sourceParagraphs, ck.context, ck.searchText)) { continue; } const srcCount = countOccurrences(sourceFullText, ck.searchText); if (srcCount === 0) continue; // key does not apply to this document const outCount = countOccurrences(outputFullText, ck.searchText); // Nothing filled for this placeholder → the mapping was entirely // unhandled. A partial reduction leaves only intentionally-retained / // selector-verified occurrences behind, which are not reported. if (outCount >= srcCount) { // A context-qualified replacement may intentionally render the same // visible text as the source (for example, staticizing an atomic REF // field from "Section 6.1" to "Section {section}" and then filling // the tag with "6.1"). Text counts cannot prove that replacement. // In that narrow case, the structural transition from a cached field // result to ordinary text does: do not report the full qualified key // as a leftover once the field at that qualified location is gone. const sourceWasField = hasQualifiedFieldBackedOccurrence(sourceParagraphs, ck.context, ck.searchText); const outputIsField = hasQualifiedFieldBackedOccurrence(outputParagraphs, ck.context, ck.searchText); if (!sourceWasField || outputIsField) leftovers.add(ck.label); } } } else { const outputParagraphs = extractParagraphInfos(docxPath); for (const ck of contextKeys) { if (isSourcePlaceholderText(ck.searchText) && hasQualifiedLeftover(outputParagraphs, ck.context, ck.searchText)) { leftovers.add(ck.label); } } } } return [...leftovers]; } /** * Count formatting anomalies: single-character underlined runs adjacent to * non-underlined runs. This pattern is the hallmark of quota-based text * redistribution corrupting run-level formatting. */ export function countFormattingAnomalies(docxPath) { const zip = new AdmZip(docxPath); const parser = new DOMParser(); const parts = enumerateTextParts(zip); const partNames = getGeneralTextPartNames(parts); let count = 0; for (const partName of partNames) { const entry = zip.getEntry(partName); if (!entry) continue; const xmlContent = entry.getData().toString('utf-8'); const doc = parser.parseFromString(xmlContent, 'text/xml'); const paras = doc.getElementsByTagNameNS(W_NS, 'p'); for (let i = 0; i < paras.length; i++) { const runs = paras[i].getElementsByTagNameNS(W_NS, 'r'); for (let j = 0; j < runs.length; j++) { const run = runs[j]; const tEls = run.getElementsByTagNameNS(W_NS, 't'); let runText = ''; for (let k = 0; k < tEls.length; k++) { runText += tEls[k].textContent ?? ''; } if (runText.length !== 1) continue; // Check if this run is underlined const rPr = run.getElementsByTagNameNS(W_NS, 'rPr'); if (rPr.length === 0) continue; const uEls = rPr[0].getElementsByTagNameNS(W_NS, 'u'); if (uEls.length === 0) continue; // Check adjacent run (next) for non-underlined if (j + 1 < runs.length) { const nextRun = runs[j + 1]; const nextRPr = nextRun.getElementsByTagNameNS(W_NS, 'rPr'); const nextHasUnderline = nextRPr.length > 0 && nextRPr[0].getElementsByTagNameNS(W_NS, 'u').length > 0; if (!nextHasUnderline) { count++; } } } } } return count; } /** * Extract all text from general OOXML text parts (document, headers, footers, endnotes). */ /** * Extract footnote paragraph text, which `extractAllText()` deliberately omits. * * `getGeneralTextPartNames()` excludes `word/footnotes.xml` because the cleaner * has to special-case its separator paragraphs. That constraint is about * mutation; the sigil-doubling checks only read, and a footnote rendering * `8%%` is a corrupt executed document that must not verify clean. */ export function extractFootnoteText(docxPath) { const zip = new AdmZip(docxPath); const parts = enumerateTextParts(zip); if (!parts.footnotes) return ''; const entry = zip.getEntry(parts.footnotes); if (!entry) return ''; const parser = new DOMParser(); const doc = parser.parseFromString(entry.getData().toString('utf-8'), 'text/xml'); const paragraphs = []; const paras = doc.getElementsByTagNameNS(W_NS, 'p'); for (let i = 0; i < paras.length; i++) { const tElements = paras[i].getElementsByTagNameNS(W_NS, 't'); const textParts = []; for (let j = 0; j < tElements.length; j++) { textParts.push(tElements[j].textContent ?? ''); } if (textParts.length > 0) paragraphs.push(textParts.join('')); } return paragraphs.join('\n'); } export function extractAllText(docxPath) { const zip = new AdmZip(docxPath); const parser = new DOMParser(); const parts = enumerateTextParts(zip); const partNames = getGeneralTextPartNames(parts); const allParagraphs = []; for (const partName of partNames) { const entry = zip.getEntry(partName); if (!entry) continue; const xml = entry.getData().toString('utf-8'); const doc = parser.parseFromString(xml, 'text/xml'); const paras = doc.getElementsByTagNameNS(W_NS, 'p'); for (let i = 0; i < paras.length; i++) { const tElements = paras[i].getElementsByTagNameNS(W_NS, 't'); const textParts = []; for (let j = 0; j < tElements.length; j++) { textParts.push(tElements[j].textContent ?? ''); } if (textParts.length > 0) { allParagraphs.push(textParts.join('')); } } } return allParagraphs.join('\n'); } /** * Extract raw XML from word/document.xml only (for footnote ref counting). */ function extractDocumentXml(docxPath) { const zip = new AdmZip(docxPath); const entry = zip.getEntry('word/document.xml'); if (!entry) return ''; return entry.getData().toString('utf-8'); } //# sourceMappingURL=verifier.js.map