UNPKG

open-agreements

Version:

Open-source legal template filling CLI and library

422 lines • 18 kB
import AdmZip from 'adm-zip'; import { writeFileSync } from 'node:fs'; import { DOMParser, XMLSerializer } from '@xmldom/xmldom'; import { replaceParagraphTextRange } from '@usejunior/docx-core'; import { copyEntriesSkippingDirs, enumerateTextParts, getGeneralTextPartNames, preserveXmlSpace } from './ooxml-parts.js'; import { BLANK_PLACEHOLDER } from '../fill-utils.js'; const W_NS = 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'; /** * Normalize residual bracket artifacts in generated fieldSelector documents. * * Only operates when declarative rules are provided (via normalize.json). * Without declarative rules, returns zero-change stats (no-op). * Uses formatting-preserving replaceParagraphTextRange for all text mutations. */ export async function normalizeBracketArtifacts(inputPath, outputPath, options) { const zip = new AdmZip(inputPath); const parser = new DOMParser(); const serializer = new XMLSerializer(); const parts = enumerateTextParts(zip); const partNames = getGeneralTextPartNames(parts); const stats = { unbracketedSegments: 0, removedSegments: 0, removedParagraphs: 0, normalizedParagraphs: 0, formattingFallbackCount: 0, declarativeRuleApplications: 0, declarativeRuleMatchCounts: {}, declarativeRuleMutationCounts: {}, declarativeRuleExpectationFailures: [], declarativeRuleExamples: [], }; const rules = options?.rules ?? []; const fieldValues = options?.fieldValues ?? {}; const blankPlaceholder = options?.blankPlaceholder ?? BLANK_PLACEHOLDER; const hasDeclarativeRules = rules.length > 0; // No declarative rules → no-op. Brackets remain visible for lawyers. if (!hasDeclarativeRules) { const outZip = new AdmZip(); copyEntriesSkippingDirs(zip, outZip); writeFileSync(outputPath, outZip.toBuffer()); return stats; } for (const rule of rules) { if (!(rule.id in stats.declarativeRuleMatchCounts)) { stats.declarativeRuleMatchCounts[rule.id] = 0; stats.declarativeRuleMutationCounts[rule.id] = 0; } } for (const partName of partNames) { const entry = zip.getEntry(partName); if (!entry) continue; const xml = entry.getData().toString('utf-8'); const doc = parser.parseFromString(xml, 'text/xml'); const paragraphs = doc.getElementsByTagNameNS(W_NS, 'p'); let lastHeading = ''; for (let i = 0; i < paragraphs.length; i++) { const para = paragraphs[i]; const original = extractParagraphText(para); if (!original.trim()) continue; const trimmedOriginal = original.trim(); if (isHeadingLike(trimmedOriginal)) { lastHeading = trimmedOriginal; } const declarativeResult = applyDeclarativeRulesToParagraph({ text: original, heading: lastHeading, rules, fieldValues, blankPlaceholder, }); if (declarativeResult.applied) { stats.declarativeRuleApplications += 1; } if (declarativeResult.rule_id) { const ruleId = declarativeResult.rule_id; stats.declarativeRuleMatchCounts[ruleId] = (stats.declarativeRuleMatchCounts[ruleId] ?? 0) + 1; if (declarativeResult.text !== original) { stats.declarativeRuleMutationCounts[ruleId] = (stats.declarativeRuleMutationCounts[ruleId] ?? 0) + 1; } const existingExamples = stats.declarativeRuleExamples.filter((example) => example.rule_id === ruleId).length; if (existingExamples < 2) { stats.declarativeRuleExamples.push({ rule_id: ruleId, heading: lastHeading, before: original, after: declarativeResult.text, }); } } let finalText = declarativeResult.text; // Sanitize double-dollar artifacts ($$ or $ $) → single $ finalText = finalText.replace(/\$[\s\u00A0\t]*\$/g, '$'); if (finalText !== original) { try { const replacement = applyEditHunks(para, original, finalText); para.parentNode.replaceChild(replacement, para); } catch (error) { const reason = error instanceof Error ? error.message : String(error); throw new Error(`Normalization refused in ${partName}, paragraph ${i}: ${reason}`); } stats.normalizedParagraphs += 1; } } zip.updateFile(partName, Buffer.from(serializer.serializeToString(doc), 'utf-8')); } for (const rule of rules) { const actualMatches = stats.declarativeRuleMatchCounts[rule.id] ?? 0; if (rule.expected_min_matches !== undefined && actualMatches < rule.expected_min_matches) { stats.declarativeRuleExpectationFailures.push(`${rule.id}: expected at least ${rule.expected_min_matches} match(es), found ${actualMatches}`); } if (rule.expected_max_matches !== undefined && actualMatches > rule.expected_max_matches) { stats.declarativeRuleExpectationFailures.push(`${rule.id}: expected at most ${rule.expected_max_matches} match(es), found ${actualMatches}`); } } const outZip = new AdmZip(); copyEntriesSkippingDirs(zip, outZip); writeFileSync(outputPath, outZip.toBuffer()); return stats; } function codePoints(text) { const points = Array.from(text), offsets = [0]; for (const point of points) { const code = point.charCodeAt(0); if (point.length === 1 && code >= 0xD800 && code <= 0xDFFF) throw new Error('Unpaired surrogate in normalization text'); offsets.push(offsets[offsets.length - 1] + point.length); } return { points, offsets }; } /** * Return exact deletion hunks when `newText` is a subsequence of `oldText`. * The leftmost embedding is deterministic and, unlike a wide replacement, * never crosses text that survives the transformation (including REF field * results). The caller still verifies the reconstructed text before mutation. */ export function computeDeletionHunks(oldText, newText, offset = 0) { if (newText.length >= oldText.length) return null; const old = codePoints(oldText), next = codePoints(newText); const left = []; let oldIndex = 0; for (const char of next.points) { const found = old.points.indexOf(char, oldIndex); if (found < 0) return null; left.push(found); oldIndex = found + 1; } const retained = new Set(left); const hunks = []; let start = -1; for (let i = 0; i <= old.points.length; i++) { if (i < old.points.length && !retained.has(i)) { if (start < 0) start = i; } else if (start >= 0) { hunks.push({ start: offset + old.offsets[start], end: offset + old.offsets[i], replacement: '' }); start = -1; } } return hunks; } // At most 16 MB of alignment cells per changed paragraph. Larger ambiguous // transformations fail visibly rather than falling back to a wide style rewrite. const MAX_ALIGNMENT_CELLS = 4_000_000; /** * Compute DISJOINT edit hunks turning `oldText` into `newText`, in ascending * `start` order. Exact codepoint LCS retains even one-character interior anchors; * returned offsets remain UTF-16 offsets for the DOCX mutation API. Ties prefer * deletion, making repeated-text alignment deterministic. */ export function computeEditHunks(oldText, newText, offset = 0) { const old = codePoints(oldText), next = codePoints(newText); const deletionHunks = computeDeletionHunks(oldText, newText, offset); if (deletionHunks) return deletionHunks; let start = 0; while (start < Math.min(old.points.length, next.points.length) && old.points[start] === next.points[start]) start++; let oldEnd = old.points.length, newEnd = next.points.length; while (oldEnd > start && newEnd > start && old.points[oldEnd - 1] === next.points[newEnd - 1]) { oldEnd--; newEnd--; } if (oldEnd === start && newEnd === start) return []; const a = old.points.slice(start, oldEnd), b = next.points.slice(start, newEnd); if (!a.length || !b.length) return [{ start: offset + old.offsets[start], end: offset + old.offsets[oldEnd], replacement: b.join('') }]; const width = b.length + 1, cells = (a.length + 1) * width; if (cells > MAX_ALIGNMENT_CELLS) throw new Error(`Exact normalization alignment budget exceeded (${cells} cells)`); const lengths = new Uint32Array(cells); for (let i = a.length - 1; i >= 0; i--) { for (let j = b.length - 1; j >= 0; j--) { lengths[i * width + j] = a[i] === b[j] ? 1 + lengths[(i + 1) * width + j + 1] : Math.max(lengths[(i + 1) * width + j], lengths[i * width + j + 1]); } } const hunks = []; let i = 0, j = 0, current; while (i < a.length || j < b.length) { if (i < a.length && j < b.length && a[i] === b[j]) { if (current) hunks.push(current); current = undefined; i++; j++; } else { current ??= { start: offset + old.offsets[start + i], end: offset + old.offsets[start + i], replacement: '' }; if (i < a.length && (j === b.length || lengths[(i + 1) * width + j] >= lengths[i * width + j + 1])) { current.end = offset + old.offsets[start + ++i]; } else current.replacement += b[j++]; } } if (current) hunks.push(current); return hunks; } /** * Apply the disjoint hunks turning `original` into `finalText` to a paragraph, * highest-offset-first so earlier offsets stay valid as text length changes. * Stage all edits on a clone. Never commit a partly-mutated paragraph or flatten * formatting when a field/container boundary is unsupported. */ function applyEditHunks(para, original, finalText) { const hunks = computeEditHunks(original, finalText); let replay = original; for (const hunk of hunks.slice().reverse()) replay = replay.slice(0, hunk.start) + hunk.replacement + replay.slice(hunk.end); if (replay !== finalText) throw new Error('Exact normalization replay mismatch'); let staged = para.cloneNode(true); for (let i = hunks.length - 1; i >= 0; i--) { const hunk = hunks[i]; if (replaceRangeInSingleSafeTextNode(staged, hunk)) continue; const attempt = staged.cloneNode(true); try { replaceParagraphTextRange(attempt, hunk.start, hunk.end, hunk.replacement); staged = attempt; } catch { throw new Error('Unsupported normalization field or container boundary'); } } if (extractParagraphText(staged) !== finalText) throw new Error('Normalized paragraph text does not match planned output'); return staged; } /** * Preserve the existing run for a range wholly contained in one ordinary w:t node. * Safe-docx intentionally rejects a boundary that it maps to the preceding * field-result run; direct mutation is nevertheless safe when the requested * characters are demonstrably in the following non-field text node. This * preserves the run, its rPr, and all REF field machinery. */ function replaceRangeInSingleSafeTextNode(para, hunk) { if (hunk.end <= hunk.start) return false; const textNodes = []; let visibleOffset = 0; const fieldPhases = []; const visit = (node, unsafeAncestor = false) => { if (node.nodeType !== 1) return; const element = node; const unsafe = unsafeAncestor || (element.namespaceURI === W_NS && ['fldSimple', 'ins', 'del', 'moveFrom', 'moveTo'].includes(element.localName ?? '')); if (element.namespaceURI === W_NS && element.localName === 'fldChar') { const type = element.getAttributeNS(W_NS, 'fldCharType') || element.getAttribute('w:fldCharType'); if (type === 'begin') fieldPhases.push('instruction'); else if (type === 'separate' && fieldPhases.length > 0) fieldPhases[fieldPhases.length - 1] = 'result'; else if (type === 'end') fieldPhases.pop(); return; } if (element.namespaceURI === W_NS && element.localName === 't') { if (fieldPhases.at(-1) === 'instruction') return; const text = element.textContent ?? ''; textNodes.push({ node: element, start: visibleOffset, end: visibleOffset + text.length, unsafe, fieldResult: fieldPhases.length > 0, }); visibleOffset += text.length; return; } for (let child = element.firstChild; child; child = child.nextSibling) visit(child, unsafe); }; visit(para); const target = textNodes.find(({ start, end }) => hunk.start >= start && hunk.end <= end); if (target?.unsafe) throw new Error('Unsupported normalization simple field or revision ancestry'); if (!target || target.fieldResult) return false; const text = target.node.textContent ?? ''; const localStart = hunk.start - target.start; const localEnd = hunk.end - target.start; target.node.textContent = text.slice(0, localStart) + hunk.replacement + text.slice(localEnd); if ((target.node.textContent ?? '').startsWith(' ') || (target.node.textContent ?? '').endsWith(' ')) { preserveXmlSpace(target.node); } return true; } function isHeadingLike(text) { return !text.startsWith('.') && !text.includes('[') && text.replaceAll('.', '').trim().length > 0; } function applyDeclarativeRulesToParagraph(params) { const { text, heading, rules, fieldValues, blankPlaceholder } = params; let mutated = text; let matchedRule = false; let matchedRuleId; for (const rule of rules) { if (!matchesHeading(rule, heading)) continue; if (!mutated.includes(rule.paragraph_contains)) continue; if (rule.paragraph_end_contains && !mutated.includes(rule.paragraph_end_contains)) continue; matchedRule = true; matchedRuleId = rule.id; for (const [token, template] of Object.entries(rule.replacements ?? {})) { const resolved = resolveTemplateValue(template, fieldValues, blankPlaceholder); mutated = mutated.split(token).join(resolved); } if (rule.trim_unmatched_trailing_bracket) { const openCount = (mutated.match(/\[/g) ?? []).length; const closeCount = (mutated.match(/\]/g) ?? []).length; if (closeCount > openCount) { mutated = mutated.replace(/\]+$/g, ''); } } mutated = mutated .replace(/\s{2,}/g, ' ') .replace(/\s+\./g, '.') .replace(/^\.\s*\.\s*/, '. ') .trim(); break; } // Lightweight fallback cleanup for declarative mode: // 1) Preserve bracket-prefixed heading labels by removing only leading '['. // 2) Trim unmatched trailing ']' artifacts that remain after option pruning. mutated = normalizeBracketPrefixedHeading(mutated); mutated = trimUnmatchedTrailingBrackets(mutated); return { text: mutated, applied: matchedRule || mutated !== text, rule_id: matchedRuleId }; } function matchesHeading(rule, heading) { if (rule.ignore_heading) return true; if (heading.includes(rule.section_heading)) return true; for (const alias of rule.section_heading_any ?? []) { if (heading.includes(alias)) return true; } return false; } function resolveTemplateValue(template, fieldValues, blankPlaceholder) { return template.replace(/\{([a-zA-Z0-9_]+)\}/g, (_full, key) => { const value = fieldValues[key]; if (typeof value === 'boolean') return value ? 'true' : 'false'; if (typeof value === 'string' && value.trim().length > 0) return value; if (Array.isArray(value) && value.every((entry) => typeof entry === 'string') && value.length > 0) { return value.join(', '); } return blankPlaceholder; }); } function normalizeBracketPrefixedHeading(text) { const trimmed = text.trim(); if (!trimmed.startsWith('[')) return text; if (trimmed.includes(']')) return text; if (trimmed.length > 90) return text; if (trimmed.includes('_')) return text; if (/[.:;!?].+[.:;!?]/.test(trimmed)) return text; if (!/^\[[A-Z][A-Za-z0-9 ,&()''/.-]+$/.test(trimmed)) return text; return trimmed.slice(1).trim(); } function trimUnmatchedTrailingBrackets(text) { let out = text; const openCount = (out.match(/\[/g) ?? []).length; let closeCount = (out.match(/\]/g) ?? []).length; while (closeCount > openCount && /\]\s*$/.test(out)) { out = out.replace(/\]\s*$/, '').trimEnd(); closeCount -= 1; } return out; } function extractParagraphText(para) { const tElements = para.getElementsByTagNameNS(W_NS, 't'); const parts = []; for (let i = 0; i < tElements.length; i++) { parts.push(tElements[i].textContent ?? ''); } return parts.join(''); } //# sourceMappingURL=bracket-normalizer.js.map