open-agreements
Version:
Open-source legal template filling CLI and library
388 lines • 21.5 kB
JavaScript
/**
* Unified fill pipeline — shared orchestration for all three fill paths
* (bundled templates, external/vendored, and fieldSelectors).
*
* Each path is now a thin wrapper that builds PipelineOptions and calls
* runFillPipeline(). The data-prep and fill steps come from fill-pipeline.ts;
* this module handles the surrounding orchestration: temp dir, copy, clean→patch,
* selections, fill, verify, cleanup.
*/
import { readFileSync, writeFileSync, copyFileSync, mkdtempSync, rmSync } from 'node:fs';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import AdmZip from 'adm-zip';
import { DOMParser, XMLSerializer } from '@xmldom/xmldom';
import { getParagraphText, replaceParagraphTextRange } from '@usejunior/docx-core';
import { listCommands } from 'docx-templates';
import { prepareFillData, fillDocx } from './fill-pipeline.js';
import { cleanDocument } from './field-selector/cleaner.js';
import { patchDocument } from './field-selector/patcher.js';
import { applySelectorContracts } from './selectors/index.js';
import { applySelections, classifySelectionMatches, describeSelectionOption } from './selector.js';
import { enumerateTextParts, getGeneralTextPartNames, rezipWithoutDirEntries } from './field-selector/ooxml-parts.js';
import { applyReferenceFieldActions } from './field-selector/reference-fields.js';
const W_NS = 'http://schemas.openxmlformats.org/wordprocessingml/2006/main';
/**
* Static command-reference analysis of the final pre-fill document.
*
* Uses docx-templates listCommands() — the same parser fillDocx's
* createReport() uses — to enumerate {…} commands and collect every
* identifier referenced in INS/IF/FOR command expressions. This is a static
* superset of what the fill will read (it does not execute conditionals),
* NOT runtime value-read telemetry; for the current template corpus all
* command expressions are bare identifiers / `FOR x IN y` / `$x.prop`, so
* the intersection with data keys is exact.
*
* Known accepted imprecision: fillDocx strips drafting-note paragraphs after
* this analysis, so a token that lives only inside a stripped drafting note
* would still be counted.
*/
async function analyzeFillCommands(templateBuf) {
const commands = await listCommands(templateBuf.buffer.slice(templateBuf.byteOffset, templateBuf.byteOffset + templateBuf.byteLength), ['{', '}']);
const referencedIdentifiers = new Set();
for (const command of commands) {
if (command.type !== 'INS' && command.type !== 'IF' && command.type !== 'FOR')
continue;
for (const match of command.code.matchAll(/[A-Za-z_][A-Za-z0-9_]*/g)) {
referencedIdentifiers.add(match[0]);
}
}
return { commandCount: commands.length, referencedIdentifiers };
}
/**
* Run the unified fill pipeline:
* 1. Create temp dir
* 2. Copy inputPath → temp source.docx
* 3. If cleanPatch: clean → patch
* 4. prepareFillData (useBlankPlaceholder: true)
* 5. If selectionsConfig: apply selections
* 6. fillDocx → temp filled.docx
* 7. Copy filled.docx → outputPath
* 8. Run verify(outputPath), warn on failures
* 9. Cleanup temp dir (unless keepIntermediate)
*/
export async function runFillPipeline(options) {
const { inputPath, outputPath, values, fields, priorityFieldNames = [], cleanPatch, selectorManifests, onSelectorResolved, selectionsConfig, selectionsZeroMatchPolicy = 'warn', coerceBooleans = false, computeDisplayFields, fixSmartQuotes = false, verify, prePatchProcess, referenceFieldsConfig, postProcess, keepIntermediate = false, } = options;
const tempDir = mkdtempSync(join(tmpdir(), 'fill-pipeline-'));
const syntheticFieldKeys = new Set(fields
.filter((field) => field.type === 'multiselect' && field.derive_booleans === true)
.flatMap((field) => (field.options ?? []).map((option) => `${option}_enabled`)));
// The cover-notice derived boolean is synthetic — exclude it from fieldsUsed.
if (options.confirmClauses && options.confirmClauses.length > 0) {
syntheticFieldKeys.add('any_confirmation_pending');
}
let stages;
const warnings = [];
try {
// Step 2: Copy source to temp dir
const sourcePath = join(tempDir, 'source.docx');
copyFileSync(inputPath, sourcePath);
// Step 3: Clean → Patch (if configured)
let currentPath = sourcePath;
// Cleaned-but-unfilled source, threaded to verify() as the formatting-anomaly
// baseline so pre-existing source anomalies are not reported as fill defects.
let cleanedSourcePath;
if (cleanPatch) {
const cleanedPath = join(tempDir, 'cleaned.docx');
await cleanDocument(sourcePath, cleanedPath, cleanPatch.cleanConfig);
cleanedSourcePath = cleanedPath;
let structuralInputPath = cleanedPath;
if (prePatchProcess) {
const structurallyProcessedPath = join(tempDir, 'structural.docx');
await prePatchProcess(cleanedPath, structurallyProcessedPath);
structuralInputPath = structurallyProcessedPath;
}
if (referenceFieldsConfig) {
const referenceFieldsPath = join(tempDir, 'reference-fields.docx');
applyReferenceFieldActions(structuralInputPath, referenceFieldsPath, referenceFieldsConfig);
structuralInputPath = referenceFieldsPath;
}
// Selector-contract patch (deterministic locators) runs between clean and
// the legacy patch. It rewrites resolved occurrences to {field_id} tags;
// those fields' keys have already been removed from cleanPatch.replacements
// (the declarative migrated_keys cutover) so each field is patched once.
let patchInputPath = structuralInputPath;
if (selectorManifests && selectorManifests.length > 0) {
const selectedPath = join(tempDir, 'selected.docx');
const selectorResult = await applySelectorContracts(structuralInputPath, selectedPath, selectorManifests);
onSelectorResolved?.(selectorResult.fields);
for (const warning of selectorResult.warnings) {
console.warn(`Selector warning: ${warning}`);
}
patchInputPath = selectedPath;
}
const patchedPath = join(tempDir, 'patched.docx');
const patchResult = await patchDocument(patchInputPath, patchedPath, cleanPatch.replacements);
// Suppress zero-match warnings for keys whose search text appears in
// cleaned content (removeRanges, removeParagraphPatterns, removeBeforePattern).
// These keys target text that was intentionally removed by clean.json.
const cleanPatterns = [
...cleanPatch.cleanConfig.removeParagraphPatterns,
...cleanPatch.cleanConfig.removeRanges.flatMap(r => [r.start, r.end]),
...(cleanPatch.cleanConfig.removeBeforePattern ? [cleanPatch.cleanConfig.removeBeforePattern] : []),
];
const unexpectedZeroMatch = patchResult.zeroMatchKeys.filter(key => {
// Extract just the search text portion (after > for context keys)
const sepIdx = key.indexOf(' > ');
const searchText = sepIdx !== -1 ? key.slice(sepIdx + 3) : key;
// Suppress if any clean pattern relates to this key's search text
return !cleanPatterns.some(p => p.includes(searchText) || searchText.includes(p));
});
if (unexpectedZeroMatch.length > 0) {
warnings.push(`patch: ${unexpectedZeroMatch.length} replacement key(s) had zero matches: ${unexpectedZeroMatch.join(', ')}`);
}
currentPath = patchedPath;
stages = { clean: cleanedPath, patch: patchedPath, fill: '' };
}
else if (prePatchProcess) {
const structurallyProcessedPath = join(tempDir, 'structural.docx');
await prePatchProcess(sourcePath, structurallyProcessedPath);
currentPath = structurallyProcessedPath;
}
// Step 4: Prepare fill data.
// Display-field computation is instrumented so field-usage reporting can
// credit the SOURCE fields consumed while deriving a computed key (e.g.
// party_1_company feeds party_1_company_display, and only the _display
// key appears in the document).
const computeReads = new Set();
const computeWrites = new Set();
const instrumentedCompute = computeDisplayFields
? (d) => {
const proxy = new Proxy(d, {
get(target, prop, receiver) {
if (typeof prop === 'string')
computeReads.add(prop);
return Reflect.get(target, prop, receiver);
},
set(target, prop, value, receiver) {
if (typeof prop === 'string')
computeWrites.add(prop);
return Reflect.set(target, prop, value, receiver);
},
});
computeDisplayFields(proxy);
}
: undefined;
const data = prepareFillData({
values,
fields,
priorityFieldNames,
useBlankPlaceholder: true,
coerceBooleans,
computeDisplayFields: instrumentedCompute,
confirmClauses: options.confirmClauses,
});
// Step 5: Read current buffer; apply selections if configured
let templateBuf = readFileSync(currentPath);
if (selectionsConfig) {
const preSelectPath = join(tempDir, 'pre-select.docx');
const postSelectPath = join(tempDir, 'post-select.docx');
writeFileSync(preSelectPath, templateBuf);
const selectionsResult = await applySelections(preSelectPath, postSelectPath, selectionsConfig, data);
templateBuf = readFileSync(postSelectPath);
// #720: an UNSELECTED selections option that was never actually removed is
// not a no-op. The intent is "delete this alternative", so nothing having
// been deleted means the filled document ships BOTH alternatives —
// legally wrong, and invisible to every other check because the result
// still renders cleanly. Two faults produce it: a marker that matched
// nothing (source drift), and a group that matched but resolved no
// location to act on. `appliedCount` catches both; `matchCount` alone
// catches only the first.
//
// The `engaged group` qualifier is what separates the dangerous case from
// the benign one. A group whose OTHER options matched is demonstrably
// present in this document, so a zero-match sibling means that option's
// marker drifted away from source prose that is still there. A group where
// NOTHING matched is a config/document mismatch (a partial fixture, a
// different source variant); nothing was left half-removed there.
//
// POLICY. The dangerous case is a warning by DEFAULT, not a hard failure,
// and that default is a deliberate concession to the bundled corpus rather
// than a judgement that the condition is tolerable. Turning this guard on
// surfaced three pre-existing marker/document mismatches in bundled
// Common Paper configs (an "Order Form" cover page whose marker says
// "Cover Page"; a straight-vs-curly apostrophe in the workers'-compensation
// insurance marker; a SOW-term marker naming `{term_duration_value}` where
// the patched text carries `{rejection_period_value}`). Those configs live
// under `templates/`, which this repo does not own — it is replaced
// wholesale by an upstream forward sync — so failing closed today would
// refuse to fill half a dozen shipped templates over defects that cannot
// be fixed here. Callers that own their configs can opt into failing
// closed with `selectionsZeroMatchPolicy: 'error'`, and the default should
// flip once the upstream configs are corrected.
const anomalies = classifySelectionMatches(selectionsResult);
if (anomalies.unremovedInEngagedGroup.length > 0) {
const detail = anomalies.unremovedInEngagedGroup
.map((o) => `${describeSelectionOption(o)} [matched ${o.matchCount}, removed ${o.appliedCount}]`)
.join('; ');
const message = `selections: ${anomalies.unremovedInEngagedGroup.length} unselected option(s) were not removed while a ` +
`sibling option in the same group did match. The alternative each was meant to remove is still in the ` +
`document, so the filled output carries BOTH alternatives. A zero match count means the marker text ` +
`drifted from the source; a non-zero match count with nothing removed means the group resolved no ` +
`location to act on (e.g. its options sit in different table cells). Offending option(s): ${detail}`;
if (selectionsZeroMatchPolicy === 'error') {
throw new Error(`[selections] ${message}`);
}
warnings.push(message);
}
if (selectionsZeroMatchPolicy === 'error' && anomalies.selectedZeroMatch.length > 0) {
const detail = anomalies.selectedZeroMatch
.map((o) => `${describeSelectionOption(o)} [matched ${o.matchCount}]`)
.join('; ');
throw new Error(`[selections] selected option(s) are absent from the document; refusing to emit a document that did ` +
`not implement the requested choice: ${detail}`);
}
for (const o of anomalies.unremovedInInertGroup) {
warnings.push(`selections: unselected option ${describeSelectionOption(o)} matched zero paragraphs, and no option in ` +
`that group matched either — the group does not apply to this document.`);
}
for (const o of anomalies.selectedZeroMatch) {
warnings.push(`selections: selected option ${describeSelectionOption(o)} matched zero paragraphs — the alternative ` +
`meant to be kept is absent from the document.`);
}
}
// Step 5.5: Analyze fill commands on the exact buffer handed to fillDocx —
// after clean → selector-contracts → patch → selections, so patch-injected
// tokens (e.g. the yc-safe externals) are counted.
const { commandCount, referencedIdentifiers } = await analyzeFillCommands(templateBuf);
if (commandCount === 0) {
warnings.push('Template artifact contains no machine-fillable fields after clean/patch/selections; ' +
'the document is returned unchanged (manual-fill variant). Provided values were not applied.');
}
// Step 6: Fill
const filledBuf = await fillDocx({
templateBuffer: templateBuf,
data,
fixSmartQuotes,
});
const filledPath = join(tempDir, 'filled.docx');
writeFileSync(filledPath, filledBuf);
if (stages) {
stages.fill = filledPath;
}
// Collapse double spaces left by empty field substitutions (e.g. {initial_word_lower} → "")
await normalizeEmptyFillWhitespaceInDocx(filledPath);
// Step 7: Stage a delivery candidate. Fatal structural verification runs
// against this temporary file before the caller's output path is touched.
const deliveryCandidatePath = join(tempDir, 'delivery-candidate.docx');
copyFileSync(filledPath, deliveryCandidatePath);
// Optional post-processing for field-selector-specific normalization.
if (postProcess) {
await postProcess(deliveryCandidatePath);
// Declarative post-processing can itself remove an optional carrier and
// expose the same empty-value whitespace seam, so normalize once more on
// the final artifact.
await normalizeEmptyFillWhitespaceInDocx(deliveryCandidatePath);
}
// Step 8: Verify. Fatal structural failures refuse delivery; established
// advisory checks continue to surface as warnings for compatibility.
const verifyResult = await verify(deliveryCandidatePath, cleanedSourcePath, referencedIdentifiers);
if (!verifyResult.passed) {
const failedChecks = verifyResult.checks.filter((c) => !c.passed);
const failures = failedChecks
.map((c) => `${c.name}: ${c.details ?? 'failed'}`)
.join('; ');
console.warn(`Warning: verification issues: ${failures}`);
for (const check of failedChecks) {
warnings.push(`verify: ${check.name}: ${check.details ?? 'failed'}`);
}
const fatalChecks = failedChecks.filter((check) => check.fatal);
if (fatalChecks.length > 0) {
throw new Error(`Delivery blocked by fatal verification: ${fatalChecks
.map((check) => `${check.name}: ${check.details ?? 'failed'}`)
.join('; ')}`);
}
}
copyFileSync(deliveryCandidatePath, outputPath);
if (keepIntermediate) {
console.log(`Intermediate files preserved at: ${tempDir}`);
}
// A key counts as "used" when the document references it directly, or when
// it was read while deriving a computed key the document references
// (source fields of *_display values must be credited — only the computed
// key appears in the document).
const anyComputedKeyReferenced = [...computeWrites].some((key) => referencedIdentifiers.has(key));
const usedKeys = new Set(referencedIdentifiers);
if (anyComputedKeyReferenced) {
for (const key of computeReads)
usedKeys.add(key);
}
const fieldsUsed = Object.keys(data).filter((key) => usedKeys.has(key) && !syntheticFieldKeys.has(key));
const providedKeys = new Set(Object.keys(values));
return {
outputPath,
fieldsUsed,
providedFieldsUsed: fieldsUsed.filter((key) => providedKeys.has(key)),
fillCommandCount: commandCount,
warnings,
stages,
};
}
finally {
// Step 9: Cleanup
if (!keepIntermediate) {
rmSync(tempDir, { recursive: true, force: true });
}
}
}
/**
* Close whitespace seams left by empty substitutions within text paragraphs:
* collapse doubled spaces and remove whitespace immediately before punctuation.
*
* Paragraphs containing Word field characters (w:fldChar) are skipped because
* getParagraphText treats field-code runs as empty, so "Page " + PAGE field +
* " of " looks like "Page of " and the replacement would delete the field run
* that sits between the two space-bearing runs.
*/
export async function normalizeEmptyFillWhitespaceInDocx(docxPath) {
const zip = new AdmZip(docxPath);
const parser = new DOMParser();
const serializer = new XMLSerializer();
const parts = enumerateTextParts(zip);
const partNames = getGeneralTextPartNames(parts);
let anyModified = false;
for (const partName of partNames) {
const entry = zip.getEntry(partName);
if (!entry)
continue;
const doc = parser.parseFromString(entry.getData().toString('utf-8'), 'text/xml');
const paragraphs = doc.getElementsByTagNameNS(W_NS, 'p');
let partModified = false;
for (let i = 0; i < paragraphs.length; i++) {
const para = paragraphs[i];
if (para.getElementsByTagNameNS(W_NS, 'fldChar').length > 0) {
continue;
}
let text = getParagraphText(para);
let match;
while ((match = / {2,}/.exec(text)) !== null) {
try {
replaceParagraphTextRange(para, match.index, match.index + match[0].length, ' ');
partModified = true;
}
catch {
break;
}
text = getParagraphText(para);
}
while ((match = /[ \t\u00a0]+(?=[,.;:!?])/.exec(text)) !== null) {
try {
replaceParagraphTextRange(para, match.index, match.index + match[0].length, '');
partModified = true;
}
catch {
break;
}
text = getParagraphText(para);
}
}
if (partModified) {
zip.updateFile(partName, Buffer.from(serializer.serializeToString(doc), 'utf-8'));
anyModified = true;
}
}
if (anyModified) {
writeFileSync(docxPath, rezipWithoutDirEntries(zip).toBuffer());
}
}
//# sourceMappingURL=unified-pipeline.js.map