dry-ts
Version:
Find candidate duplicate TypeScript code by comparing normalized AST structure.
294 lines (293 loc) • 14.2 kB
JavaScript
import fs from "node:fs";
import ts from "typescript";
import { FingerprintInterner } from "./NormalizedNode.js";
import { TypeScriptNormalizer } from "./TypeScriptNormalizer.js";
// Parses and fingerprints files in a single AST walk, without materializing the
// normalized tree. Fingerprints are content hashes, so output is deterministic
// regardless of how files are split across scanner instances or worker threads.
export class FileScanner {
normalizer = new TypeScriptNormalizer();
interner = new FingerprintInterner();
markerHashes = new Map();
scanFiles(files, minLines, minNodes = 1, excludeKinds = EMPTY_KIND_SET, minDistinctKinds = 0, excludeTaggedTemplates = false) {
return files.flatMap((file) => this.scanFile(file, minLines, minNodes, excludeKinds, minDistinctKinds, excludeTaggedTemplates));
}
scanFile(file, minLines, minNodes = 1, excludeKinds = EMPTY_KIND_SET, minDistinctKinds = 0, excludeTaggedTemplates = false) {
const text = fs.readFileSync(file, "utf8");
const sourceFile = ts.createSourceFile(file, text, ts.ScriptTarget.Latest, false, scriptKind(file));
const parseDiagnostics = sourceFile
.parseDiagnostics;
if (parseDiagnostics && parseDiagnostics.length > 0) {
const first = parseDiagnostics[0];
const message = ts.flattenDiagnosticMessageText(first.messageText, "\n");
throw new Error(`Unable to parse ${file}: ${message}`);
}
// Post-order hashes of every kept node; a subtree always owns the contiguous
// range it appended, so an entry's fingerprints are a slice of this array.
const hashes = [];
const entries = [];
let nextOrder = 0;
// Kind-diversity floor (plan 008). Only tracked when active so the default
// (off) scan path adds no cost. `tags` holds one node-kind tag per visited
// node, post-order, so a candidate's subtree is the slice [tagStart, end);
// markers do not count toward kind diversity by design.
const trackKinds = minDistinctKinds > 0;
const tags = [];
const visit = (node) => {
const order = nextOrder++;
const rangeStart = hashes.length;
const tagStart = tags.length;
const childHashes = [];
for (const marker of this.normalizer.markers(node)) {
const markerHash = this.markerHash(marker);
hashes.push(markerHash);
childHashes.push(markerHash);
}
node.forEachChild((child) => {
if (this.normalizer.keepsStructuralChild(child)) {
childHashes.push(visit(child));
}
});
const tag = this.normalizer.tag(node);
const hash = this.interner.idFor(tag, childHashes);
hashes.push(hash);
if (trackKinds) {
tags.push(tag);
}
if (candidateRootKinds.has(node.kind) &&
!excludeKinds.has(node.kind) &&
hashes.length - rangeStart >= minNodes &&
!hasIgnoreDirective(text, node) &&
(!trackKinds || distinctKindCount(tags, tagStart) >= minDistinctKinds) &&
(!excludeTaggedTemplates || !isTaggedTemplateValued(node))) {
const { startLine, endLine } = lineRangeFor(sourceFile, node);
if (endLine - startLine + 1 >= minLines) {
entries.push({
order,
entry: {
file,
startLine,
endLine,
nodes: hashes.length - rangeStart,
fingerprints: sortedUnique(hashes, rangeStart),
kind: candidateKindNameByKind.get(node.kind),
name: declarationName(node, sourceFile),
},
});
}
}
return hash;
};
sourceFile.forEachChild((child) => {
if (this.normalizer.keepsStructuralChild(child)) {
visit(child);
}
});
// Entries were collected post-order; report them in document (pre-)order.
return entries.sort((left, right) => left.order - right.order).map(({ entry }) => entry);
}
markerHash(marker) {
let hash = this.markerHashes.get(marker);
if (hash === undefined) {
hash = this.interner.idFor(marker, []);
this.markerHashes.set(marker, hash);
}
return hash;
}
}
// Distinct node-kind tags over a candidate's subtree slice [start, end). Built
// only when the floor is active (guarded at the call site), so it never touches
// the default scan path.
function distinctKindCount(tags, start) {
const seen = new Set();
for (let i = start; i < tags.length; i += 1) {
seen.add(tags[i]);
}
return seen.size;
}
function sortedUnique(hashes, start) {
const sorted = new Float64Array(hashes.length - start);
for (let i = start; i < hashes.length; i += 1) {
sorted[i - start] = hashes[i];
}
sorted.sort();
let writeIndex = 0;
for (let i = 0; i < sorted.length; i += 1) {
if (i === 0 || sorted[i] !== sorted[i - 1]) {
sorted[writeIndex] = sorted[i];
writeIndex += 1;
}
}
return sorted.slice(0, writeIndex);
}
// Single source of truth for the candidate root kinds: the declaration shapes
// dry-ts treats as comparable units. Each entry carries the canonical name a
// user types for --exclude-kinds plus a plain-English blurb for the help/README
// docs. The name is spelled out explicitly rather than derived from
// ts.SyntaxKind[kind]: TS reverse-enum lookup returns the marker alias for
// boundary kinds (e.g. ts.SyntaxKind[VariableStatement] is "FirstStatement"),
// which is not the name users expect or that the docs advertise. Order here is
// the order shown to users.
const candidateKinds = [
{ name: "ClassDeclaration", kind: ts.SyntaxKind.ClassDeclaration, blurb: "a `class Foo {}` declaration" },
{
name: "InterfaceDeclaration",
kind: ts.SyntaxKind.InterfaceDeclaration,
blurb: "an `interface Foo {}` declaration",
},
{ name: "TypeAliasDeclaration", kind: ts.SyntaxKind.TypeAliasDeclaration, blurb: "a `type Foo = ...` alias" },
{ name: "EnumDeclaration", kind: ts.SyntaxKind.EnumDeclaration, blurb: "an `enum Foo {}` declaration" },
{
name: "ModuleDeclaration",
kind: ts.SyntaxKind.ModuleDeclaration,
blurb: "a `namespace Foo {}` / `module Foo {}` block",
},
{ name: "FunctionDeclaration", kind: ts.SyntaxKind.FunctionDeclaration, blurb: "a `function foo() {}` declaration" },
{
name: "MethodDeclaration",
kind: ts.SyntaxKind.MethodDeclaration,
blurb: "a method body in a class or object literal: `foo() {}`",
},
{ name: "Constructor", kind: ts.SyntaxKind.Constructor, blurb: "a class `constructor() {}`" },
{ name: "GetAccessor", kind: ts.SyntaxKind.GetAccessor, blurb: "a getter: `get foo() {}`" },
{ name: "SetAccessor", kind: ts.SyntaxKind.SetAccessor, blurb: "a setter: `set foo(v) {}`" },
{
name: "PropertyDeclaration",
kind: ts.SyntaxKind.PropertyDeclaration,
blurb: "a class field: `foo = ...` / `foo: T`",
},
{
name: "PropertySignature",
kind: ts.SyntaxKind.PropertySignature,
blurb: "a property in an interface/type: `foo: T`",
},
{
name: "MethodSignature",
kind: ts.SyntaxKind.MethodSignature,
blurb: "a method signature in an interface/type: `foo(): T`",
},
{ name: "CallSignature", kind: ts.SyntaxKind.CallSignature, blurb: "a callable signature in a type: `(arg: T): U`" },
{
name: "ConstructSignature",
kind: ts.SyntaxKind.ConstructSignature,
blurb: "a constructable signature in a type: `new (): T`",
},
{ name: "IndexSignature", kind: ts.SyntaxKind.IndexSignature, blurb: "an index signature: `[key: string]: T`" },
{
name: "VariableStatement",
kind: ts.SyntaxKind.VariableStatement,
blurb: "a `const` / `let` / `var` statement (the whole declaration line)",
},
{ name: "EnumMember", kind: ts.SyntaxKind.EnumMember, blurb: "a single member inside an enum" },
{ name: "ArrowFunction", kind: ts.SyntaxKind.ArrowFunction, blurb: "an arrow function used as a value: `() => {}`" },
{ name: "FunctionExpression", kind: ts.SyntaxKind.FunctionExpression, blurb: "a `function () {}` used as a value" },
];
const candidateRootKinds = new Set(candidateKinds.map((entry) => entry.kind));
const EMPTY_KIND_SET = new Set();
// Derived from candidateKinds so the two can never drift.
const candidateKindByName = new Map(candidateKinds.map((entry) => [entry.name, entry.kind]));
// Reverse of candidateKindByName: the canonical name dry-ts reports for a
// candidate root kind. Lookup is always populated at the call site (guarded by
// candidateRootKinds), so the get() there is non-null.
const candidateKindNameByKind = new Map(candidateKinds.map((entry) => [entry.kind, entry.name]));
export const candidateKindNames = candidateKinds.map((entry) => entry.name);
// For help and README docs.
export const candidateKindDescriptions = candidateKinds.map(({ name, blurb }) => ({ name, blurb }));
// Resolves --exclude-kinds names to SyntaxKinds, validating each against the
// candidate set. An unknown or non-candidate name throws rather than silently
// no-op'ing — a silent gate-flag bypass is a footgun.
export function resolveExcludeKinds(names) {
const kinds = new Set();
for (const name of names) {
const kind = candidateKindByName.get(name);
if (kind === undefined) {
throw new Error(`Unknown candidate kind: ${name}`);
}
kinds.add(kind);
}
return kinds;
}
function scriptKind(file) {
if (file.endsWith(".jsx")) {
return ts.ScriptKind.JSX;
}
if (file.endsWith(".js")) {
return ts.ScriptKind.JS;
}
if (file.endsWith(".tsx")) {
return ts.ScriptKind.TSX;
}
return ts.ScriptKind.TS;
}
// Source-level escape hatch. A `// dry-ignore` (or `dry-ignore-next-line`)
// comment in a node's leading trivia suppresses that node as a candidate. We
// read the existing `text` via getLeadingCommentRanges — forEachChild skips
// comment trivia, so there is no second parse. Suppression is scoped to the
// node whose trivia carries the directive: the comment must sit on the specific
// declaration the user means (a directive on a wrapping VariableStatement does
// not reach a nested ArrowFunction, which keeps its own leading trivia).
function hasIgnoreDirective(text, node) {
const ranges = ts.getLeadingCommentRanges(text, node.getFullStart());
if (!ranges) {
return false;
}
for (const range of ranges) {
const raw = text.substring(range.pos, range.end);
const body = range.kind === ts.SyntaxKind.MultiLineCommentTrivia ? raw.slice(2, -2) : raw.slice(2);
if (/^\s*dry-ignore(-next-line)?\b/.test(body)) {
return true;
}
}
return false;
}
// CSS-in-JS / styled-components reducer (--exclude-tagged-templates, plan 034).
// Styled-components and friends (`const X = styled(Button)`…``, `styled('span')`…``,
// `css`…``, `gql`…``) normalize to a near-identical AST — a VariableStatement whose
// initializer is a TaggedTemplateExpression — so they cluster across dozens of files
// despite sharing no logic. We drop ANY tagged-template-valued candidate rather than
// scoping to the `styled` tag specifically: every tagged-template idiom shares the same
// false-positive shape, and matching by structure (not tag identifier) is simpler and
// catches `css`/`gql`/styled aliases without a tag allowlist. Guarded at the call site
// so the default (off) scan path never pays for it.
function isTaggedTemplateValued(node) {
if (ts.isTaggedTemplateExpression(node)) {
return true;
}
// The dominant case: `const X = styled(...)`…`` parses as a VariableStatement.
// Unwrap to the first declaration's initializer and check that.
if (ts.isVariableStatement(node)) {
const initializer = node.declarationList.declarations[0]?.initializer;
return initializer !== undefined && ts.isTaggedTemplateExpression(initializer);
}
// Other value-position roots that can directly hold a tagged template.
if (ts.isPropertyDeclaration(node) || ts.isPropertyAssignment(node)) {
return node.initializer !== undefined && ts.isTaggedTemplateExpression(node.initializer);
}
return false;
}
function lineRangeFor(sourceFile, node) {
return {
startLine: sourceFile.getLineAndCharacterOfPosition(node.getStart(sourceFile, false)).line + 1,
endLine: sourceFile.getLineAndCharacterOfPosition(node.getEnd()).line + 1,
};
}
// A constructor has no name node, so label it by keyword; a VariableStatement is
// named after its first binding (`const foo = ...` -> "foo"), but only when that
// binding is a plain identifier — a destructuring pattern (`const {a, b} = ...`)
// has no single identifier name, and its getText() would be the whole pattern
// (often multi-line, which breaks the single-line text format), so report null.
// Everything else defers to TypeScript's name resolver (null for anonymous
// arrows, unnamed function expressions, call/index signatures). getText needs the
// sourceFile passed because the AST is parsed without parent pointers
// (setParentNodes=false).
function declarationName(node, sourceFile) {
if (ts.isConstructorDeclaration(node)) {
return "constructor";
}
if (ts.isVariableStatement(node)) {
const first = node.declarationList.declarations[0];
return first && ts.isIdentifier(first.name) ? first.name.getText(sourceFile) : null;
}
const nameNode = ts.getNameOfDeclaration(node);
return nameNode ? nameNode.getText(sourceFile) : null;
}