clarity-pattern-parser
Version:
Parsing Library for Typescript and Javascript.
792 lines (641 loc) • 30.7 kB
text/typescript
import { Node } from "../ast/Node";
import { tokens } from "./decorators/tokens";
import { generateErrorMessage } from "../patterns/generate_error_message";
export interface GrammarFile {
resource: string;
expression: string;
}
import { Context } from "../patterns/Context";
import { Expression } from "../patterns/Expression";
import { Literal } from "../patterns/Literal";
import { Not } from "../patterns/Not";
import { Optional } from "../patterns/Optional";
import { Options } from "../patterns/Options";
import { Pattern } from "../patterns/Pattern";
import { Reference } from "../patterns/Reference";
import { Regex } from "../patterns/Regex";
import { Repeat, RepeatOptions } from "../patterns/Repeat";
import { RightAssociated } from "../patterns/RightAssociated";
import { Sequence } from "../patterns/Sequence";
import { Block } from "../patterns/Block";
import { TakeUntil } from "../patterns/TakeUntil";
import { grammar } from "./patterns/grammar"
let anonymousIndexId = 0;
function defaultImportResolver(_path: string, _basePath: string | null): Promise<GrammarFile> {
throw new Error("No import resolver supplied.");
}
function defaultImportResolverSync(_path: string, _basePath: string | null): GrammarFile {
throw new Error("No import resolver supplied.");
}
export type Decorator = (pattern: Pattern, arg?: string | boolean | number | null | Record<string, any> | any[]) => void;
const defaultDecorators: Record<string, Decorator> = {
tokens: tokens
};
// Node names that are operators/whitespace and should be filtered during building
const skipNodes: Record<string, boolean> = {
"+": true,
"|": true,
"<|>": true,
"optionalWS": true,
"optionalLS": true,
"ws": true,
"ls": true,
};
export interface GrammarOptions {
resolveImport?: (resource: string, originResource: string | null) => Promise<GrammarFile>;
resolveImportSync?: (resource: string, originResource: string | null) => GrammarFile;
originResource?: string | null;
params?: Pattern[];
decorators?: Record<string, Decorator>;
}
class ParseContext {
patternsByName = new Map<string, Pattern>();
importedPatternsByName = new Map<string, Pattern>();
paramsByName = new Map<string, Pattern>();
decorators: Record<string, Decorator>;
constructor(params: Pattern[], decorators: Record<string, Decorator> = {}) {
params.forEach(p => this.paramsByName.set(p.name, p));
this.decorators = { ...decorators, ...defaultDecorators };
}
getImportedPatterns() {
return Array.from(this.importedPatternsByName.values());
}
getParams() {
return Array.from(this.paramsByName.values());
}
}
export class Grammar {
private _options: GrammarOptions;
private _parseContext: ParseContext;
private _params: Pattern[];
private _originResource?: string | null;
private _resolveImport: (resource: string, originResource: string | null) => Promise<GrammarFile>;
private _resolveImportSync: (resource: string, originResource: string | null) => GrammarFile;
constructor(options: GrammarOptions = {}) {
this._options = options;
this._params = options.params || [];
this._originResource = options.originResource || null;
this._parseContext = new ParseContext(this._params, options.decorators || {});
this._resolveImport = options.resolveImport == null ? defaultImportResolver : options.resolveImport;
this._resolveImportSync = options.resolveImportSync == null ? defaultImportResolverSync : options.resolveImportSync;
}
async import(path: string) {
const grammarFile = await this._resolveImport(path, null);
const grammar = new Grammar({
resolveImport: this._resolveImport,
originResource: grammarFile.resource,
params: this._params,
decorators: this._parseContext.decorators
});
return grammar.parse(grammarFile.expression);
}
async parse(expression: string) {
this._parseContext = new ParseContext(this._params, this._parseContext.decorators);
const ast = this._tryToParse(expression);
await this._resolveImports(ast);
this._buildPatterns(ast);
return this._buildPatternRecord();
}
parseString(expression: string) {
this._parseContext = new ParseContext(this._params, this._parseContext.decorators);
const ast = this._tryToParse(expression);
this._resolveImportsSync(ast);
this._buildPatterns(ast);
return this._buildPatternRecord();
}
// --- Static convenience methods ---
static parse(expression: string, options?: GrammarOptions) {
const g = new Grammar(options);
return g.parse(expression);
}
static import(path: string, options?: GrammarOptions) {
const g = new Grammar(options);
return g.import(path);
}
static parseString(expression: string, options?: GrammarOptions) {
const g = new Grammar(options);
return g.parseString(expression);
}
// --- Parsing ---
private _tryToParse(cpat: string): Node {
const { ast, cursor } = grammar.exec(cpat, true);
if (ast == null) {
const message = generateErrorMessage(grammar, cursor);
throw new Error(`[Invalid Grammar] ${message}`);
}
return ast;
}
// --- Expression Flattening (Phase 2) ---
private _flattenExpressionsRecursive(node: Node) {
// Process children first (bottom-up)
for (const child of node.children) {
this._flattenExpressionsRecursive(child);
}
switch (node.name) {
case "sequenceExpr":
case "optionsExpr":
case "greedyOptionsExpr":
this._unwrapNode(node.name, node);
break;
}
}
private _unwrapNode(type: string, node: Node) {
for (let x = 0; x < node.children.length; x++) {
const child = node.children[x];
if (child.name === type) {
node.spliceChildren(x, 1, ...child.children);
x--;
}
}
}
// --- Pattern Record ---
private _buildPatternRecord() {
const patterns: Record<string, Pattern> = {};
const allPatterns = Array.from(this._parseContext.patternsByName.values());
this._parseContext.patternsByName.forEach((p, name) => {
patterns[name] = new Context(name, p, allPatterns.filter(o => o !== p));
});
return patterns;
}
// --- Pattern Building (Phase 3) ---
private _buildPatterns(ast: Node) {
const statementsNode = ast.find(n => n.name === "statements");
if (statementsNode == null) {
return;
}
const allStatements = statementsNode.children;
// First pass: build pattern assignments
for (let i = 0; i < allStatements.length; i++) {
const statementNode = allStatements[i];
if (statementNode.name === "patternAssignment") {
const nameNode = statementNode.find(n => n.name === "patternName") as Node;
const name = nameNode.value;
// Find the pattern expression (the RHS of the assignment)
const patternExprNode = statementNode.children.find(n =>
n.name !== "patternName" && !skipNodes[n.name] && n.name !== "assign"
);
if (patternExprNode == null) {
continue;
}
// Flatten nested binary expressions
this._flattenExpressionsRecursive(patternExprNode);
let pattern = this._buildPattern(name, patternExprNode);
// For alias assignments (RHS is just a patternIdentifier), rename to the assignment name
if (patternExprNode.name === "patternIdentifier" && pattern.name !== name) {
pattern = pattern.clone(name);
}
this._applyDecorators(i, allStatements, pattern);
this._parseContext.patternsByName.set(name, pattern);
} else if (statementNode.name === "exportPattern") {
const exportName = statementNode.value;
const pattern = this._getPattern(exportName).clone();
this._parseContext.patternsByName.set(exportName, pattern);
}
}
}
private _buildPattern(name: string, node: Node): Pattern {
switch (node.name) {
case "literal": {
// Use the literal's content as the name when in anonymous position
const litName = name.startsWith("anonymous-pattern-") ? this._resolveStringValue(node.value) : name;
return this._buildLiteral(litName, node);
}
case "regex": {
// Use the regex's content as the name when in anonymous position
const regName = name.startsWith("anonymous-pattern-") ? node.value.slice(1, -1) : name;
return this._buildRegex(regName, node);
}
case "sequenceExpr":
return this._buildSequence(name, node);
case "optionsExpr":
return this._buildOptions(name, node);
case "greedyOptionsExpr":
return this._buildGreedyOptions(name, node);
case "repeatExpr":
return this._buildRepeat(name, node);
case "patternGroupExpr":
return this._buildPatternGroup(name, node);
case "notExpr":
return this._buildNot(name, node);
case "optionalExpr":
return this._buildOptional(name, node);
case "rightAssociationExpr":
return this._buildRightAssociation(node);
case "patternIdentifier": {
// Use the pattern's own name for references (important for Expression recursion detection)
// The caller (_buildPatterns) renames for alias assignments
return this._getPattern(node.value).clone();
}
case "takeUntilExpr":
return this._buildTakeUntil(name, node);
case "blockExpr":
return this._buildBlock(name, node);
case "blockDelimiter":
throw new Error(`Block delimiter [${this._extractBlockDelimiterValue(node)}] must be used at both the start and end of a sequence. Example: ["{"] content ["}"]`);
}
throw new Error(`Couldn't build node: ${node.name}.`);
}
private _buildLiteral(name: string, node: Node) {
return new Literal(name, this._resolveStringValue(node.value));
}
private _buildRegex(name: string, node: Node) {
const value = node.value.slice(1, node.value.length - 1);
return new Regex(name, value);
}
private _buildSequence(name: string, node: Node) {
const patternChildren = node.children.filter(n => !skipNodes[n.name]);
const patterns = patternChildren.map(n => this._buildPattern(`anonymous-pattern-${anonymousIndexId++}`, n));
return new Sequence(name, patterns);
}
private _buildBlock(name: string, node: Node) {
const delimiters = node.children.filter(n => n.name === "blockDelimiter");
const openLiteral = new Literal("open", this._extractBlockDelimiterValue(delimiters[0]));
const closeLiteral = new Literal("close", this._extractBlockDelimiterValue(delimiters[1]));
// Find the content node — everything that isn't a delimiter or whitespace
const contentChildren = node.children.filter(n =>
n.name !== "blockDelimiter" && !skipNodes[n.name]
);
// Wildcard: ["{"] ... ["}"]
if (contentChildren.length === 1 && contentChildren[0].name === "wildcard") {
return new Block(name, openLiteral, null, closeLiteral);
}
// Content pattern: ["{"] patternExpr ["}"]
const contentPattern = this._buildPattern(`anonymous-pattern-${anonymousIndexId++}`, contentChildren[0]);
return new Block(name, openLiteral, contentPattern, closeLiteral);
}
private _extractBlockDelimiterValue(node: Node): string {
const literalNode = node.children.find(n => n.name === "literal");
if (literalNode == null) {
throw new Error("Block delimiter missing literal value.");
}
return this._resolveStringValue(literalNode.value);
}
private _buildOptions(name: string, node: Node) {
const patternChildren = node.children.filter(n => !skipNodes[n.name]);
const patterns = patternChildren.map(n => {
return this._buildPattern(`anonymous-pattern-${anonymousIndexId++}`, n);
});
const hasRecursivePattern = patterns.some(p => this._isRecursive(name, p));
if (hasRecursivePattern) {
try {
return new Expression(name, patterns);
} catch { }
}
return new Options(name, patterns);
}
private _buildGreedyOptions(name: string, node: Node) {
const patternChildren = node.children.filter(n => !skipNodes[n.name]);
const patterns = patternChildren.map(n => this._buildPattern(`anonymous-pattern-${anonymousIndexId++}`, n));
return new Options(name, patterns, true);
}
private _buildRepeat(name: string, node: Node) {
let isOptional = false;
// Find the main pattern (first non-structural child inside parens)
const patternNode = node.children.find(n =>
!skipNodes[n.name] && n.name !== "(" && n.name !== ")" &&
n.name !== "repeatOptions" && n.name !== "optionalDelimiter" &&
n.name !== "delimiter" && n.name !== "oneOrMore" &&
n.name !== "zeroOrMore" && n.name !== "repeatBounds"
);
if (patternNode == null) {
throw new Error(`Repeat pattern missing inner pattern.`);
}
const pattern = this._buildPattern(`anonymous-pattern-${anonymousIndexId++}`, patternNode);
const options: RepeatOptions = {
min: 1,
max: Infinity
};
// Handle delimiter
const delimiterNode = node.find(n => n.name === "delimiter");
if (delimiterNode != null) {
const delimPatternNode = delimiterNode.children.find(n =>
!skipNodes[n.name] && n.name !== "," && n.name !== "optionalTrim" && n.name !== "trim"
);
if (delimPatternNode != null) {
options.divider = this._buildPattern(`anonymous-pattern-${anonymousIndexId++}`, delimPatternNode);
}
const trimFlag = delimiterNode.find(n => n.name === "trim");
if (trimFlag != null) {
options.trimDivider = true;
}
}
// Handle quantifier / bounds
const repeatOptionsNode = node.find(n => n.name === "oneOrMore" || n.name === "zeroOrMore" || n.name === "repeatBounds");
if (repeatOptionsNode != null) {
if (repeatOptionsNode.name === "oneOrMore") {
options.min = 1;
options.max = Infinity;
} else if (repeatOptionsNode.name === "zeroOrMore") {
isOptional = true;
options.min = 0;
options.max = Infinity;
} else if (repeatOptionsNode.name === "repeatBounds") {
const integers = repeatOptionsNode.findAll(n => n.name === "integer");
const hasComma = repeatOptionsNode.find(n => n.name === ",") != null;
if (integers.length === 2) {
// {min, max}
options.min = Number(integers[0].value);
options.max = Number(integers[1].value);
} else if (integers.length === 1 && hasComma) {
// Check position to determine {min,} or {,max}
const commaNode = repeatOptionsNode.find(n => n.name === ",") as Node;
const intNode = integers[0];
if (intNode.startIndex < commaNode.startIndex) {
// {min,}
options.min = Number(intNode.value);
options.max = Infinity;
} else {
// {,max}
options.min = 0;
options.max = Number(intNode.value);
}
} else if (integers.length === 1 && !hasComma) {
// {exact}
const count = Number(integers[0].value);
options.min = count;
options.max = count;
}
}
}
if (isOptional) {
return new Optional(name, new Repeat(`inner-optional-${name}`, pattern, options));
}
return new Repeat(name, pattern, options);
}
private _buildPatternGroup(_name: string, node: Node) {
const innerNode = node.children.find(n =>
!skipNodes[n.name] && n.name !== "(" && n.name !== ")"
);
if (innerNode == null) {
throw new Error("Empty pattern group.");
}
return this._buildPattern(`anonymous-pattern-${anonymousIndexId++}`, innerNode);
}
private _buildNot(_name: string, node: Node) {
const innerNode = node.children.find(n => !skipNodes[n.name] && n.name !== "not");
if (innerNode == null) {
throw new Error("Not pattern missing inner pattern.");
}
const inner = this._buildPattern(`anonymous-pattern-${anonymousIndexId++}`, innerNode);
return new Not(`not-${inner.name}`, inner);
}
private _buildOptional(_name: string, node: Node) {
const innerNode = node.children.find(n => !skipNodes[n.name] && n.name !== "?");
if (innerNode == null) {
throw new Error("Optional pattern missing inner pattern.");
}
const inner = this._buildPattern(`anonymous-pattern-${anonymousIndexId++}`, innerNode);
return new Optional(`optional-${inner.name}`, inner);
}
private _buildRightAssociation(node: Node) {
const innerNode = node.children.find(n => !skipNodes[n.name] && n.name !== "right");
if (innerNode == null) {
throw new Error("RightAssociation pattern missing inner pattern.");
}
return new RightAssociated(this._buildPattern(`anonymous-pattern-${anonymousIndexId++}`, innerNode));
}
private _buildTakeUntil(name: string, node: Node) {
// The last meaningful child is the until pattern
const patternChildren = node.children.filter(n =>
!skipNodes[n.name] && n.name !== "anyChar" && n.name !== "upTo" && n.name !== "wall"
);
const untilPatternNode = patternChildren[patternChildren.length - 1];
if (untilPatternNode == null) {
throw new Error("TakeUntil pattern missing terminator pattern.");
}
const untilPattern = this._buildPattern(`anonymous-pattern-${anonymousIndexId++}`, untilPatternNode);
return new TakeUntil(name, untilPattern);
}
// --- Helpers ---
private _resolveStringValue(value: string) {
return value.replace(/\\n/g, '\n')
.replace(/\\r/g, '\r')
.replace(/\\t/g, '\t')
.replace(/\\b/g, '\b')
.replace(/\\f/g, '\f')
.replace(/\\v/g, '\v')
.replace(/\\0/g, '\0')
.replace(/\\x([0-9A-Fa-f]{2})/g, (_, hex: string) => String.fromCharCode(parseInt(hex, 16)))
.replace(/\\u([0-9A-Fa-f]{4})/g, (_, hex: string) => String.fromCharCode(parseInt(hex, 16)))
.replace(/\\(.)/g, '$1').slice(1, -1);
}
private _getPattern(name: string) {
let pattern = this._parseContext.patternsByName.get(name);
if (pattern == null) {
pattern = this._parseContext.importedPatternsByName.get(name);
}
if (pattern == null) {
pattern = this._parseContext.paramsByName.get(name);
}
if (pattern == null) {
return new Reference(name);
}
return pattern;
}
private _isRecursive(name: string, pattern: Pattern) {
if (pattern.type === "right-associated") {
pattern = pattern.children[0];
}
return this._isRecursivePattern(name, pattern);
}
private _isRecursivePattern(name: string, pattern: Pattern) {
if (pattern.type === "reference") {
return true;
}
if (pattern.children.length === 0) {
return false;
}
const firstChild = pattern.children[0];
const lastChild = pattern.children[pattern.children.length - 1];
const isLongEnough = pattern.children.length >= 2;
return pattern.type === "sequence" && isLongEnough &&
(firstChild.name === name || lastChild.name === name);
}
// --- Decorators ---
private _applyDecorators(statementIndex: number, allStatements: readonly Node[], pattern: Pattern) {
const decorators = this._parseContext.decorators;
let decoratorNodes: Node[] = [];
// Walk backwards from the statement to find preceding decorator statements
// Note: statements Repeat includes newLine divider nodes between statements
for (let i = statementIndex - 1; i >= 0; i--) {
const prev = allStatements[i];
if (prev.name === "decorationStatement" || prev.name === "methodDecorationStatement" || prev.name === "nameDecorationStatement") {
decoratorNodes.push(prev);
} else if (prev.name === "comment" || prev.name === "newLine") {
// Comments and newline dividers can appear between decorators and the statement
continue;
} else {
break;
}
}
for (const d of decoratorNodes) {
const actualDecorator = d.name === "decorationStatement" ? d.children[0] : d;
const nameNode = actualDecorator.find(n => n.name === "decorationName");
if (nameNode == null || decorators[nameNode.value] == null) {
continue;
}
if (actualDecorator.name === "nameDecorationStatement") {
decorators[nameNode.value](pattern);
} else if (actualDecorator.name === "methodDecorationStatement") {
const jsonValueNode = actualDecorator.find(n => n.name === "jsonArray" || n.name === "jsonObject" || n.name === "jsonString" || n.name === "jsonNumber" || n.name === "jsonBoolean" || n.name === "jsonNull");
if (jsonValueNode == null) {
decorators[nameNode.value](pattern);
} else {
decorators[nameNode.value](pattern, JSON.parse(jsonValueNode.value));
}
}
}
}
// --- Import Handling ---
private async _resolveImports(ast: Node) {
const importStatements = ast.findAll(n => {
return n.name === "importStatement" || n.name === "useParamsStatement";
});
for (const statement of importStatements) {
if (statement.name === "importStatement") {
await this._processImport(statement);
} else {
this._processUseParams(statement);
}
}
}
private _resolveImportsSync(ast: Node) {
const importStatements = ast.findAll(n => {
return n.name === "importStatement" || n.name === "useParamsStatement";
});
for (const statement of importStatements) {
if (statement.name === "importStatement") {
this._processImportSync(statement);
} else {
this._processUseParams(statement);
}
}
}
private _processImportSync(importStatement: Node) {
const parseContext = this._parseContext;
const resourceNode = importStatement.find(n => n.name === "resource") as Node;
if (resourceNode == null) {
throw new Error("Invalid import statement: resource is required");
}
const params = this._getWithParams(importStatement);
const resource = resourceNode.value.slice(1, -1);
const grammarFile = this._resolveImportSync(resource, this._originResource || null);
const g = new Grammar({
resolveImport: this._resolveImport,
resolveImportSync: this._resolveImportSync,
originResource: grammarFile.resource,
params,
decorators: this._parseContext.decorators
});
try {
const patterns = g.parseString(grammarFile.expression);
this._processImportNames(importStatement, patterns, parseContext, resource);
} catch (e: any) {
throw new Error(`Failed loading expression from: "${resource}". Error details: "${e.message}"`);
}
}
private async _processImport(importStatement: Node) {
const parseContext = this._parseContext;
const resourceNode = importStatement.find(n => n.name === "resource") as Node;
if (resourceNode == null) {
throw new Error("Invalid import statement: resource is required");
}
const params = this._getWithParams(importStatement);
const resource = resourceNode.value.slice(1, -1);
const grammarFile = await this._resolveImport(resource, this._originResource || null);
const g = new Grammar({
resolveImport: this._resolveImport,
originResource: grammarFile.resource,
params,
decorators: this._parseContext.decorators
});
try {
const patterns = await g.parse(grammarFile.expression);
this._processImportNames(importStatement, patterns, parseContext, resource);
} catch (e: any) {
throw new Error(`Failed loading expression from: "${resource}". Error details: "${e.message}"`);
}
}
private _processImportNames(importStatement: Node, patterns: Record<string, Pattern>, parseContext: ParseContext, resource: string) {
// Find all imported names (could be aliases or plain names)
const importedPatternsNode = importStatement.find(n => n.name === "importedPatterns");
if (importedPatternsNode == null) return;
const patternNamesNode = importedPatternsNode.find(n => n.name === "patternNames");
if (patternNamesNode == null) return;
for (const child of patternNamesNode.children) {
if (child.name === "importAlias") {
const nameNode = child.find(n => n.name === "patternName") as Node;
const aliasNode = child.find(n => n.name === "importNameAlias") as Node;
const importName = nameNode.value;
const alias = aliasNode.value;
if (parseContext.importedPatternsByName.has(alias)) {
throw new Error(`'${alias}' was already used within another import.`);
}
const pattern = patterns[importName];
if (pattern == null) {
throw new Error(`Couldn't find pattern with name: ${importName}, from import: ${resource}.`);
}
parseContext.importedPatternsByName.set(alias, pattern.clone(alias));
} else if (child.name === "patternName") {
const importName = child.value;
if (parseContext.importedPatternsByName.has(importName)) {
throw new Error(`'${importName}' was already used within another import.`);
}
const pattern = patterns[importName];
if (pattern == null) {
throw new Error(`Couldn't find pattern with name: ${importName}, from import: ${resource}.`);
}
parseContext.importedPatternsByName.set(importName, pattern);
}
// Skip comma divider nodes
}
}
private _getWithParams(importStatement: Node) {
let params: Pattern[] = [];
const withParamsNode = importStatement.find(n => n.name === "withParamsExpr");
if (withParamsNode != null) {
const statementsNode = withParamsNode.find(n => n.name === "withParamStatements");
if (statementsNode != null) {
const expression = statementsNode.value;
const importedValues = this._parseContext.getImportedPatterns();
const g = new Grammar({
params: [...importedValues, ...this._parseContext.paramsByName.values()],
originResource: this._originResource,
resolveImport: this._resolveImport,
resolveImportSync: this._resolveImportSync,
decorators: this._parseContext.decorators
});
const patterns = g.parseString(expression);
params = Array.from(Object.values(patterns));
}
}
return params;
}
private _processUseParams(useParamsStatement: Node) {
const patternsNode = useParamsStatement.find(n => n.name === "useParamPatterns");
if (patternsNode == null) {
return;
}
// Each child is a paramNameWithDefault: patternName + optionalParamDefault
const paramNodes = patternsNode.findAll(n => n.name === "paramNameWithDefault");
for (const paramNode of paramNodes) {
const nameNode = paramNode.find(n => n.name === "patternName");
if (nameNode == null) continue;
const name = nameNode.value;
// If already provided via params option, skip
if (this._parseContext.paramsByName.has(name)) {
continue;
}
// Check for default value: use params { value = default-value }
const defaultNode = paramNode.find(n => n.name === "defaultParamName");
if (defaultNode != null) {
const defaultName = defaultNode.value;
let pattern = this._parseContext.importedPatternsByName.get(defaultName);
if (pattern == null) {
pattern = new Reference(defaultName);
}
this._parseContext.importedPatternsByName.set(name, pattern);
}
}
}
}