@eliseev_s/tolk-tlb-transpiler
Version:
Transpile Tolk structs to TLB definitions and generate TypeScript wrappers for TON blockchain smart contracts
784 lines • 30 kB
JavaScript
"use strict";
// ============================================================================
// Tolk AST Parser
// ============================================================================
Object.defineProperty(exports, "__esModule", { value: true });
exports.TolkASTProcessor = void 0;
const web_tree_sitter_1 = require("web-tree-sitter");
const tree_sitter_tolk_1 = require("@eliseev_s/tree-sitter-tolk");
const fs_1 = require("fs");
const path_1 = require("path");
const generator_1 = require("./generator");
const get_methods_parser_1 = require("./get-methods-parser");
// Cache for initialized language to avoid re-loading WASM module
let languageCache = null;
let initPromise = null;
async function ensureInitialized() {
if (languageCache)
return;
if (initPromise)
return initPromise;
initPromise = (async () => {
await web_tree_sitter_1.Parser.init();
languageCache = await (0, tree_sitter_tolk_1.loadTolk)();
})();
return initPromise;
}
function createParser() {
if (!languageCache) {
throw new Error('Parser not initialized. Call TolkASTProcessor.initialize() first.');
}
const parser = new web_tree_sitter_1.Parser();
parser.setLanguage(languageCache);
return parser;
}
class TolkASTProcessor {
parser;
structs = new Map();
typeAliases = new Map();
getMethods = [];
constants = new Map();
constructor(parser) {
this.parser = parser;
}
static async initialize() {
await ensureInitialized();
const parser = createParser();
return new TolkASTProcessor(parser);
}
/**
* Convert Tolk source code directly to TLB definitions
*/
transpile(sourceCode) {
const tree = this.parser.parse(sourceCode);
if (!tree)
throw new Error('Tre is empty');
return this.processAST(tree);
}
/**
* Transpile a Tolk file with import resolution (no cycles assumed)
*/
transpileFile(entryFilePath) {
this.structs.clear();
this.typeAliases.clear();
this.getMethods = [];
this.constants.clear();
const visited = new Set();
this.collectFromFile(entryFilePath, visited);
const generator = new generator_1.TLBGenerator(this.structs, this.typeAliases);
return generator.generateTLB();
}
parse(sourceCode) {
const tree = this.parser.parse(sourceCode);
if (!tree) {
throw new Error('Failed to parse Tolk source code');
}
return tree;
}
/**
* Get parsed get methods
*/
getGetMethods() {
return this.getMethods;
}
getStructs() {
return Array.from(this.structs.values());
}
getTypeAliases() {
return Array.from(this.typeAliases.values());
}
processAST(tree) {
this.structs.clear();
this.typeAliases.clear();
this.getMethods = [];
this.constants.clear();
// First pass: collect all struct, type alias, and constant definitions
this.collectDefinitions(tree.rootNode);
// Parse get methods
this.getMethods = get_methods_parser_1.GetMethodsParser.parseGetMethods(tree.rootNode, (node) => this.resolveType(node));
// Second pass: generate TLB using the generator
const generator = new generator_1.TLBGenerator(this.structs, this.typeAliases);
return generator.generateTLB();
}
collectFromFile(filePath, visited) {
const resolved = (0, path_1.resolve)(filePath);
if (visited.has(resolved))
return;
visited.add(resolved);
if (!(0, fs_1.existsSync)(resolved)) {
throw new Error(`Imported file not found: ${filePath}`);
}
const source = (0, fs_1.readFileSync)(resolved, 'utf-8');
// Recursively process imports first
const imports = this.extractImportsFromSource(source, (0, path_1.dirname)(resolved));
for (const imp of imports) {
this.collectFromFile(imp, visited);
}
const tree = this.parser.parse(source);
if (!tree) {
throw new Error(`Failed to parse Tolk source file: ${resolved}`);
}
// Collect definitions from this file (do not clear accumulated state)
this.collectDefinitions(tree.rootNode);
}
extractImportsFromSource(sourceCode, baseDir) {
const results = [];
const importRegex = /\bimport\s+["']([^"']+)["'];/g;
let match;
while ((match = importRegex.exec(sourceCode)) !== null) {
const raw = match[1];
// Ignore stdlib and external module imports (e.g., @stdlib/...)
if (raw.startsWith('@stdlib')) {
continue;
}
const withExt = (0, path_1.extname)(raw) ? raw : `${raw}.tolk`;
results.push((0, path_1.resolve)(baseDir, withExt));
}
return results;
}
collectDefinitions(node) {
if (node.type === 'struct_declaration') {
const structInfo = this.processStructDeclaration(node);
this.structs.set(structInfo.name, structInfo);
}
else if (node.type === 'type_alias_declaration') {
const typeAlias = this.processTypeAlias(node);
this.typeAliases.set(typeAlias.name, typeAlias);
}
else if (node.type === 'constant_declaration') {
const constant = this.processConstantDeclaration(node);
if (constant) {
this.constants.set(constant.name, constant);
}
}
else if (node.type === 'enum_declaration') {
this.processEnumDeclaration(node);
}
// Recursively process children
for (const child of node.namedChildren) {
child && this.collectDefinitions(child);
}
}
processEnumDeclaration(node) {
// Fallback to text-based parsing to be robust to grammar changes
const text = node.text.trim();
// Try to capture: enum Name [: type]? { body }
const m = text.match(/^enum\s+([A-Za-z_][A-Za-z0-9_]*)\s*(?::\s*[^\{]+)?\{([\s\S]*?)\}\s*$/);
if (!m)
return;
const enumName = m[1];
const body = m[2];
// Split members by comma, semicolon, or newline, but tolerate extra whitespace
const rawItems = body
.split(/[,;\n]/)
.map((s) => s.replace(/\/\/.*$/, '').trim()) // strip // comments and trim
.filter((s) => s.length > 0);
let nextValue = 0;
for (const item of rawItems) {
const mm = item.match(/^([A-Za-z_][A-Za-z0-9_]*)\s*(?:=\s*([^\s]+))?\s*$/);
if (!mm)
continue;
const memberName = mm[1];
const valueText = mm[2];
let value;
if (valueText !== undefined) {
if (valueText.startsWith('0x') || valueText.startsWith('0X')) {
value = parseInt(valueText.slice(2), 16);
}
else if (valueText.startsWith('0b') || valueText.startsWith('0B')) {
value = parseInt(valueText.slice(2), 2);
}
else {
value = parseInt(valueText, 10);
}
if (!Number.isFinite(value))
continue;
nextValue = value; // set current value
}
else {
value = nextValue;
}
// Save constant as Enum.Member form
const fullName = `${enumName}.${memberName}`;
this.constants.set(fullName, { name: fullName, value });
nextValue = value + 1;
}
}
processStructDeclaration(node) {
const nameNode = node.childForFieldName('name');
const name = nameNode ? nameNode.text : 'UnknownStruct';
const packPrefixNode = node.childForFieldName('pack_prefix');
const prefix = packPrefixNode ? this.extractPackPrefix(packPrefixNode) : undefined;
const annotationsNode = node.childForFieldName('annotations');
const annotations = annotationsNode ? this.extractAnnotations(annotationsNode) : [];
const genericTsNode = node.childForFieldName('genericTs');
let genericParams = genericTsNode ? this.extractGenerics(genericTsNode) : [];
const fields = this.extractStructFields(node);
// Fallback: infer generics from field types if not explicitly declared (T, U, X, Y, etc.)
if (genericParams.length === 0) {
const inferred = this.inferGenericParamsFromFields(fields);
if (inferred.length > 0) {
genericParams = inferred;
}
}
return {
name,
prefix,
fields,
annotations,
genericParams,
};
}
processTypeAlias(node) {
const nameNode = node.childForFieldName('name');
const name = nameNode ? nameNode.text : 'UnknownType';
const underlyingTypeNode = node.childForFieldName('underlying_type');
const underlyingType = underlyingTypeNode
? this.resolveType(underlyingTypeNode)
: { kind: 'unknown' };
const genericTsNode = node.childForFieldName('genericTs');
const genericParams = genericTsNode ? this.extractGenerics(genericTsNode) : [];
return {
name,
underlyingType,
genericParams,
};
}
processConstantDeclaration(node) {
// Find child nodes by type
let nameNode = null;
let valueNode = null;
let typeNode = null;
for (let i = 0; i < node.namedChildCount; i++) {
const child = node.namedChild(i);
if (!child)
continue;
if (child.type === 'identifier') {
nameNode = child;
}
else if (child.type === 'number_literal') {
valueNode = child;
}
else if (child.type === 'type_identifier') {
typeNode = child;
}
}
if (!nameNode || !valueNode) {
return null;
}
const name = nameNode.text;
const valueText = valueNode.text;
// Parse the value based on format
let value;
if (valueText.startsWith('0x')) {
// Hexadecimal
value = parseInt(valueText, 16);
}
else if (valueText.startsWith('0b')) {
// Binary
value = parseInt(valueText.slice(2), 2);
}
else {
// Decimal
value = parseInt(valueText, 10);
}
if (isNaN(value)) {
return null;
}
const type = typeNode ? this.resolveType(typeNode) : undefined;
return {
name,
value,
type,
};
}
extractPackPrefix(node) {
const text = node.text.trim();
if (text.startsWith('0x')) {
// Hexadecimal format
const hexValue = text.slice(2);
const value = parseInt(hexValue, 16);
const length = hexValue.length * 4; // Each hex digit = 4 bits
return { value, length, format: 'hex' };
}
else if (text.startsWith('0b')) {
// Binary format
const binaryValue = text.slice(2);
const value = parseInt(binaryValue, 2);
const length = binaryValue.length; // Each binary digit = 1 bit
return { value, length, format: 'binary' };
}
else {
// Decimal number - assume 8 bits for simplicity
const value = parseInt(text, 10);
return { value, length: 8, format: 'hex' };
}
}
extractAnnotations(node) {
const annotations = [];
for (const child of node.namedChildren) {
if (child?.type === 'annotation') {
const nameNode = child.childForFieldName('name');
const name = nameNode ? nameNode.text : '';
annotations.push({ name, parameters: [] });
}
}
return annotations;
}
extractGenerics(node) {
// Node text expected like: <T, U = never>
const text = node.text.trim();
const start = text.indexOf('<');
const end = text.lastIndexOf('>');
if (start === -1 || end === -1 || end <= start + 1)
return [];
const inner = text.slice(start + 1, end);
// Split by commas at top level (no nested generics expected in params)
const parts = inner
.split(',')
.map((p) => p.trim())
.filter((p) => p.length > 0);
const params = [];
for (const part of parts) {
// Remove default assignment if present: T = never
const noDefault = part.split('=')[0]?.trim() || part;
// Remove constraint if present: T extends X
const name = noDefault.split(/\s+extends\s+/i)[0]?.trim() || noDefault;
if (name) {
params.push({ name });
}
}
return params;
}
inferGenericParamsFromFields(fields) {
const names = new Set();
const visit = (t) => {
if (!t)
return;
switch (t.kind) {
case 'type_identifier': {
if (/^[A-Z]$/.test(t.name)) {
names.add(t.name);
}
break;
}
case 'generic': {
// Visit generic params (e.g., Cell<T>)
for (const p of t.params)
visit(p);
break;
}
case 'nullable':
visit(t.inner);
break;
case 'tensor':
case 'tuple':
for (const it of t.items)
visit(it);
break;
case 'union':
for (const alt of t.alternatives)
visit(alt);
break;
case 'cell_ref':
visit(t.inner);
break;
default:
break;
}
};
for (const f of fields)
visit(f.type);
// Filter out names that are already known concrete types or aliases
const filtered = Array.from(names).filter((name) => !this.structs.has(name) && !this.typeAliases.has(name));
return filtered.map((name) => ({ name }));
}
extractStructFields(node) {
const fields = [];
// Find the struct_body node
let structBodyNode = null;
for (let i = 0; i < node.namedChildren.length; i++) {
const child = node.namedChildren[i];
if (child?.type === 'struct_body') {
structBodyNode = child;
break;
}
}
if (!structBodyNode) {
return fields;
}
// Now look for struct fields inside the struct_body
for (let i = 0; i < structBodyNode.namedChildren.length; i++) {
const child = structBodyNode.namedChildren[i];
// Be tolerant to field modifiers (private, readonly, etc.).
// Consider any node with both `name` and `type` fields to be a field declaration.
if (!child)
continue;
const nameNode = child.childForFieldName('name');
const typeNode = child.childForFieldName('type');
if (nameNode && typeNode) {
const defaultNode = child.childForFieldName('default') || child.childForFieldName('default_value');
fields.push({
name: nameNode.text,
type: this.resolveType(typeNode),
default: defaultNode ? defaultNode.text : undefined,
});
}
}
return fields;
}
resolveType(typeNode) {
switch (typeNode.type) {
case 'primitive_type':
return { kind: 'primitive', name: typeNode.text };
case 'type_identifier': {
const text = typeNode.text;
// Handle dict as special case
if (text === 'dict') {
return { kind: 'dict' };
}
// Handle intN pattern (int1, int2, int3, ..., int257)
const intMatch = text.match(/^int(\d+)$/);
if (intMatch && intMatch[1]) {
return { kind: 'int', width: parseInt(intMatch[1], 10) };
}
// Handle uintN pattern (uint1, uint2, uint3, ..., uint256)
const uintMatch = text.match(/^uint(\d+)$/);
if (uintMatch && uintMatch[1]) {
return { kind: 'uint', width: parseInt(uintMatch[1], 10) };
}
// Handle bitsN pattern
const bitsMatch = text.match(/^bits(\d+)$/);
if (bitsMatch && bitsMatch[1]) {
return { kind: 'bits', width: parseInt(bitsMatch[1], 10) };
}
// Handle bytesN pattern
const bytesMatch = text.match(/^bytes(\d+)$/);
if (bytesMatch && bytesMatch[1]) {
return { kind: 'bytes', size: parseInt(bytesMatch[1], 10) };
}
return { kind: 'type_identifier', name: text };
}
case 'nullable_type': {
const innerNode = typeNode.namedChild(0);
return {
kind: 'nullable',
inner: innerNode ? this.resolveType(innerNode) : { kind: 'unknown' },
};
}
case 'union_type':
return this.processUnionType(typeNode);
case 'tensor_type':
return this.processTensorType(typeNode);
case 'tuple_type':
return this.processTupleType(typeNode);
case 'type_instantiatedTs':
return this.processGenericType(typeNode);
default:
// Handle special type patterns
const text = typeNode.text;
// Handle intN pattern (int1, int2, int3, ..., int257)
const intMatch = text.match(/^int(\d+)$/);
if (intMatch && intMatch[1]) {
return { kind: 'int', width: parseInt(intMatch[1], 10) };
}
// Handle uintN pattern (uint1, uint2, uint3, ..., uint256)
const uintMatch = text.match(/^uint(\d+)$/);
if (uintMatch && uintMatch[1]) {
return { kind: 'uint', width: parseInt(uintMatch[1], 10) };
}
// Handle bitsN pattern
const bitsMatch = text.match(/^bits(\d+)$/);
if (bitsMatch && bitsMatch[1]) {
return { kind: 'bits', width: parseInt(bitsMatch[1], 10) };
}
// Handle bytesN pattern
const bytesMatch = text.match(/^bytes(\d+)$/);
if (bytesMatch && bytesMatch[1]) {
return { kind: 'bytes', size: parseInt(bytesMatch[1], 10) };
}
// Handle Cell<T> pattern - parse inner type properly for nested generics
if (text.startsWith('Cell<') && text.endsWith('>')) {
const innerType = text.slice(5, -1);
return {
kind: 'cell_ref',
inner: this.parseGenericParamToken(innerType.trim()),
};
}
// Handle map<K, V> pattern for simple cases where parser reduced to identifier
if (text.startsWith('map<') && text.endsWith('>')) {
const inner = text.slice(4, -1);
const params = this.parseGenericParams(inner);
if (params.length >= 2 && params[0] && params[1]) {
return {
kind: 'map',
key: this.parseGenericParamToken(params[0]),
value: this.parseGenericParamToken(params[1]),
};
}
}
// Handle special types
if (['RemainingBitsAndRefs', 'slice', 'builder'].includes(text)) {
return { kind: 'special', name: text };
}
// If it's a known type identifier, return it as such
return { kind: 'type_identifier', name: text };
}
}
processUnionType(node) {
const alternatives = this.flattenUnion(node);
return { kind: 'union', alternatives };
}
flattenUnion(node) {
if (node.type !== 'union_type') {
return [this.resolveType(node)];
}
const lhsNode = node.childForFieldName('lhs');
const rhsNode = node.childForFieldName('rhs');
const lhsTypes = lhsNode ? this.flattenUnion(lhsNode) : [];
const rhsTypes = rhsNode ? this.flattenUnion(rhsNode) : [];
return [...lhsTypes, ...rhsTypes];
}
processTensorType(node) {
const items = [];
for (const child of node.namedChildren) {
child && items.push(this.resolveType(child));
}
return { kind: 'tensor', items };
}
processTupleType(node) {
const items = [];
for (const child of node.namedChildren) {
if (child)
items.push(this.resolveType(child));
}
return { kind: 'tuple', items };
}
processGenericType(node) {
// Extract generic name from type_identifier child
const nameNode = node.children.find((c) => c?.type === 'type_identifier');
const name = nameNode?.text;
if (!name) {
// Fallback to text-based parsing if AST structure is unexpected
return this.processGenericTypeFallback(node.text);
}
// Extract parameters from instantiationT_list child
const paramsListNode = node.children.find((c) => c?.type === 'instantiationT_list');
if (!paramsListNode) {
return { kind: 'type_identifier', name };
}
// Get type nodes from instantiationT_list (filter out punctuation like <, >, ,)
const paramNodes = paramsListNode.namedChildren.filter((c) => c && c.type !== 'comment');
// Handle Cell<T> specially
if (name === 'Cell' && paramNodes.length > 0 && paramNodes[0]) {
return {
kind: 'cell_ref',
inner: this.resolveType(paramNodes[0]),
};
}
// Handle map<K, V>
if (name.toLowerCase() === 'map' && paramNodes.length >= 2) {
const keyNode = paramNodes[0];
const valueNode = paramNodes[1];
if (keyNode && valueNode) {
return {
kind: 'map',
key: this.resolveType(keyNode),
value: this.resolveType(valueNode),
};
}
}
// Parse all type parameters recursively through AST
const params = paramNodes
.filter((n) => n !== null)
.map((paramNode) => this.resolveType(paramNode));
return {
kind: 'generic',
name,
params,
};
}
/**
* Fallback for text-based generic parsing when AST structure is unexpected
*/
processGenericTypeFallback(text) {
const match = text.match(/^(\w+)<(.+)>$/);
if (match && match[1] && match[2]) {
const [, name, paramsText] = match;
if (name === 'Cell') {
return {
kind: 'cell_ref',
inner: this.parseGenericParamToken(paramsText.trim()),
};
}
if (name.toLowerCase() === 'map') {
const paramTokens = this.parseGenericParams(paramsText);
if (paramTokens.length >= 2 && paramTokens[0] && paramTokens[1]) {
const key = this.parseGenericParamToken(paramTokens[0]);
const value = this.parseGenericParamToken(paramTokens[1]);
return { kind: 'map', key, value };
}
}
const paramTokens = this.parseGenericParams(paramsText);
const params = paramTokens.map((p) => this.parseGenericParamToken(p.trim()));
return { kind: 'generic', name, params };
}
return { kind: 'unknown' };
}
parseGenericParams(paramsText) {
// Parse comma-separated params with proper nesting support for generics
// Handles nested generics like Map<K, List<V>> correctly
const params = [];
let depth = 0;
let current = '';
for (let i = 0; i < paramsText.length; i++) {
const ch = paramsText[i];
if (ch === '<') {
depth++;
current += ch;
}
else if (ch === '>') {
depth--;
current += ch;
}
else if (ch === ',' && depth === 0) {
if (current.trim()) {
params.push(current.trim());
}
current = '';
}
else {
current += ch;
}
}
if (current.trim()) {
params.push(current.trim());
}
return params;
}
/**
* Split a string by pipe character (|) while respecting bracket nesting.
* Returns array of alternatives if union found, or null if no union.
*/
splitUnionAlternatives(text) {
const alternatives = [];
let depth = 0;
let current = '';
for (let i = 0; i < text.length; i++) {
const ch = text[i];
if (ch === '<' || ch === '(' || ch === '[') {
depth++;
current += ch;
}
else if (ch === '>' || ch === ')' || ch === ']') {
depth--;
current += ch;
}
else if (ch === '|' && depth === 0) {
alternatives.push(current.trim());
current = '';
}
else {
current += ch;
}
}
if (current.trim()) {
alternatives.push(current.trim());
}
// Only return as union if there are multiple alternatives
return alternatives.length > 1 ? alternatives : null;
}
parseGenericParamToken(trimmed) {
// Handle union types first (e.g., CreateProposal | CastVote)
const unionAlternatives = this.splitUnionAlternatives(trimmed);
if (unionAlternatives) {
return {
kind: 'union',
alternatives: unionAlternatives.map((alt) => this.parseGenericParamToken(alt)),
};
}
// Handle nullable types (e.g., cell?, address?, MyType?)
if (trimmed.endsWith('?')) {
const innerType = trimmed.slice(0, -1);
return {
kind: 'nullable',
inner: this.parseGenericParamToken(innerType),
};
}
// Handle primitive types
if ([
'uint8',
'uint16',
'uint32',
'uint64',
'uint128',
'uint256',
'int8',
'int16',
'int32',
'int64',
'int128',
'int256',
'bool',
'address',
'slice',
'cell',
'coins',
].includes(trimmed)) {
return { kind: 'primitive', name: trimmed };
}
// Handle custom signed integer types
const intMatch = trimmed.match(/^int(\d+)$/);
if (intMatch && intMatch[1]) {
return { kind: 'int', width: parseInt(intMatch[1], 10) };
}
// Handle custom unsigned integer types
const uintMatch = trimmed.match(/^uint(\d+)$/);
if (uintMatch && uintMatch[1]) {
return { kind: 'uint', width: parseInt(uintMatch[1], 10) };
}
// Handle nested generic types like Cell<T>
const genericMatch = trimmed.match(/^(\w+)<(.+)>$/);
if (genericMatch && genericMatch[1] && genericMatch[2]) {
const [, name, innerParamsText] = genericMatch;
// Handle Cell<T> specially
if (name === 'Cell') {
return {
kind: 'cell_ref',
inner: this.parseGenericParamToken(innerParamsText.trim()),
};
}
// Handle other generic types recursively
const innerParams = this.parseGenericParams(innerParamsText);
return {
kind: 'generic',
name,
params: innerParams.map((p) => this.parseGenericParamToken(p.trim())),
};
}
// Default to type identifier
return { kind: 'type_identifier', name: trimmed };
}
/**
* Get all parsed constants
*/
getConstants() {
return this.constants;
}
/**
* Extract constants from Tolk source code
*/
extractConstants(sourceCode, predicate) {
const tree = this.parse(sourceCode);
this.processAST(tree);
const result = {};
for (const [name, constant] of this.constants) {
if (!predicate || predicate(constant)) {
result[name] = constant.value;
}
}
return result;
}
}
exports.TolkASTProcessor = TolkASTProcessor;
//# sourceMappingURL=parser.js.map