reporemix/dist/tree-sitter-parser.js

A opiniated repomix tool for Rust and NextJS projects.

"use strict";
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    var desc = Object.getOwnPropertyDescriptor(m, k);
    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
      desc = { enumerable: true, get: function() { return m[k]; } };
    }
    Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
    if (k2 === undefined) k2 = k;
    o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
    Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
    o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
    var ownKeys = function(o) {
        ownKeys = Object.getOwnPropertyNames || function (o) {
            var ar = [];
            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
            return ar;
        };
        return ownKeys(o);
    };
    return function (mod) {
        if (mod && mod.__esModule) return mod;
        var result = {};
        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
        __setModuleDefault(result, mod);
        return result;
    };
})();
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
    return new (P || (P = Promise))(function (resolve, reject) {
        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
        step((generator = generator.apply(thisArg, _arguments || [])).next());
    });
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.treeSitterParser = exports.TreeSitterParser = void 0;
/* eslint-disable @typescript-eslint/no-explicit-any */
const path = __importStar(require("node:path"));
const web_tree_sitter_1 = require("web-tree-sitter");
class TreeSitterParser {
    constructor() {
        this.initialized = false;
        this.initializationFailed = false;
        this.Parser = null;
        this.Language = null;
        this.parser = null;
        this.languages = new Map();
        this.queries = new Map();
        this.trees = new Map(); // Cache for incremental parsing
        this.languageConfigs = {
            javascript: {
                wasmPath: path.resolve(__dirname, '..', 'wasm', 'tree-sitter-javascript.wasm'),
                extensions: ['.js', '.jsx', '.mjs', '.cjs'],
                supportsComments: true,
                commentQuery: `
        (comment) @comment
      `,
            },
            typescript: {
                wasmPath: path.resolve(__dirname, '..', 'wasm', 'tree-sitter-typescript.wasm'),
                extensions: ['.ts'],
                supportsComments: true,
                commentQuery: `
        (comment) @comment
      `,
            },
            tsx: {
                wasmPath: path.resolve(__dirname, '..', 'wasm', 'tree-sitter-tsx.wasm'),
                extensions: ['.tsx'],
                supportsComments: true,
                commentQuery: `
        (comment) @comment
      `,
            },
            css: {
                wasmPath: path.resolve(__dirname, '..', 'wasm', 'tree-sitter-css.wasm'),
                extensions: ['.css', '.scss', '.sass'],
                supportsComments: true,
                commentQuery: `
        (comment) @comment
      `,
            },
            html: {
                wasmPath: path.resolve(__dirname, '..', 'wasm', 'tree-sitter-html.wasm'),
                extensions: ['.html', '.htm'],
                supportsComments: true,
                commentQuery: `
        (comment) @comment
      `,
            },
            python: {
                wasmPath: path.resolve(__dirname, '..', 'wasm', 'tree-sitter-python.wasm'),
                extensions: ['.py', '.pyi'],
                supportsComments: true,
                commentQuery: `
        (comment) @comment
      `,
            },
            rust: {
                wasmPath: path.resolve(__dirname, '..', 'wasm', 'tree-sitter-rust.wasm'),
                extensions: ['.rs'],
                supportsComments: true,
                commentQuery: `
        (line_comment) @comment
        (block_comment) @comment
      `,
            },
            toml: {
                wasmPath: path.resolve(__dirname, '..', 'wasm', 'tree-sitter-toml.wasm'),
                extensions: ['.toml'],
                supportsComments: true,
                commentQuery: `
        (comment) @comment
      `,
            },
            json: {
                wasmPath: path.resolve(__dirname, '..', 'wasm', 'tree-sitter-json.wasm'),
                extensions: ['.json'],
                supportsComments: false,
                commentQuery: '',
            },
            go: {
                wasmPath: path.resolve(__dirname, '..', 'wasm', 'tree-sitter-go.wasm'),
                extensions: ['.go'],
                supportsComments: true,
                commentQuery: `
        (comment) @comment
      `,
            },
            java: {
                wasmPath: path.resolve(__dirname, '..', 'wasm', 'tree-sitter-java.wasm'),
                extensions: ['.java'],
                supportsComments: true,
                commentQuery: `
        (line_comment) @comment
        (block_comment) @comment
      `,
            },
            c: {
                wasmPath: path.resolve(__dirname, '..', 'wasm', 'tree-sitter-c.wasm'),
                extensions: ['.c', '.h'],
                supportsComments: true,
                commentQuery: `
        (comment) @comment
      `,
            },
            cpp: {
                wasmPath: path.resolve(__dirname, '..', 'wasm', 'tree-sitter-cpp.wasm'),
                extensions: ['.cpp', '.cc', '.cxx', '.hpp', '.hh', '.hxx'],
                supportsComments: true,
                commentQuery: `
        (comment) @comment
      `,
            },
        };
    }
    init() {
        return __awaiter(this, void 0, void 0, function* () {
            var _a;
            if (this.initialized)
                return;
            if (this.initializationFailed)
                return;
            try {
                console.log('Initializing Tree-sitter parser...');
                // Import web-tree-sitter with proper ES module handling
                const module = yield Promise.resolve().then(() => __importStar(require('web-tree-sitter')));
                // Handle different module formats (CommonJS vs ES modules)
                // Type assertions are necessary due to the dynamic nature of the import
                const typedModule = module;
                if (typedModule.default) {
                    // ES module with default export
                    this.Parser = typedModule.default;
                    this.Language = typedModule.default.Language;
                }
                else if (typedModule.Parser) {
                    // CommonJS or direct export
                    this.Parser = typedModule.Parser;
                    this.Language = (_a = typedModule.Language) !== null && _a !== void 0 ? _a : null;
                }
                else {
                    // Module is the Parser itself
                    this.Parser = typedModule;
                    this.Language = typedModule.Language;
                }
                if (!this.Parser) {
                    throw new Error('Could not find Parser in web-tree-sitter module');
                }
                // Initialize with proper configuration
                yield this.Parser.init({
                    // Provide locateFile function for custom WASM loading
                    locateFile: (scriptName, scriptDirectory) => {
                        console.log(`Loading WASM file: ${scriptName} from ${scriptDirectory}`);
                        // In Node.js, we need to provide the full path to the WASM file
                        if (scriptName === 'tree-sitter.wasm') {
                            return path.join(scriptDirectory, scriptName);
                        }
                        return scriptName;
                    },
                });
                this.parser = new this.Parser();
                console.log('Tree-sitter parser core initialized successfully');
                this.initialized = true;
                console.log('Tree-sitter parser initialized');
            }
            catch (error) {
                console.error('Failed to initialize Tree-sitter:', error);
                this.initializationFailed = true;
                throw error;
            }
        });
    }
    loadLanguages() {
        return __awaiter(this, void 0, void 0, function* () {
            // Removed, as loading is now lazy
        });
    }
    isInitialized() {
        return this.initialized;
    }
    hasInitializationFailed() {
        return this.initializationFailed;
    }
    getLanguageForFile(filePath) {
        return __awaiter(this, void 0, void 0, function* () {
            var _a, _b;
            const ext = path.extname(filePath).toLowerCase();
            const loadedLanguage = (_a = this.languages.get(ext)) !== null && _a !== void 0 ? _a : null;
            if (loadedLanguage)
                return loadedLanguage;
            const config = Object.values(this.languageConfigs).find((c) => c.extensions.includes(ext));
            if (!config)
                return null;
            try {
                const wasmPath = config.wasmPath;
                const language = yield ((_b = this.Language) === null || _b === void 0 ? void 0 : _b.load(wasmPath));
                if (!language)
                    return null;
                this.languages.set(ext, language);
                // Create query if supported
                if (config.supportsComments && config.commentQuery.trim()) {
                    const query = new web_tree_sitter_1.Query(language, config.commentQuery);
                    this.queries.set(ext, query);
                }
                return language;
            }
            catch (error) {
                console.warn(`Failed to load language for ${ext}:`, error);
                return null;
            }
        });
    }
    getLanguageConfig(filePath) {
        const ext = path.extname(filePath).toLowerCase();
        for (const [langName, config] of Object.entries(this.languageConfigs)) {
            if (config.extensions.includes(ext)) {
                return config;
            }
        }
        return null;
    }
    getQueryForFile(filePath) {
        const ext = path.extname(filePath).toLowerCase();
        return this.queries.get(ext) || null;
    }
    parseFile(content, filePath, previousTree) {
        return __awaiter(this, void 0, void 0, function* () {
            if (!this.initialized || !this.parser) {
                console.warn('Tree-sitter parser not initialized');
                return null;
            }
            const language = yield this.getLanguageForFile(filePath);
            if (!language) {
                console.warn(`No language found for file: ${filePath}`);
                return null;
            }
            try {
                this.parser.setLanguage(language);
                const timeout = 100; // ms
                const parsePromise = new Promise((resolve) => {
                    if (!this.parser) {
                        throw new Error('Parser not initialized');
                    }
                    const tree = this.parser.parse(content, previousTree);
                    resolve(tree);
                });
                const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Parse timeout')), timeout));
                const tree = yield Promise.race([parsePromise, timeoutPromise]);
                if (!tree)
                    return null;
                if (this.hasSyntaxErrors(tree)) {
                    console.warn(`Syntax errors in ${filePath}`);
                }
                this.trees.set(filePath, tree);
                return tree;
            }
            catch (error) {
                console.error(`Failed to parse file ${filePath}:`, error);
                return null;
            }
        });
    }
    hasSyntaxErrors(tree) {
        function check(node) {
            if (node.type === 'ERROR')
                return true;
            for (let i = 0; i < node.childCount; i++) {
                const child = node.child(i);
                if (child && check(child))
                    return true;
            }
            return false;
        }
        return check(tree.rootNode);
    }
    // Extract comments using Tree-sitter queries (more accurate than traversal)
    extractComments(tree, filePath) {
        const query = this.getQueryForFile(filePath);
        if (!query || !tree) {
            return [];
        }
        try {
            const captures = query.captures(tree.rootNode);
            const comments = [];
            for (const capture of captures) {
                if (capture.name === 'comment') {
                    comments.push({
                        type: capture.node.type,
                        text: capture.node.text,
                        startIndex: capture.node.startIndex,
                        endIndex: capture.node.endIndex,
                        startPosition: capture.node.startPosition,
                        endPosition: capture.node.endPosition,
                    });
                }
            }
            return comments.sort((a, b) => a.startIndex - b.startIndex);
        }
        catch (error) {
            console.error(`Failed to extract comments from ${filePath}:`, error);
            return [];
        }
    }
    isMinified(tree) {
        const lines = tree.rootNode.text.split('\n').length;
        if (lines < 5)
            return false;
        const avgLineLength = tree.rootNode.text.length / lines;
        let nodeDepth = 0;
        function calculateDepth(node, depth) {
            nodeDepth = Math.max(nodeDepth, depth);
            for (let i = 0; i < node.childCount; i++) {
                const child = node.child(i);
                if (child)
                    calculateDepth(child, depth + 1);
            }
        }
        calculateDepth(tree.rootNode, 0);
        return avgLineLength > 80 && nodeDepth < 5; // Adjust thresholds based on heuristics
    }
    // Get supported file extensions
    getSupportedExtensions() {
        const extensions = new Set();
        for (const config of Object.values(this.languageConfigs)) {
            for (const ext of config.extensions) {
                extensions.add(ext);
            }
        }
        return Array.from(extensions).sort();
    }
    // Enhanced comment shortening with Tree-sitter
    shortenComments(content, filePath) {
        return __awaiter(this, void 0, void 0, function* () {
            if (!this.initialized || this.initializationFailed) {
                console.warn('Tree-sitter not available for comment shortening');
                return content;
            }
            try {
                const tree = yield this.parseFile(content, filePath);
                if (!tree)
                    return content;
                if (this.isMinified(tree)) {
                    console.log(`Skipping minified file: ${filePath}`);
                    return content;
                }
                const comments = this.extractComments(tree, filePath);
                if (comments.length === 0) {
                    return content;
                }
                const maxLen = 75;
                let newContent = content;
                // Process comments in reverse order to avoid index shifting
                const sortedComments = [...comments].reverse();
                for (const comment of sortedComments) {
                    if (comment.text.length <= maxLen) {
                        continue;
                    }
                    const shortenedComment = this.shortenSingleComment(comment, content, filePath, maxLen);
                    // Replace the comment in the content using template literals
                    newContent = `${newContent.slice(0, comment.startIndex)}${shortenedComment}${newContent.slice(comment.endIndex)}`;
                }
                return newContent;
            }
            catch (error) {
                console.error(`Tree-sitter comment processing failed for ${filePath}:`, error);
                return content;
            }
        });
    }
    shortenSingleComment(comment, content, filePath, maxLen) {
        const ext = path.extname(filePath).toLowerCase();
        const originalText = comment.text;
        // Extract leading whitespace from original content at comment position
        const lineStart = content.lastIndexOf('\n', comment.startIndex) + 1;
        const indent = content.substring(lineStart, comment.startIndex);
        if (originalText.length <= maxLen)
            return originalText;
        let prefix = '';
        let suffix = '';
        let contentStart = 0;
        let contentEnd = originalText.length;
        const isMultiLine = originalText.includes('\n');
        if (originalText.startsWith('//')) {
            prefix = '// ';
            suffix = '';
            contentStart = 3;
        }
        else if (originalText.startsWith('/*')) {
            prefix = '/* ';
            suffix = ext === '.css' ? '*/' : ' */';
            contentStart = 3;
            contentEnd -= 2;
            if (isMultiLine)
                prefix = '/*\n * ';
        }
        else if (originalText.startsWith('<!--')) {
            prefix = '<!-- ';
            suffix = ' -->';
            contentStart = 5;
            contentEnd -= 3;
        }
        else if (originalText.startsWith('#')) {
            prefix = '# ';
            suffix = '';
            contentStart = 2;
        }
        const coreContent = originalText.substring(contentStart, contentEnd).trim();
        const available = maxLen - indent.length - prefix.length - suffix.length - 3; // for '...'
        if (available <= 0)
            return `${indent}${prefix}...${suffix}`;
        let truncated = coreContent.substring(0, available);
        if (isMultiLine)
            truncated = truncated.split('\n')[0].trim();
        return `${indent}${prefix}${truncated}...${suffix}`;
    }
    // Update existing tree with edits (for incremental parsing)
    updateTree(filePath, edit) {
        const tree = this.trees.get(filePath);
        tree === null || tree === void 0 ? void 0 : tree.edit(edit);
    }
    // Clean up resources
    cleanup() {
        var _a;
        try {
            // Clean up queries
            for (const query of this.queries.values()) {
                query === null || query === void 0 ? void 0 : query.delete();
            }
            this.queries.clear();
            // Clean up trees
            for (const tree of this.trees.values()) {
                tree === null || tree === void 0 ? void 0 : tree.delete();
            }
            this.trees.clear();
            // Clean up parser
            (_a = this.parser) === null || _a === void 0 ? void 0 : _a.delete();
            console.log('Tree-sitter resources cleaned up');
        }
        catch (error) {
            console.error('Error during cleanup:', error);
        }
    }
    // Get parser statistics
    getStats() {
        return {
            initialized: this.initialized,
            languageCount: this.languages.size,
            supportedExtensions: this.getSupportedExtensions(),
            cachedTrees: this.trees.size,
            availableQueries: this.queries.size,
        };
    }
}
exports.TreeSitterParser = TreeSitterParser;
// Singleton instance
exports.treeSitterParser = new TreeSitterParser();
// Clean up on process exit
process.on('exit', () => {
    exports.treeSitterParser.cleanup();
});
process.on('SIGINT', () => {
    exports.treeSitterParser.cleanup();
    process.exit(0);
});
process.on('SIGTERM', () => {
    exports.treeSitterParser.cleanup();
    process.exit(0);
});