fileweaver/index.js

A CLI tool to weave files together based on regex patterns

#!/usr/bin/env node

import { program } from 'commander';
import fs from 'fs/promises';
import path from 'path';
import { glob } from 'glob';
import ora from 'ora';
import cliProgress from 'cli-progress';
import chalk from 'chalk';

// Función para detectar el lenguaje de programación basado en la extensión
function detectLanguage(filePath) {
  const extension = path.extname(filePath).toLowerCase();
  const extensionMap = {
    '.js': 'javascript',
    '.jsx': 'javascript',
    '.ts': 'typescript',
    '.tsx': 'typescript',
    '.py': 'python',
    '.rb': 'ruby',
    '.java': 'java',
    '.c': 'c',
    '.cpp': 'cpp',
    '.cs': 'csharp',
    '.go': 'go',
    '.rs': 'rust',
    '.php': 'php',
    '.html': 'html',
    '.css': 'css',
    '.scss': 'scss',
    '.md': 'markdown',
    '.json': 'json',
    '.yml': 'yaml',
    '.yaml': 'yaml',
    '.sh': 'bash',
    '.bash': 'bash',
    '.sql': 'sql',
    '.swift': 'swift',
    '.kt': 'kotlin',
    '.dart': 'dart',
  };
  
  return extensionMap[extension] || 'plaintext';
}

// Función para estimar el número de tokens (muy aproximado)
function estimateTokens(text) {
  // Aproximadamente 4 caracteres por token para la mayoría de los LLMs
  return Math.ceil(text.length / 4);
}

function generateTree(files, baseDir) {
  // Ordenar los archivos para una mejor visualización
  files = files.sort();
  
  // Convertir rutas absolutas a relativas
  const relativeFiles = files.map(file => path.relative(baseDir, file));
  
  // Crear estructura de árbol
  const tree = {};
  for (const file of relativeFiles) {
    const parts = file.split(path.sep);
    let current = tree;
    
    for (const part of parts) {
      if (!current[part]) {
        current[part] = {};
      }
      current = current[part];
    }
  }

  // Función para generar la representación en string del árbol
  function printTree(node, prefix = '', isLast = true) {
    const entries = Object.entries(node);
    let result = '';

    for (let [i, [key, value]] of entries.entries()) {
      const isLastEntry = i === entries.length - 1;
      const connector = isLastEntry ? '└── ' : '├── ';
      const newPrefix = prefix + (isLastEntry ? '    ' : '│   ');
      
      result += prefix + connector + chalk.cyan(key) + '\n';
      
      if (Object.keys(value).length > 0) {
        result += printTree(value, newPrefix, isLastEntry);
      }
    }
    
    return result;
  }

  return printTree(tree);
}

// Función para generar un resumen del proyecto
async function generateProjectSummary(directory, files) {
  let summary = "";
  
  // Buscar README.md
  const readmePath = path.join(directory, 'README.md');
  try {
    const readmeExists = await fs.access(readmePath).then(() => true).catch(() => false);
    if (readmeExists) {
      const readmeContent = await fs.readFile(readmePath, 'utf8');
      summary += "## Project README Summary\n\n";
      
      // Extraer solo las primeras líneas significativas del README
      const readmeLines = readmeContent.split('\n');
      const titleAndDescription = readmeLines.slice(0, 15).join('\n');
      summary += titleAndDescription + "\n\n";
    }
  } catch (error) {
    // Ignorar errores de README
  }
  
  // Buscar package.json para proyectos JS/TS
  const packageJsonPath = path.join(directory, 'package.json');
  try {
    const packageJsonExists = await fs.access(packageJsonPath).then(() => true).catch(() => false);
    if (packageJsonExists) {
      const packageJsonContent = await fs.readFile(packageJsonPath, 'utf8');
      const packageJson = JSON.parse(packageJsonContent);
      
      summary += "## Project Dependencies\n\n";
      summary += "### Main Dependencies:\n";
      if (packageJson.dependencies) {
        const mainDeps = Object.entries(packageJson.dependencies)
          .slice(0, 10) // Limitar a 10 dependencias principales
          .map(([name, version]) => `- ${name}: ${version}`)
          .join('\n');
        summary += mainDeps + "\n\n";
      }
      
      if (packageJson.scripts && Object.keys(packageJson.scripts).length > 0) {
        summary += "### Main Scripts:\n";
        const scripts = Object.entries(packageJson.scripts)
          .filter(([name]) => ['start', 'build', 'dev', 'test'].includes(name))
          .map(([name, command]) => `- ${name}: \`${command}\``)
          .join('\n');
        summary += scripts + "\n\n";
      }
    }
  } catch (error) {
    // Ignorar errores de package.json
  }
  
  // Estadísticas del código
  summary += "## Project Statistics\n\n";
  summary += `- Total Files: ${files.length}\n`;
  
  // Contar archivos por tipo
  const fileTypes = {};
  for (const file of files) {
    const ext = path.extname(file);
    fileTypes[ext] = (fileTypes[ext] || 0) + 1;
  }
  
  const topFileTypes = Object.entries(fileTypes)
    .sort((a, b) => b[1] - a[1])
    .slice(0, 5)
    .map(([ext, count]) => `- ${ext || 'No extension'}: ${count} files`)
    .join('\n');
  
  summary += "### File Types:\n" + topFileTypes + "\n";
  
  return summary;
}

program
  .name('fileweaver')
  .description('A powerful CLI tool for weaving files together with advanced pattern matching capabilities')
  .version('1.0.0')
  .option('-r, --regex <pattern>', 'regex pattern to match files')
  .option('-t, --tree <true|false>', 'add tree to output file')
  .option('-p, --prompt <prompt>', 'add prompt to output file')
  .option('-ir, --ignoreregex <pattern>', 'regex pattern to ignore files')
  .option('-d, --directory <path>', 'directory path', process.cwd())
  .option('-h, --headers', 'add file headers to content', false)
  .option('-o, --output <file>', 'output file name', 'output.txt')
  .option('-m, --max-size <size>', 'maximum output size in KB', Infinity)
  .option('-nc, --no-comments', 'strip comments from code files', false)
  .option('-s, --summary', 'include project summary', false)
  .option('-md, --markdown', 'format output as markdown', false)
  .option('-tok, --tokens', 'show token estimation', false)
  .option('-i, --important <files>', 'comma-separated list of important file patterns to prioritize')
  .parse(process.argv);

const options = program.opts();

async function weaveFiles() {
  const spinner = ora();
  const progressBar = new cliProgress.SingleBar({
    format: chalk.cyan('{bar}') + ' | {percentage}% | {value}/{total} Files | {file}',
    barCompleteChar: '\u2588',
    barIncompleteChar: '\u2591',
    hideCursor: true
  });

  try {
    // Iniciando búsqueda
    spinner.start(chalk.blue('Scanning directory...'));
    
    const directory = path.resolve(options.directory);
    const stats = await fs.stat(directory);
    
    if (!stats.isDirectory()) {
      spinner.fail(chalk.red('Error: Specified path is not a directory'));
      process.exit(1);
    }

    // Buscar archivos
    const searchPattern = path.join(directory, '**/*');
    let files = await glob(searchPattern, { 
      nodir: true,
      ignore: ['**/node_modules/**', '**/dist/**', '**/build/**', '**/.git/**'],
      absolute: true
    });

    // Aplicar filtros
    if (options.regex) {
      spinner.text = chalk.blue('Applying regex filter...');
      try {
        const regex = new RegExp(options.regex);
        files = files.filter(file => {
          // Usar la ruta relativa para aplicar la expresión regular
          const relativePath = path.relative(directory, file);
          return regex.test(relativePath);
        });
      } catch (error) {
        spinner.fail(chalk.red(`Error: Invalid regex pattern: ${error.message}`));
        process.exit(1);
      }
    }

    if (options.ignoreregex) {
      spinner.text = chalk.blue('Applying ignore patterns...');
      try {
        const ignoreRegex = new RegExp(options.ignoreregex);
        files = files.filter(file => {
          const relativePath = path.relative(directory, file);
          return !ignoreRegex.test(relativePath) && !ignoreRegex.test(file);
        });
      } catch (error) {
        spinner.fail(chalk.red(`Error: Invalid ignore regex pattern: ${error.message}`));
        process.exit(1);
      }
    }

    // Priorizar archivos importantes
    if (options.important) {
      spinner.text = chalk.blue('Prioritizing important files...');
      const importantPatterns = options.important.split(',').map(p => p.trim());
      
      // Separar archivos en importantes y no importantes
      const importantFiles = [];
      const otherFiles = [];
      
      for (const file of files) {
        const relativePath = path.relative(directory, file);
        const isImportant = importantPatterns.some(pattern => {
          try {
            return new RegExp(pattern).test(relativePath);
          } catch (error) {
            return relativePath.includes(pattern);
          }
        });
        
        if (isImportant) {
          importantFiles.push(file);
        } else {
          otherFiles.push(file);
        }
      }
      
      // Reorganizar la lista de archivos
      files = [...importantFiles, ...otherFiles];
    }

    if (files.length === 0) {
      spinner.fail(chalk.red('Error: No files found matching the specified patterns'));
      process.exit(1);
    }

    spinner.succeed(chalk.green(`Found ${files.length} files to process`));

    const tree = generateTree(files, directory);

    // Mostrar árbol de archivos
    console.log(chalk.yellow('\nFiles to be processed:'));
    console.log(tree);

    // Generar resumen del proyecto si está habilitado
    let summary = '';
    if (options.summary) {
      spinner.start(chalk.blue('Generating project summary...'));
      summary = await generateProjectSummary(directory, files);
      spinner.succeed(chalk.green('Generated project summary'));
    }

    // Iniciar la barra de progreso
    progressBar.start(files.length, 0, { file: 'Starting...' });

    // Leer y concatenar archivos
    let output = '';
    let processedFiles = 0;
    let totalTokens = 0;
    let totalSize = 0;
    const maxSize = options.maxSize * 1024; // Convertir a bytes

    // Agregar resumen al principio si está habilitado y usando markdown
    if (options.summary && options.markdown) {
      output += summary + '\n\n';
      totalSize += Buffer.from(summary).length;
      totalTokens += estimateTokens(summary);
    }

    for (const file of files) {
      try {
        let content = await fs.readFile(file, 'utf8');
        const relativePath = path.relative(directory, file);
        const language = detectLanguage(file);
        
        // Eliminar comentarios si la opción está habilitada
        if (options.noComments) {
          // Implementación muy básica - necesitaría una biblioteca de parsing real para hacerlo bien
          if (['javascript', 'typescript', 'java', 'cpp', 'csharp', 'go'].includes(language)) {
            content = content
              .replace(/\/\/.*$/gm, '') // Comentarios de línea
              .replace(/\/\*[\s\S]*?\*\//g, '') // Comentarios de bloque
              .replace(/^\s*\n/gm, ''); // Líneas vacías resultantes
          } else if (language === 'python') {
            content = content
              .replace(/#.*$/gm, '') // Comentarios de línea
              .replace(/^\s*\n/gm, ''); // Líneas vacías resultantes
          }
        }
        
        // Controlar tamaño máximo
        const contentSize = Buffer.from(content).length;
        if (totalSize + contentSize > maxSize && maxSize !== Infinity) {
          console.log(chalk.yellow(`\nReached maximum size limit (${options.maxSize}KB). Skipping remaining files.`));
          break;
        }
        
        // Formato de salida
        if (options.markdown) {
          output += `## File: ${relativePath}\n\n`;
          output += `\`\`\`${language}\n${content}\n\`\`\`\n\n`;
        } else if (options.headers) {
          output += `\n${'='.repeat(50)}\n`;
          output += `File: ${relativePath}\n`;
          output += `Language: ${language}\n`;
          if (options.tokens) {
            const fileTokens = estimateTokens(content);
            output += `Estimated Tokens: ${fileTokens}\n`;
            totalTokens += fileTokens;
          }
          output += `${'='.repeat(50)}\n\n`;
          output += content + '\n';
        } else {
          output += content + '\n';
          if (options.tokens && !options.markdown) {
            totalTokens += estimateTokens(content);
          }
        }
        
        totalSize += contentSize;
        processedFiles++;
        
        // Actualizar barra de progreso
        progressBar.update(processedFiles, { 
          file: chalk.blue(path.basename(file))
        });
      } catch (error) {
        console.error(chalk.red(`\nWarning: Could not read file ${file}: ${error.message}`));
      }
    }

    // Agregar árbol si está habilitada la opción
    if (options.tree) {
      const treeHeader = options.markdown ? '## Directory Tree\n\n```\n' : '\n' + '='.repeat(50) + '\nDirectory Tree:\n' + '='.repeat(50) + '\n\n';
      const treeFooter = options.markdown ? '\n```\n' : '';
      
      output += treeHeader + tree + treeFooter;
    }

    // Agregar resumen al final si está habilitado y no usando markdown
    if (options.summary && !options.markdown) {
      output += '\n' + '='.repeat(50) + '\n';
      output += 'Project Summary:\n';
      output += '='.repeat(50) + '\n\n';
      output += summary + '\n';
    }
    
    // Agregar prompt si está especificado
    if (options.prompt) {
      const promptHeader = options.markdown ? '## Prompt\n\n' : '\n' + '='.repeat(50) + '\nPrompt:\n' + '='.repeat(50) + '\n\n';
      
      output += promptHeader + options.prompt + '\n';
    }

    // Agregar información de tokens
    if (options.tokens) {
      const tokenHeader = options.markdown ? '## Token Estimation\n\n' : '\n' + '='.repeat(50) + '\nToken Estimation:\n' + '='.repeat(50) + '\n\n';
      
      output += tokenHeader + `Total estimated tokens: ${totalTokens}\n`;
      output += `This is approximately ${Math.round(totalTokens/1000)}K tokens.\n`;
    }

    progressBar.stop();

    // Escribir resultado
    spinner.start(chalk.blue('Saving output file...'));
    const outputPath = path.resolve(options.output);
    await fs.writeFile(outputPath, output.trim());
    
    const fileSizeKB = Math.round(totalSize / 1024);
    spinner.succeed(chalk.green(
      `Successfully processed ${processedFiles} file${processedFiles !== 1 ? 's' : ''} (${fileSizeKB}KB) and saved to ${outputPath}`
    ));
    
    if (options.tokens) {
      console.log(chalk.blue(`Estimated tokens: ${totalTokens} (approx. ${Math.round(totalTokens/1000)}K tokens)`));
    }
  } catch (error) {
    spinner.fail(chalk.red(`Error: ${error.message}`));
    process.exit(1);
  }
}

weaveFiles();