UNPKG

repomix

Version:

A tool to pack repository contents to single file for AI consumption

564 lines (563 loc) 18.9 kB
const DEPENDENCY_FILES = { 'package.json': { language: 'Node.js', parser: parsePackageJson }, 'requirements.txt': { language: 'Python', parser: parseRequirementsTxt }, 'pyproject.toml': { language: 'Python', parser: parsePyprojectToml }, Pipfile: { language: 'Python', parser: parsePipfile }, 'go.mod': { language: 'Go', parser: parseGoMod }, 'Cargo.toml': { language: 'Rust', parser: parseCargoToml }, 'composer.json': { language: 'PHP', parser: parseComposerJson }, Gemfile: { language: 'Ruby', parser: parseGemfile }, 'pom.xml': { language: 'Java', parser: parsePomXml }, 'build.gradle': { language: 'Java/Kotlin', parser: parseBuildGradle }, 'build.gradle.kts': { language: 'Kotlin', parser: parseBuildGradle }, }; function parsePackageJson(content) { try { const pkg = JSON.parse(content); const dependencies = []; const devDependencies = []; const frameworks = []; if (pkg.dependencies) { for (const [name, version] of Object.entries(pkg.dependencies)) { dependencies.push({ name, version: String(version) }); if (name === 'react' || name === 'react-dom') frameworks.push('React'); if (name === 'vue') frameworks.push('Vue'); if (name === 'next') frameworks.push('Next.js'); if (name === 'nuxt') frameworks.push('Nuxt'); if (name === '@angular/core') frameworks.push('Angular'); if (name === 'express') frameworks.push('Express'); if (name === 'fastify') frameworks.push('Fastify'); if (name === 'hono') frameworks.push('Hono'); if (name === 'svelte') frameworks.push('Svelte'); } } if (pkg.devDependencies) { for (const [name, version] of Object.entries(pkg.devDependencies)) { devDependencies.push({ name, version: String(version), isDev: true }); if (name === 'typescript') frameworks.push('TypeScript'); } } let packageManager; if (pkg.packageManager) { const pm = String(pkg.packageManager); if (pm.startsWith('pnpm')) packageManager = 'pnpm'; else if (pm.startsWith('yarn')) packageManager = 'yarn'; else if (pm.startsWith('npm')) packageManager = 'npm'; else if (pm.startsWith('bun')) packageManager = 'bun'; } return { dependencies, devDependencies, frameworks: [...new Set(frameworks)], packageManager, }; } catch { return {}; } } function parseRequirementsTxt(content) { const dependencies = []; const frameworks = []; for (const line of content.split('\n')) { const trimmed = line.trim(); if (!trimmed || trimmed.startsWith('#') || trimmed.startsWith('-')) continue; const match = trimmed.match(/^([a-zA-Z0-9_-]+)([=<>!~]+)?(.+)?$/); if (match) { const name = match[1]; const version = match[3]; dependencies.push({ name, version }); if (name.toLowerCase() === 'django') frameworks.push('Django'); if (name.toLowerCase() === 'flask') frameworks.push('Flask'); if (name.toLowerCase() === 'fastapi') frameworks.push('FastAPI'); if (name.toLowerCase() === 'pytorch' || name.toLowerCase() === 'torch') frameworks.push('PyTorch'); if (name.toLowerCase() === 'tensorflow') frameworks.push('TensorFlow'); } } return { dependencies, frameworks: [...new Set(frameworks)] }; } function parsePyprojectToml(content) { const dependencies = []; const frameworks = []; const depsMatch = content.match(/\[project\.dependencies\]([\s\S]*?)(?=\[|$)/); if (depsMatch) { const depsSection = depsMatch[1]; const depLines = depsSection.match(/"([^"]+)"/g); if (depLines) { for (const dep of depLines) { const name = dep .replace(/"/g, '') .split(/[=<>!~]/)[0] .trim(); if (name) { dependencies.push({ name }); if (name.toLowerCase() === 'django') frameworks.push('Django'); if (name.toLowerCase() === 'flask') frameworks.push('Flask'); if (name.toLowerCase() === 'fastapi') frameworks.push('FastAPI'); } } } } return { dependencies, frameworks: [...new Set(frameworks)] }; } function parsePipfile(content) { const dependencies = []; const packagesMatch = content.match(/\[packages\]([\s\S]*?)(?=\[|$)/); if (packagesMatch) { const lines = packagesMatch[1].split('\n'); for (const line of lines) { const match = line.match(/^([a-zA-Z0-9_-]+)\s*=/); if (match) { dependencies.push({ name: match[1] }); } } } return { dependencies }; } function parseGoMod(content) { const dependencies = []; const frameworks = []; const requireMatch = content.match(/require\s*\(([\s\S]*?)\)/); if (requireMatch) { const lines = requireMatch[1].split('\n'); for (const line of lines) { const match = line.trim().match(/^([^\s]+)\s+([^\s]+)/); if (match) { const name = match[1]; const version = match[2]; dependencies.push({ name, version }); if (name.includes('gin-gonic/gin')) frameworks.push('Gin'); if (name.includes('labstack/echo')) frameworks.push('Echo'); if (name.includes('gofiber/fiber')) frameworks.push('Fiber'); } } } return { dependencies, frameworks: [...new Set(frameworks)] }; } function parseCargoToml(content) { const dependencies = []; const frameworks = []; const depsMatch = content.match(/\[dependencies\]([\s\S]*?)(?=\[|$)/); if (depsMatch) { const lines = depsMatch[1].split('\n'); for (const line of lines) { const match = line.match(/^([a-zA-Z0-9_-]+)\s*=/); if (match) { const name = match[1]; dependencies.push({ name }); if (name === 'actix-web') frameworks.push('Actix'); if (name === 'axum') frameworks.push('Axum'); if (name === 'rocket') frameworks.push('Rocket'); if (name === 'tokio') frameworks.push('Tokio'); } } } return { dependencies, frameworks: [...new Set(frameworks)] }; } function parseComposerJson(content) { try { const composer = JSON.parse(content); const dependencies = []; const frameworks = []; if (composer.require) { for (const [name, version] of Object.entries(composer.require)) { if (name !== 'php') { dependencies.push({ name, version: String(version) }); if (name.startsWith('laravel/')) frameworks.push('Laravel'); if (name.startsWith('symfony/')) frameworks.push('Symfony'); } } } return { dependencies, frameworks: [...new Set(frameworks)] }; } catch { return {}; } } function parseGemfile(content) { const dependencies = []; const frameworks = []; const gemMatches = content.matchAll(/gem\s+['"]([^'"]+)['"]/g); for (const match of gemMatches) { const name = match[1]; dependencies.push({ name }); if (name === 'rails') frameworks.push('Rails'); if (name === 'sinatra') frameworks.push('Sinatra'); } return { dependencies, frameworks: [...new Set(frameworks)] }; } function parsePomXml(content) { const dependencies = []; const frameworks = []; const depMatches = content.matchAll(/<dependency>[\s\S]*?<artifactId>([^<]+)<\/artifactId>[\s\S]*?<\/dependency>/g); for (const match of depMatches) { const name = match[1]; dependencies.push({ name }); if (name.includes('spring')) frameworks.push('Spring'); } return { dependencies, frameworks: [...new Set(frameworks)] }; } function parseBuildGradle(content) { const dependencies = []; const frameworks = []; const depMatches = content.matchAll(/(?:implementation|compile)\s*['"(]([^'"()]+)['"]/g); for (const match of depMatches) { const dep = match[1]; const parts = dep.split(':'); const name = parts.length >= 2 ? parts[1] : dep; dependencies.push({ name }); if (dep.includes('spring')) frameworks.push('Spring'); if (dep.includes('ktor')) frameworks.push('Ktor'); } return { dependencies, frameworks: [...new Set(frameworks)] }; } const VERSION_FILES = { '.node-version': parseNodeVersion, '.nvmrc': parseNodeVersion, '.ruby-version': parseRubyVersion, '.python-version': parsePythonVersion, '.go-version': parseGoVersion, '.java-version': parseJavaVersion, '.tool-versions': parseToolVersions, }; function parseNodeVersion(content) { const version = content.trim(); if (version) { return [{ runtime: 'Node.js', version }]; } return []; } function parseRubyVersion(content) { const version = content.trim(); if (version) { return [{ runtime: 'Ruby', version }]; } return []; } function parsePythonVersion(content) { const version = content.trim(); if (version) { return [{ runtime: 'Python', version }]; } return []; } function parseGoVersion(content) { const version = content.trim(); if (version) { return [{ runtime: 'Go', version }]; } return []; } function parseJavaVersion(content) { const version = content.trim(); if (version) { return [{ runtime: 'Java', version }]; } return []; } const CONFIG_FILE_PATTERNS = [ 'package.json', 'package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'bun.lockb', 'requirements.txt', 'pyproject.toml', 'Pipfile', 'Pipfile.lock', 'poetry.lock', 'go.mod', 'go.sum', 'Cargo.toml', 'Cargo.lock', 'composer.json', 'composer.lock', 'Gemfile', 'Gemfile.lock', 'pom.xml', 'build.gradle', 'build.gradle.kts', 'settings.gradle', 'settings.gradle.kts', 'tsconfig.json', 'jsconfig.json', 'vite.config.ts', 'vite.config.js', 'vite.config.mjs', 'vitest.config.ts', 'vitest.config.js', 'vitest.config.mjs', 'webpack.config.js', 'webpack.config.ts', 'rollup.config.js', 'rollup.config.ts', 'esbuild.config.js', 'turbo.json', 'biome.json', 'biome.jsonc', '.eslintrc', '.eslintrc.js', '.eslintrc.cjs', '.eslintrc.json', '.eslintrc.yaml', '.eslintrc.yml', 'eslint.config.js', 'eslint.config.mjs', 'eslint.config.cjs', '.prettierrc', '.prettierrc.js', '.prettierrc.json', '.prettierrc.yaml', '.prettierrc.yml', 'prettier.config.js', '.stylelintrc', '.stylelintrc.json', '.node-version', '.nvmrc', '.ruby-version', '.python-version', '.go-version', '.java-version', '.tool-versions', 'Dockerfile', 'docker-compose.yml', 'docker-compose.yaml', 'compose.yml', 'compose.yaml', '.github', '.gitlab-ci.yml', 'Jenkinsfile', '.circleci', '.travis.yml', '.editorconfig', '.gitignore', '.gitattributes', ]; function parseToolVersions(content) { const versions = []; const runtimeNameMap = { nodejs: 'Node.js', node: 'Node.js', ruby: 'Ruby', python: 'Python', golang: 'Go', go: 'Go', java: 'Java', rust: 'Rust', elixir: 'Elixir', erlang: 'Erlang', php: 'PHP', perl: 'Perl', deno: 'Deno', bun: 'Bun', }; for (const line of content.split('\n')) { const trimmed = line.trim(); if (!trimmed || trimmed.startsWith('#')) continue; const parts = trimmed.split(/\s+/); if (parts.length >= 2) { const tool = parts[0].toLowerCase(); const version = parts[1]; const runtime = runtimeNameMap[tool] || tool; versions.push({ runtime, version }); } } return versions; } const ROOT_DIR_LABEL = '.'; const getDirPath = (filePath) => { const lastSlash = filePath.lastIndexOf('/'); return lastSlash === -1 ? ROOT_DIR_LABEL : filePath.substring(0, lastSlash); }; export const detectTechStack = (processedFiles) => { const packageDirs = new Set(); for (const file of processedFiles) { const fileName = file.path.split('/').pop() || file.path; if (DEPENDENCY_FILES[fileName]) { packageDirs.add(getDirPath(file.path)); } } if (packageDirs.size === 0) { return []; } const resultMap = new Map(); for (const dir of packageDirs) { resultMap.set(dir, { path: dir, languages: [], frameworks: [], dependencies: [], devDependencies: [], runtimeVersions: [], configFiles: [], }); } for (const file of processedFiles) { const fileName = file.path.split('/').pop() || file.path; const dirPath = getDirPath(file.path); const result = resultMap.get(dirPath); if (!result) continue; const config = DEPENDENCY_FILES[fileName]; if (config) { result.languages.push(config.language); const parsed = config.parser(file.content); if (parsed.dependencies) { result.dependencies.push(...parsed.dependencies); } if (parsed.devDependencies) { result.devDependencies.push(...parsed.devDependencies); } if (parsed.frameworks) { result.frameworks.push(...parsed.frameworks); } if (parsed.packageManager && !result.packageManager) { result.packageManager = parsed.packageManager; } } const versionParser = VERSION_FILES[fileName]; if (versionParser) { const versions = versionParser(file.content); result.runtimeVersions.push(...versions); } if (CONFIG_FILE_PATTERNS.includes(fileName)) { result.configFiles.push(fileName); } } for (const result of resultMap.values()) { result.languages = [...new Set(result.languages)]; result.frameworks = [...new Set(result.frameworks)]; result.configFiles = [...new Set(result.configFiles)]; result.dependencies = deduplicateByName(result.dependencies); result.devDependencies = deduplicateByName(result.devDependencies); result.runtimeVersions = deduplicateRuntimeVersions(result.runtimeVersions); } return [...resultMap.values()].sort((a, b) => { if (a.path === ROOT_DIR_LABEL) return -1; if (b.path === ROOT_DIR_LABEL) return 1; return a.path.localeCompare(b.path); }); }; const deduplicateByName = (deps) => { const seen = new Set(); return deps.filter((dep) => { if (seen.has(dep.name)) return false; seen.add(dep.name); return true; }); }; const deduplicateRuntimeVersions = (versions) => { const seen = new Set(); return versions.filter((v) => { const normalizedVersion = v.version.replace(/^v/, ''); const key = `${v.runtime}:${normalizedVersion}`; if (seen.has(key)) return false; seen.add(key); return true; }); }; export const generateTechStackMd = (techStacks) => { const lines = ['# Tech Stacks', '']; for (const techStack of techStacks) { lines.push(`## Tech Stack: ${techStack.path}`, ''); if (techStack.languages.length > 0) { lines.push('### Languages'); lines.push(''); for (const lang of techStack.languages) { lines.push(`- ${lang}`); } lines.push(''); } if (techStack.frameworks.length > 0) { lines.push('### Frameworks'); lines.push(''); for (const fw of techStack.frameworks) { lines.push(`- ${fw}`); } lines.push(''); } if (techStack.runtimeVersions.length > 0) { lines.push('### Runtime Versions'); lines.push(''); for (const rv of techStack.runtimeVersions) { lines.push(`- ${rv.runtime}: ${rv.version}`); } lines.push(''); } if (techStack.packageManager) { lines.push('### Package Manager'); lines.push(''); lines.push(`- ${techStack.packageManager}`); lines.push(''); } if (techStack.dependencies.length > 0) { lines.push('### Dependencies'); lines.push(''); for (const dep of techStack.dependencies) { const version = dep.version ? ` (${dep.version})` : ''; lines.push(`- ${dep.name}${version}`); } lines.push(''); } if (techStack.devDependencies.length > 0) { lines.push('### Dev Dependencies'); lines.push(''); for (const dep of techStack.devDependencies) { const version = dep.version ? ` (${dep.version})` : ''; lines.push(`- ${dep.name}${version}`); } lines.push(''); } if (techStack.configFiles.length > 0) { lines.push('### Configuration Files'); lines.push(''); for (const file of techStack.configFiles) { lines.push(`- ${file}`); } lines.push(''); } } return lines.join('\n').trim(); };