codesnap-analyzer
Version:
Create comprehensive snapshots of your codebase with token counting for LLMs
259 lines (258 loc) ⢠11.2 kB
JavaScript
;
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.DirectoryAnalyzer = void 0;
exports.analyze = analyze;
// src/core/analyzer.ts
const fs_extra_1 = __importDefault(require("fs-extra"));
const path_1 = __importDefault(require("path"));
const minimatch_1 = require("minimatch");
const analyze_1 = require("../constants/analyze");
const patterns_1 = require("../constants/patterns");
const file_1 = require("../utils/file");
const token_counter_1 = require("../utils/token-counter");
const formatter_1 = require("../services/formatter");
class DirectoryAnalyzer {
constructor(directory, options = {}) {
this.directory = path_1.default.resolve(directory);
this.baseDirectory = this.directory;
this.options = options;
this.ignorePatterns = new Set();
this.loadIgnorePatterns();
}
loadIgnorePatterns() {
// Add default patterns first
patterns_1.DEFAULT_IGNORE_PATTERNS.forEach((pattern) => this.ignorePatterns.add(pattern));
// Find and process all .gitignore files from root to target directory
let currentDir = this.directory;
const gitignoreFiles = [];
// Traverse up the directory tree to find all .gitignore files
while (currentDir !== path_1.default.dirname(currentDir)) {
const gitignorePath = path_1.default.join(currentDir, ".gitignore");
if (fs_extra_1.default.existsSync(gitignorePath)) {
gitignoreFiles.unshift(gitignorePath); // Add to start for correct priority
}
currentDir = path_1.default.dirname(currentDir);
}
// Process found .gitignore files
if (gitignoreFiles.length > 0) {
console.log("\nđ Found .gitignore files:");
gitignoreFiles.forEach((filePath) => {
console.log(` â
${filePath}`);
this.processGitignoreFile(filePath);
});
}
else {
console.log("\nâ ď¸ No .gitignore files found in directory hierarchy");
console.log(" Using default ignore patterns only");
}
// Log all active patterns
console.log("\nđ Active ignore patterns:");
console.log("\n Default patterns:");
patterns_1.DEFAULT_IGNORE_PATTERNS.forEach((pattern) => {
console.log(` - ${pattern}`);
});
if (gitignoreFiles.length > 0) {
console.log("\n From .gitignore files:");
Array.from(this.ignorePatterns)
.filter((pattern) => !patterns_1.DEFAULT_IGNORE_PATTERNS.includes(pattern))
.forEach((pattern) => {
console.log(` - ${pattern}`);
});
}
}
processGitignoreFile(gitignorePath) {
try {
const gitignoreDir = path_1.default.dirname(gitignorePath);
const content = fs_extra_1.default.readFileSync(gitignorePath, "utf-8");
const patterns = content
.split("\n")
.map((line) => line.trim())
.filter((line) => line && !line.startsWith("#"));
patterns.forEach((pattern) => {
if (pattern.startsWith("!")) {
// Handle negation patterns
const negatedPattern = pattern.slice(1);
this.ignorePatterns.delete(this.normalizePattern(negatedPattern, gitignoreDir));
}
else {
// Handle regular patterns
this.ignorePatterns.add(this.normalizePattern(pattern, gitignoreDir));
}
});
}
catch (error) {
console.warn(` â ď¸ Error processing ${gitignorePath}:`, error);
}
}
normalizePattern(pattern, gitignoreDir) {
// Remove leading slash
pattern = pattern.startsWith("/") ? pattern.slice(1) : pattern;
// Handle absolute paths relative to the gitignore location
const relativeToBase = path_1.default.relative(this.baseDirectory, gitignoreDir);
pattern = relativeToBase ? `${relativeToBase}/${pattern}` : pattern;
return pattern.replace(/\\/g, "/");
}
shouldIgnore(filePath) {
const relativePath = path_1.default
.relative(this.baseDirectory, filePath)
.replace(/\\/g, "/");
// Always ignore .venv and venv directories
if (relativePath.startsWith(".venv/") ||
relativePath.startsWith("venv/") ||
relativePath === ".venv" ||
relativePath === "venv") {
return true;
}
return Array.from(this.ignorePatterns).some((pattern) => {
// Convert the pattern to a proper minimatch pattern
let minimatchPattern = pattern;
// Handle patterns that should match directories and their contents
if (pattern.endsWith("/**")) {
return relativePath.startsWith(pattern.slice(0, -3));
}
// Handle directory-only patterns (ending with /)
if (pattern.endsWith("/")) {
minimatchPattern = pattern.slice(0, -1);
return new minimatch_1.Minimatch(minimatchPattern, {
dot: true,
matchBase: true,
}).match(relativePath);
}
// Handle standard file patterns
return new minimatch_1.Minimatch(pattern, {
dot: true,
matchBase: true,
}).match(relativePath);
});
}
async getAllFiles(dir) {
try {
const entries = await fs_extra_1.default.readdir(dir, { withFileTypes: true });
const files = [];
for (const entry of entries) {
const fullPath = path_1.default.join(dir, entry.name);
const relativePath = path_1.default
.relative(this.baseDirectory, fullPath)
.replace(/\\/g, "/");
// Early check for virtual environment directories
if (entry.isDirectory() &&
(entry.name === ".venv" || entry.name === "venv")) {
console.log(` âď¸ Skipping virtual environment directory: ${relativePath}`);
continue;
}
if (entry.isDirectory()) {
if (this.shouldIgnore(fullPath)) {
console.log(` âď¸ Skipping directory: ${relativePath} (matches ignore pattern)`);
continue;
}
files.push(...(await this.getAllFiles(fullPath)));
}
else {
if (this.shouldIgnore(fullPath)) {
console.log(` âď¸ Skipping file: ${relativePath} (matches ignore pattern)`);
continue;
}
files.push(fullPath);
}
}
return files.sort();
}
catch (error) {
console.error(`Error reading directory ${dir}:`, error);
return [];
}
}
async analyze() {
console.log("\nđ Starting directory analysis...");
console.log(`đ Base directory: ${this.directory}`);
const stats = {
totalFiles: 0,
totalSize: 0,
};
const files = [];
const maxFileSize = this.options.maxFileSize || analyze_1.FILE_LIMITS.MAX_FILE_SIZE;
console.log(`\nâď¸ Settings:`);
console.log(` Max file size: ${(maxFileSize / 1024 / 1024).toFixed(2)} MB`);
console.log(` Max total files: ${analyze_1.FILE_LIMITS.MAX_FILES}`);
console.log(` Max total size: ${(analyze_1.FILE_LIMITS.MAX_TOTAL_SIZE / 1024 / 1024).toFixed(2)} MB`);
console.log("\nđ Scanning for files...");
const allFiles = await this.getAllFiles(this.directory);
console.log(`⨠Found ${allFiles.length} files total`);
console.log("\nđ Processing files:");
for (const filePath of allFiles) {
const relativePath = path_1.default
.relative(this.directory, filePath)
.replace(/\\/g, "/");
const fileSize = (await fs_extra_1.default.stat(filePath)).size;
const isText = await file_1.FileUtils.isTextFile(filePath);
let content = null;
if (isText && fileSize <= maxFileSize) {
try {
content = await file_1.FileUtils.readFileContent(filePath);
console.log(` â
Reading: ${relativePath}`);
}
catch (error) {
console.warn(` â Failed to read: ${relativePath} - Error: ${error}`);
continue;
}
}
else {
const skipReason = [];
if (!isText)
skipReason.push("binary file");
if (fileSize > maxFileSize)
skipReason.push("file too large");
console.log(` âď¸ Skipping content for: ${relativePath} (${skipReason.join(", ")})`);
}
files.push({
path: relativePath,
content,
size: fileSize,
});
stats.totalFiles += 1;
stats.totalSize += fileSize;
if (stats.totalFiles > analyze_1.FILE_LIMITS.MAX_FILES) {
console.log("\nâ ď¸ Reached maximum file limit");
break;
}
if (stats.totalSize > analyze_1.FILE_LIMITS.MAX_TOTAL_SIZE) {
console.log("\nâ ď¸ Reached maximum total size limit");
break;
}
}
// Calculate tokens
const allContent = files
.map((file) => file.content)
.filter((content) => content !== null)
.join("\n");
const tokenCounts = await token_counter_1.TokenCounter.countTokens(allContent);
return {
files: files.sort((a, b) => a.path.localeCompare(b.path)),
stats,
tokenCounts,
};
}
}
exports.DirectoryAnalyzer = DirectoryAnalyzer;
async function analyze(directory, options = {}) {
const analyzer = new DirectoryAnalyzer(directory, options);
const { files, stats, tokenCounts } = await analyzer.analyze();
const summary = formatter_1.OutputFormatter.createSummary(directory, stats, tokenCounts);
const tree = formatter_1.OutputFormatter.createTree(files);
const content = formatter_1.OutputFormatter.createContent(files);
if (options.output) {
const outputContent = `${summary}\n\n${tree}\n\n${content}`;
await fs_extra_1.default.ensureDir(path_1.default.dirname(options.output));
await fs_extra_1.default.writeFile(options.output, outputContent, "utf-8");
}
return {
files,
stats,
tokenCounts,
summary,
tree,
};
}