spaider
Version:
Deterministic-first AI code assistant that crawls your codebase to implement changes using open source LLMs
196 lines • 7.81 kB
JavaScript
;
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Discovery = void 0;
const compromise_1 = __importDefault(require("compromise"));
const path = __importStar(require("path"));
const child_process_1 = require("child_process");
const util_1 = require("util");
const utils_1 = require("../lib/utils");
const const_1 = require("../lib/const");
const logger_1 = require("./logger");
var Discovery;
(function (Discovery) {
const execAsync = (0, util_1.promisify)(child_process_1.exec);
async function discoverFromPaths(filePaths, excludePaths, projectRoot) {
if (filePaths.length === 0)
return [];
const resolvedPaths = filePaths.map((filePath) => {
return (0, utils_1.resolveFilePath)(filePath, projectRoot);
});
return (0, utils_1.filterPathsWithinProject)(resolvedPaths, excludePaths, projectRoot);
}
Discovery.discoverFromPaths = discoverFromPaths;
async function discoverFromSearchTerms(searchTerms, excludePaths, projectRoot) {
if (searchTerms.length === 0)
return [];
const extractedTerms = [
...new Set(searchTerms.flatMap((term) => extractSearchTerms(term))),
];
if (extractedTerms.length === 0)
return [];
try {
const candidateFiles = await searchFiles(extractedTerms, projectRoot);
const filteredFiles = (0, utils_1.filterPathsWithinProject)(candidateFiles, excludePaths, projectRoot);
return filteredFiles.slice(0, extractedTerms.length * const_1.MAX_FILES_PER_TERM);
}
catch {
return [];
}
}
Discovery.discoverFromSearchTerms = discoverFromSearchTerms;
function extractSearchTerms(searchTerm) {
// Use NLP to extract meaningful terms
const doc = (0, compromise_1.default)(searchTerm);
const keywords = new Set();
// Define broad terms to filter out
const broadTerms = new Set([
"files",
"file",
"code",
"configuration",
"setup",
"any",
"existing",
"related",
"implementation",
"component",
"components",
"module",
"modules",
"system",
"application",
"app",
"project",
"service",
"services",
"utils",
"utilities",
"helper",
"helpers",
"common",
"shared",
"general",
"basic",
"simple",
"main",
"primary",
"secondary",
]);
// Extract topics, technical terms, and entities
doc
.topics()
.out("array")
.forEach((term) => {
if (!broadTerms.has(term.toLowerCase())) {
keywords.add(term);
}
});
doc
.nouns()
.out("array")
.forEach((noun) => {
if (noun.length > 2 &&
/^[a-zA-Z][a-zA-Z0-9]*$/.test(noun) &&
!broadTerms.has(noun.toLowerCase())) {
keywords.add(noun);
}
});
// Extract quoted content (file paths, specific terms)
const quotedMatches = searchTerm.match(/'([^']+)'|"([^"]+)"|`([^`]+)`/g);
if (quotedMatches) {
quotedMatches.forEach((match) => {
const cleaned = match.replace(/['"`]/g, "");
if (cleaned.length > 2 && !broadTerms.has(cleaned.toLowerCase())) {
keywords.add(cleaned);
}
});
}
// Filter out any remaining broad terms and return
return Array.from(keywords)
.filter((term) => !broadTerms.has(term.toLowerCase()))
.slice(0, const_1.MAX_SEARCH_TERMS_PER_HINT);
}
async function searchFiles(symbols, projectRoot, filePatterns = ["*.ts", "*.tsx", "*.js", "*.jsx"]) {
if (symbols.length === 0)
return [];
const escapedSymbols = symbols.map((symbol) => symbol.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"));
const pattern = `\\b(${escapedSymbols.join("|")})\\b`;
const typePatterns = filePatterns.map((p) => `-g "${p}"`).join(" ");
// Add exclusions for common directories that should be ignored
// Use ** to exclude node_modules at any depth
const exclusions = [
"--glob=!**/node_modules/**",
"--glob=!**/dist/**",
"--glob=!**/build/**",
"--glob=!**/.git/**",
"--glob=!**/coverage/**",
"--glob=!**/.next/**",
"--glob=!**/.nuxt/**",
].join(" ");
// Limit results and add timeout to prevent hanging
const rgCommand = `rg -l ${typePatterns} ${exclusions} --no-ignore-vcs --max-count ${50} "${pattern}" ${projectRoot}`;
logger_1.Logger.info(`Search: ${pattern}`);
logger_1.Logger.debug(`Search command: ${rgCommand}`);
try {
// Add 10 second timeout
const result = await Promise.race([
execAsync(rgCommand),
new Promise((_, reject) => setTimeout(() => reject(new Error("Search timeout")), const_1.MAX_SEARCH_TIMEOUT)),
]);
const files = result.stdout
.trim()
.split("\n")
.filter((line) => line.length > 0)
.map((file) => path.resolve(file))
.filter((file) => {
// Ensure the file is within the project root
const relativePath = path.relative(projectRoot, file);
return (!relativePath.startsWith("..") && !path.isAbsolute(relativePath));
})
.slice(0, const_1.MAX_SEARCH_FILES);
return [...new Set((0, utils_1.filterPathsWithinProject)(files, [], projectRoot))];
}
catch (error) {
logger_1.Logger.warn(`Search failed for pattern: ${pattern}`, error?.message || "Unknown error");
return [];
}
}
Discovery.searchFiles = searchFiles;
})(Discovery || (exports.Discovery = Discovery = {}));
//# sourceMappingURL=discovery.js.map