@stackmemoryai/stackmemory
Version:
Project-scoped memory for AI coding tools. Durable context across sessions with MCP integration, frames, smart retrieval, Claude Code skills, and automatic hooks.
633 lines (631 loc) • 18.7 kB
JavaScript
import { fileURLToPath as __fileURLToPath } from 'url';
import { dirname as __pathDirname } from 'path';
const __filename = __fileURLToPath(import.meta.url);
const __dirname = __pathDirname(__filename);
import { Logger } from "../core/monitoring/logger.js";
import {
isChromaDBEnabled,
getChromaDBConfig
} from "../core/config/storage-config.js";
import * as fs from "fs";
import * as path from "path";
import * as crypto from "crypto";
import { execSync } from "child_process";
import ignore from "ignore";
async function getChromaDBAdapter() {
try {
return await import("../core/storage/chromadb-adapter.js");
} catch {
throw new Error(
"chromadb is not installed. Install it with: npm install chromadb"
);
}
}
class RepoIngestionSkill {
constructor(config, userId, teamId) {
this.config = config;
this.userId = userId;
this.teamId = teamId;
this.logger = new Logger("RepoIngestionSkill");
this.chromaEnabled = isChromaDBEnabled();
if (this.chromaEnabled) {
const chromaConfig = getChromaDBConfig();
if (chromaConfig && chromaConfig.apiKey) {
this.adapterConfig = {
apiKey: config?.apiKey || chromaConfig.apiKey,
tenant: config?.tenant || chromaConfig.tenant || "default_tenant",
database: config?.database || chromaConfig.database || "default_database",
collectionName: config?.collectionName || "stackmemory_repos"
};
}
}
}
logger;
adapter = null;
metadataCache = /* @__PURE__ */ new Map();
fileHashCache = /* @__PURE__ */ new Map();
chromaEnabled = false;
adapterConfig = null;
/**
* Check if ChromaDB is available for use
*/
isAvailable() {
return this.chromaEnabled && this.adapter !== null;
}
async initialize() {
if (!this.chromaEnabled || !this.adapterConfig) {
this.logger.warn(
"ChromaDB not enabled. Repository ingestion features are unavailable."
);
this.logger.warn('Run "stackmemory init --chromadb" to enable ChromaDB.');
return;
}
if (!this.adapter && this.adapterConfig) {
try {
const { ChromaDBAdapter } = await getChromaDBAdapter();
this.adapter = await ChromaDBAdapter.create(
this.adapterConfig,
this.userId,
this.teamId
);
} catch (error) {
this.logger.warn(
`Failed to initialize ChromaDB: ${error instanceof Error ? error.message : String(error)}`
);
this.logger.warn(
"chromadb is optional. Install it with: npm install chromadb"
);
return;
}
}
if (this.adapter) {
await this.adapter.initialize();
}
await this.loadMetadataCache();
}
/**
* Ingest a repository into ChromaDB
*/
async ingestRepository(repoPath, repoName, options = {}) {
if (!this.isAvailable()) {
return {
success: false,
message: 'ChromaDB not enabled. Run "stackmemory init --chromadb" to enable semantic search features.'
};
}
const startTime = Date.now();
try {
this.logger.info(`Starting repository ingestion for ${repoName}`);
if (!fs.existsSync(repoPath)) {
throw new Error(`Repository path not found: ${repoPath}`);
}
const metadata = await this.getRepoMetadata(repoPath, repoName);
const existingMetadata = this.metadataCache.get(metadata.repoId);
if (options.incremental && existingMetadata && !options.forceUpdate) {
const changedFiles = await this.getChangedFiles(
repoPath,
existingMetadata.lastCommit,
metadata.lastCommit
);
if (changedFiles.length === 0) {
return {
success: true,
message: "No changes detected since last ingestion"
};
}
this.logger.info(
`Incremental update: ${changedFiles.length} files changed`
);
}
const files = await this.getRepoFiles(repoPath, options);
this.logger.info(`Found ${files.length} files to process`);
let filesProcessed = 0;
let chunksCreated = 0;
let totalSize = 0;
for (const file of files) {
try {
const chunks = await this.processFile(
file,
repoPath,
repoName,
metadata,
options
);
for (const chunk of chunks) {
await this.storeChunk(chunk, metadata);
chunksCreated++;
}
filesProcessed++;
totalSize += fs.statSync(file).size;
if (filesProcessed % 100 === 0) {
this.logger.info(
`Processed ${filesProcessed}/${files.length} files`
);
}
} catch (error) {
this.logger.warn(`Failed to process file ${file}:`, error);
}
}
metadata.filesCount = filesProcessed;
metadata.totalSize = totalSize;
metadata.lastIngested = Date.now();
await this.saveMetadata(metadata);
const timeElapsed = Date.now() - startTime;
this.logger.info(
`Repository ingestion complete: ${filesProcessed} files, ${chunksCreated} chunks in ${timeElapsed}ms`
);
return {
success: true,
message: `Successfully ingested ${repoName}`,
stats: {
filesProcessed,
chunksCreated,
timeElapsed,
totalSize
}
};
} catch (error) {
this.logger.error("Repository ingestion failed:", error);
return {
success: false,
message: `Failed to ingest repository: ${error instanceof Error ? error.message : "Unknown error"}`
};
}
}
/**
* Update an existing repository in ChromaDB
*/
async updateRepository(repoPath, repoName, options = {}) {
const startTime = Date.now();
try {
const metadata = await this.getRepoMetadata(repoPath, repoName);
const existingMetadata = this.metadataCache.get(metadata.repoId);
if (!existingMetadata) {
return this.ingestRepository(repoPath, repoName, options);
}
const changedFiles = await this.getChangedFiles(
repoPath,
existingMetadata.lastCommit,
metadata.lastCommit
);
if (changedFiles.length === 0) {
return {
success: true,
message: "No changes detected",
stats: {
filesUpdated: 0,
filesAdded: 0,
filesRemoved: 0,
timeElapsed: Date.now() - startTime
}
};
}
let filesUpdated = 0;
let filesAdded = 0;
let filesRemoved = 0;
for (const change of changedFiles) {
const filePath = path.join(repoPath, change.path);
if (change.status === "deleted") {
await this.removeFileChunks(change.path, metadata.repoId);
filesRemoved++;
} else if (change.status === "added") {
const chunks = await this.processFile(
filePath,
repoPath,
repoName,
metadata,
options
);
for (const chunk of chunks) {
await this.storeChunk(chunk, metadata);
}
filesAdded++;
} else if (change.status === "modified") {
await this.removeFileChunks(change.path, metadata.repoId);
const chunks = await this.processFile(
filePath,
repoPath,
repoName,
metadata,
options
);
for (const chunk of chunks) {
await this.storeChunk(chunk, metadata);
}
filesUpdated++;
}
}
metadata.lastIngested = Date.now();
await this.saveMetadata(metadata);
const timeElapsed = Date.now() - startTime;
return {
success: true,
message: `Successfully updated ${repoName}`,
stats: {
filesUpdated,
filesAdded,
filesRemoved,
timeElapsed
}
};
} catch (error) {
this.logger.error("Repository update failed:", error);
return {
success: false,
message: `Failed to update repository: ${error instanceof Error ? error.message : "Unknown error"}`
};
}
}
/**
* Search code in ingested repositories
*/
async searchCode(query, options) {
if (!this.isAvailable() || !this.adapter) {
this.logger.warn("ChromaDB not enabled. Code search unavailable.");
return [];
}
try {
const filters = {
type: ["code_chunk"]
};
if (options?.repoName) {
filters.repo_name = options.repoName;
}
if (options?.language) {
filters.language = options.language;
}
const results = await this.adapter.queryContexts(
query,
options?.limit || 20,
filters
);
return results.map((result) => ({
filePath: result.metadata.file_path,
content: result.content,
score: 1 - result.distance,
// Convert distance to similarity score
startLine: result.metadata.start_line,
endLine: result.metadata.end_line,
repoName: result.metadata.repo_name
}));
} catch (error) {
this.logger.error("Code search failed:", error);
return [];
}
}
/**
* Get repository metadata
*/
async getRepoMetadata(repoPath, repoName) {
const branch = this.getCurrentBranch(repoPath);
const lastCommit = this.getLastCommit(repoPath);
const repoId = `${repoName}_${branch}`.replace(/[^a-zA-Z0-9_-]/g, "_");
const { language, framework } = await this.detectLanguageAndFramework(repoPath);
return {
repoId,
repoName,
branch,
lastCommit,
lastIngested: Date.now(),
filesCount: 0,
totalSize: 0,
language,
framework
};
}
/**
* Get current git branch
*/
getCurrentBranch(repoPath) {
try {
return execSync("git rev-parse --abbrev-ref HEAD", {
cwd: repoPath,
encoding: "utf8"
}).trim();
} catch {
return "main";
}
}
/**
* Get last commit hash
*/
getLastCommit(repoPath) {
try {
return execSync("git rev-parse HEAD", {
cwd: repoPath,
encoding: "utf8"
}).trim();
} catch {
return "unknown";
}
}
/**
* Get changed files between commits
*/
async getChangedFiles(repoPath, fromCommit, toCommit) {
try {
const diff = execSync(
`git diff --name-status ${fromCommit}..${toCommit}`,
{
cwd: repoPath,
encoding: "utf8"
}
);
return diff.split("\n").filter((line) => line.trim()).map((line) => {
const [status, ...pathParts] = line.split(" ");
return {
path: pathParts.join(" "),
status: status === "A" ? "added" : status === "D" ? "deleted" : "modified"
};
});
} catch {
return [];
}
}
/**
* Get repository files to process
*/
async getRepoFiles(repoPath, options) {
const files = [];
const ig = ignore();
const gitignorePath = path.join(repoPath, ".gitignore");
if (fs.existsSync(gitignorePath)) {
ig.add(fs.readFileSync(gitignorePath, "utf8"));
}
const defaultExcludes = [
"node_modules",
".git",
"dist",
"build",
"coverage",
".env",
"*.log",
...options.excludePatterns || []
];
ig.add(defaultExcludes);
const extensions = options.extensions || [
".ts",
".tsx",
".js",
".jsx",
".py",
".java",
".go",
".rs",
".c",
".cpp",
".h",
".hpp",
".cs",
".rb",
".php",
".swift",
".kt",
".scala",
".r",
".m",
".sql",
".yaml",
".yml",
".json"
];
if (options.includeDocs) {
extensions.push(".md", ".rst", ".txt");
}
const maxFileSize = options.maxFileSize || 1024 * 1024;
const walkDir = (dir, baseDir = repoPath) => {
const entries = fs.readdirSync(dir, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dir, entry.name);
const relativePath = path.relative(baseDir, fullPath);
if (ig.ignores(relativePath)) {
continue;
}
if (entry.isDirectory()) {
walkDir(fullPath, baseDir);
} else if (entry.isFile()) {
const ext = path.extname(entry.name);
if (!extensions.includes(ext)) {
continue;
}
if (!options.includeTests && (entry.name.includes(".test.") || entry.name.includes(".spec.") || relativePath.includes("__tests__") || relativePath.includes("test/") || relativePath.includes("tests/"))) {
continue;
}
const stats = fs.statSync(fullPath);
if (stats.size > maxFileSize) {
this.logger.debug(`Skipping large file: ${relativePath}`);
continue;
}
files.push(fullPath);
}
}
};
walkDir(repoPath);
return files;
}
/**
* Process a file into chunks
*/
async processFile(filePath, repoPath, repoName, metadata, options) {
const relativePath = path.relative(repoPath, filePath);
const content = fs.readFileSync(filePath, "utf8");
const lines = content.split("\n");
const language = this.detectFileLanguage(filePath);
const chunkSize = options.chunkSize || 100;
const chunks = [];
const fileHash = crypto.createHash("md5").update(content).digest("hex");
const cachedHash = this.fileHashCache.get(relativePath);
if (cachedHash === fileHash && !options.forceUpdate) {
return [];
}
this.fileHashCache.set(relativePath, fileHash);
for (let i = 0; i < lines.length; i += chunkSize) {
const chunkLines = lines.slice(i, Math.min(i + chunkSize, lines.length));
const chunkContent = chunkLines.join("\n");
if (chunkContent.trim().length === 0) {
continue;
}
const chunkId = `${metadata.repoId}_${relativePath}_${i}`;
const chunkHash = crypto.createHash("md5").update(chunkContent).digest("hex");
chunks.push({
id: chunkId,
filePath: relativePath,
content: chunkContent,
startLine: i + 1,
endLine: Math.min(i + chunkSize, lines.length),
hash: chunkHash,
language
});
}
return chunks;
}
/**
* Store a chunk in ChromaDB
*/
async storeChunk(chunk, metadata) {
if (!this.adapter) {
throw new Error("ChromaDB adapter not available");
}
const documentContent = `File: ${chunk.filePath} (Lines ${chunk.startLine}-${chunk.endLine})
Language: ${chunk.language}
Repository: ${metadata.repoName}/${metadata.branch}
${chunk.content}`;
if (!this.adapter) {
throw new Error("ChromaDB adapter not initialized");
}
await this.adapter.storeContext("observation", documentContent, {
type: "code_chunk",
repo_id: metadata.repoId,
repo_name: metadata.repoName,
branch: metadata.branch,
file_path: chunk.filePath,
start_line: chunk.startLine,
end_line: chunk.endLine,
language: chunk.language,
framework: metadata.framework,
chunk_hash: chunk.hash,
last_commit: metadata.lastCommit
});
}
/**
* Remove file chunks from ChromaDB
*/
async removeFileChunks(filePath, repoId) {
this.logger.debug(
`Would remove chunks for file: ${filePath} from repo: ${repoId}`
);
}
/**
* Detect file language
*/
detectFileLanguage(filePath) {
const ext = path.extname(filePath).toLowerCase();
const languageMap = {
".ts": "typescript",
".tsx": "typescript",
".js": "javascript",
".jsx": "javascript",
".py": "python",
".java": "java",
".go": "go",
".rs": "rust",
".c": "c",
".cpp": "cpp",
".cs": "csharp",
".rb": "ruby",
".php": "php",
".swift": "swift",
".kt": "kotlin",
".scala": "scala",
".r": "r",
".sql": "sql",
".yaml": "yaml",
".yml": "yaml",
".json": "json",
".md": "markdown"
};
return languageMap[ext] || "unknown";
}
/**
* Detect language and framework
*/
async detectLanguageAndFramework(repoPath) {
const packageJsonPath = path.join(repoPath, "package.json");
if (fs.existsSync(packageJsonPath)) {
try {
const packageJson = JSON.parse(
fs.readFileSync(packageJsonPath, "utf8")
);
const deps = {
...packageJson.dependencies,
...packageJson.devDependencies
};
let framework;
if (deps.react) framework = "react";
else if (deps.vue) framework = "vue";
else if (deps.angular) framework = "angular";
else if (deps.express) framework = "express";
else if (deps.next) framework = "nextjs";
else if (deps.svelte) framework = "svelte";
return {
language: deps.typescript ? "typescript" : "javascript",
framework
};
} catch {
}
}
if (fs.existsSync(path.join(repoPath, "requirements.txt")) || fs.existsSync(path.join(repoPath, "setup.py"))) {
return { language: "python" };
}
if (fs.existsSync(path.join(repoPath, "go.mod"))) {
return { language: "go" };
}
if (fs.existsSync(path.join(repoPath, "Cargo.toml"))) {
return { language: "rust" };
}
if (fs.existsSync(path.join(repoPath, "pom.xml")) || fs.existsSync(path.join(repoPath, "build.gradle"))) {
return { language: "java" };
}
return { language: "unknown" };
}
/**
* Load metadata cache
*/
async loadMetadataCache() {
this.metadataCache.clear();
}
/**
* Save metadata
*/
async saveMetadata(metadata) {
this.metadataCache.set(metadata.repoId, metadata);
}
/**
* Get repository statistics
*/
async getRepoStats(repoName) {
const stats = {
totalRepos: this.metadataCache.size,
totalFiles: 0,
totalChunks: 0,
languages: {},
frameworks: {}
};
for (const metadata of this.metadataCache.values()) {
if (!repoName || metadata.repoName === repoName) {
stats.totalFiles += metadata.filesCount;
if (metadata.language) {
stats.languages[metadata.language] = (stats.languages[metadata.language] || 0) + 1;
}
if (metadata.framework) {
stats.frameworks[metadata.framework] = (stats.frameworks[metadata.framework] || 0) + 1;
}
}
}
return stats;
}
}
export {
RepoIngestionSkill
};
//# sourceMappingURL=repo-ingestion-skill.js.map