termcode
Version:
Superior terminal AI coding agent with enterprise-grade security, intelligent error recovery, performance monitoring, and plugin system - Advanced Claude Code alternative
239 lines (238 loc) • 9.52 kB
JavaScript
import { promises as fs } from "node:fs";
import path from "node:path";
import os from "node:os";
import { getProvider } from "../providers/index.js";
import { log } from "../util/logging.js";
class LocalVectorStore {
dbPath;
initialized = false;
constructor() {
const offlineDir = path.join(os.homedir(), ".termcode", "offline");
this.dbPath = path.join(offlineDir, "vectors.json"); // Using JSON for simplicity, could upgrade to SQLite
}
async initialize() {
if (this.initialized)
return;
try {
await fs.mkdir(path.dirname(this.dbPath), { recursive: true });
// Check if database exists, create if not
try {
await fs.access(this.dbPath);
}
catch {
await fs.writeFile(this.dbPath, JSON.stringify([]), "utf8");
}
this.initialized = true;
log.info("Local vector store initialized");
}
catch (error) {
log.error("Failed to initialize local vector store:", error);
throw error;
}
}
async addEntries(entries) {
await this.initialize();
try {
const existing = await this.loadEntries();
// Remove existing entries for the same files to avoid duplicates
const filtered = existing.filter(entry => !entries.some(newEntry => newEntry.repoPath === entry.repoPath &&
newEntry.filePath === entry.filePath &&
newEntry.metadata.lineStart === entry.metadata.lineStart));
const updated = [...filtered, ...entries];
await fs.writeFile(this.dbPath, JSON.stringify(updated, null, 2), "utf8");
log.success(`Added ${entries.length} entries to local vector store`);
}
catch (error) {
log.error("Failed to add entries to vector store:", error);
throw error;
}
}
async searchSimilar(queryEmbedding, repoPath, limit = 12) {
await this.initialize();
try {
const entries = await this.loadEntries();
// Filter by repo if specified
const filteredEntries = repoPath
? entries.filter(entry => entry.repoPath === repoPath)
: entries;
// Calculate similarity scores
const scored = filteredEntries.map(entry => ({
...entry,
score: this.cosineSimilarity(queryEmbedding, entry.embedding)
}));
// Sort by similarity and return top results
scored.sort((a, b) => b.score - a.score);
return scored.slice(0, limit);
}
catch (error) {
log.error("Failed to search vector store:", error);
throw error;
}
}
async indexRepository(repoPath, provider = "ollama", model = "mxbai-embed-large") {
await this.initialize();
const providerInstance = getProvider(provider);
const entries = [];
log.info(`Indexing repository for offline use: ${path.basename(repoPath)}`);
try {
const files = await this.getCodeFiles(repoPath);
for (const filePath of files) {
const fullPath = path.join(repoPath, filePath);
const content = await fs.readFile(fullPath, "utf8");
const stats = await fs.stat(fullPath);
// Split large files into chunks
const chunks = this.chunkContent(content, 1000);
for (let i = 0; i < chunks.length; i++) {
const chunk = chunks[i];
const hash = this.simpleHash(chunk.content);
try {
// Generate embedding using local model
const embeddings = await providerInstance.embed([chunk.content], { model });
entries.push({
id: `${repoPath}:${filePath}:${i}`,
repoPath,
filePath,
content: chunk.content,
embedding: embeddings[0],
metadata: {
lastModified: stats.mtime.getTime(),
fileType: path.extname(filePath),
lineStart: chunk.lineStart,
lineEnd: chunk.lineEnd,
hash
}
});
}
catch (error) {
log.warn(`Failed to embed chunk from ${filePath}:`, error);
}
}
}
await this.addEntries(entries);
return entries.length;
}
catch (error) {
log.error("Failed to index repository:", error);
throw error;
}
}
async removeRepository(repoPath) {
await this.initialize();
try {
const entries = await this.loadEntries();
const filtered = entries.filter(entry => entry.repoPath !== repoPath);
await fs.writeFile(this.dbPath, JSON.stringify(filtered, null, 2), "utf8");
log.success(`Removed repository from local vector store: ${path.basename(repoPath)}`);
}
catch (error) {
log.error("Failed to remove repository from vector store:", error);
throw error;
}
}
async getStats() {
await this.initialize();
try {
const entries = await this.loadEntries();
const repos = [...new Set(entries.map(e => e.repoPath))];
const stats = await fs.stat(this.dbPath);
return {
totalEntries: entries.length,
repositories: repos,
totalSize: stats.size,
lastUpdated: stats.mtime.toISOString()
};
}
catch (error) {
return {
totalEntries: 0,
repositories: [],
totalSize: 0,
lastUpdated: new Date().toISOString()
};
}
}
async loadEntries() {
try {
const content = await fs.readFile(this.dbPath, "utf8");
return JSON.parse(content);
}
catch {
return [];
}
}
async getCodeFiles(repoPath) {
const files = [];
const extensions = ['.ts', '.js', '.tsx', '.jsx', '.py', '.go', '.rs', '.java', '.c', '.cpp', '.h'];
async function walkDir(dir, basePath = "") {
const entries = await fs.readdir(dir, { withFileTypes: true });
for (const entry of entries) {
if (entry.name.startsWith('.'))
continue;
const fullPath = path.join(dir, entry.name);
const relativePath = path.join(basePath, entry.name);
if (entry.isDirectory()) {
if (!['node_modules', 'dist', 'build', '.git'].includes(entry.name)) {
await walkDir(fullPath, relativePath);
}
}
else if (extensions.includes(path.extname(entry.name))) {
files.push(relativePath);
}
}
}
await walkDir(repoPath);
return files;
}
chunkContent(content, maxLines = 1000) {
const lines = content.split('\n');
const chunks = [];
for (let i = 0; i < lines.length; i += maxLines) {
const chunk = lines.slice(i, i + maxLines);
chunks.push({
content: chunk.join('\n'),
lineStart: i + 1,
lineEnd: Math.min(i + maxLines, lines.length)
});
}
return chunks;
}
cosineSimilarity(a, b) {
const dotProduct = a.reduce((sum, val, i) => sum + val * b[i], 0);
const magnitudeA = Math.sqrt(a.reduce((sum, val) => sum + val * val, 0));
const magnitudeB = Math.sqrt(b.reduce((sum, val) => sum + val * val, 0));
return dotProduct / (magnitudeA * magnitudeB);
}
simpleHash(content) {
let hash = 0;
for (let i = 0; i < content.length; i++) {
const char = content.charCodeAt(i);
hash = ((hash << 5) - hash) + char;
hash = hash & hash; // Convert to 32-bit integer
}
return hash.toString(36);
}
}
export const localVectorStore = new LocalVectorStore();
// Offline-compatible retrieval function
export async function retrieveOffline(repoPath, query, k = 12) {
try {
// Generate embedding for query using local model
const provider = getProvider("ollama");
const embeddings = await provider.embed([query], { model: "mxbai-embed-large" });
const queryEmbedding = embeddings[0];
// Search local vector store
const results = await localVectorStore.searchSimilar(queryEmbedding, repoPath, k);
// Convert to RetrievalChunk format
return results.map(result => ({
file: result.filePath,
start: result.metadata.lineStart,
end: result.metadata.lineEnd,
text: result.content,
score: 1.0 // Score already calculated in search
}));
}
catch (error) {
log.error("Offline retrieval failed:", error);
return [];
}
}