openai-code/src/vectorindex.mjs

An unofficial proxy layer that lets you use Anthropic Claude Code with any OpenAI API backend.

import { addToIndex, search, getByPath, storeToDisk, updateByPath, loadFromDisk, removeByPath, needsUpdate } from './vectordb.mjs'
import { safeReadFileSync, getAllSourceFilePaths } from './fs.mjs';
import { createEmbedding } from './openai.mjs';
import { escapeContent } from './prompts.mjs';

export const embedSingleDocument = async (filePath, _content) => {

  console.log("- Embedding single document:", filePath, "...")
  
  const content = _content ? _content : safeReadFileSync(filePath, 'utf-8')

  if (content === null) {
    return null
  }
  return createEmbedding(content)
}

/**
 * For every request, before searching the vector database,
 * this function ensures all relevant code files are added.
 *
 * It parses the extractedContext to get a list of file paths,
 * reads the file content for each path that is not yet indexed,
 * creates an embedding for the content, and adds it to the index.
 */
export const indexRelevantCodeFromContext = async (workingDirectory) => {
  
  console.log("- Updating Vector Database index from context:", workingDirectory)
  const absoluteFilePaths = getAllSourceFilePaths(workingDirectory)

  let allDocuments = loadFromDisk(workingDirectory)

  for (const filePath of absoluteFilePaths) {
    const existingDocument = getByPath(filePath)
    // Check if the file is not indexed or if it is recently updated
    if (!existingDocument || existingDocument.recentlyUpdated) {
      try {
        const embedding = await embedSingleDocument(filePath)
        if (!existingDocument) {
          addToIndex(filePath, embedding)
        } else {
          updateByPath(filePath, embedding)
        }
      } catch (error) {
        console.error(`Error indexing file ${filePath}:`, error)
      }
    }
  }

  // leftover documents that are not in the context should be removed
  const pathsToRemove = allDocuments.filter(doc => !absoluteFilePaths.includes(doc.path))
  for (const doc of pathsToRemove) {
    try {
      // remove document from index
      removeByPath(doc.path);
      console.log(`Removed document from index: ${doc.path}`)
    } catch (error) {
      console.error(`Error removing document ${doc.path}:`, error)
    }
  }

  allDocuments = loadFromDisk(workingDirectory)

  // any (outside) change: update embedding
  for (const document of allDocuments) {
    try {
      // safely read the file content
      const content = safeReadFileSync(document.path, 'utf-8')
      
      // check if the document needs an update
      if (needsUpdate(document.path)) {
        const newEmbedding = await createEmbedding(content)
        updateByPath(document.path, newEmbedding)
        console.log(`Updated document in index: ${document.path}`)
      }
    } catch (error) {
      console.error(`Error processing document ${document.path}:`, error)
    }
  }
  storeToDisk(workingDirectory)

  console.log("- Finished updating Vector Database index from context:", workingDirectory)
}

export const getRelevantCodeForPrompt = async (queryText, topK = 3) => {
  const embedding = await createEmbedding(queryText)
  const results = search({ embedding }, topK)
  let formatted = ''
  for (const result of results) {
    formatted += `<maybe_relevant_code comment="This is just top ${topK} candidates. Look further if necessary."><path>${result.path}</path><code>${escapeContent(result.source)}</code></maybe_relevant_code>\n`
  }
  return formatted
}