mongodb-rag
Version:
RAG (Retrieval Augmented Generation) library for MongoDB Vector Search
3,955 lines ā¢ 123 kB
Markdown
# cli/asciiLogo.js
```js
// src/cli/asciiLogo.js
import chalk from 'chalk';
export function displayLogo() {
// Using template literals for better readability
const logo = `
###%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%###
##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%##
##%%%%%%%%%%%%%%%%%%##%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#%%%%%%%%%%%%%%%##
#%%%%%%%%%%%%%%%%%%%*:-*#%%%%%%%%%%%%%##*+-::........:-=*##%%%%%%%%%%%%%##=.*%%%%%%%%%%%%%%%%#
@#%%%%%%%%%%%%%%%%%%%%+...:=*#%%%%%%#+-:....................:+#%%%%%%%#+-:..=%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%%#-.....:-*#%%%%#+-......:.:::::....-+#%%%%%*=:.....:*%%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%%%%#=:::::::.-*%%%%%*=:::::::::::-*%%%%%*-:::::::::+#%%%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%%%%%%%*=-::::::::=#@%%%+-::::::=#%%%#+-::::::::-+##%%%%%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%#+-:::::::+%%%#+:::-#%%%+-:::::::=*#%%%%%%%%%%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%%%%#==*%%@@@@%#+-:::::+%%%*-+%%%+:--::-=*%%%%%%%*==#%%%%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%%%#-::::+%@@@@@@%#+-----#%%%%%#=----=*%%%%%%%%*-::::*%%%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%%#=:::=%%@%+**#%@%%%+----*@%%#----=#%%%%#*++#@%%+-::-*%%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%%+---=%%%+-====*%%@%%%=---*@*----*%%%%#*=----=%%%+--:=#%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%#----#%%=-===+%#:.-#@%%+---=---+%%%#-..+%+----=%%#=---*%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%*----%%#-====#%%-.:#@%%%*=---=#%%%%%+..+@%+----#%%=---+%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%*----#%#=====#%%%@%%%%@%%%%%%%%%%#%%%@@%%%+====*%%=---+%%%%%%%%%%%%%%%%%#@
MongoDB RAG - Vector Search Magic
@#%%%%%%%%%%%%%%%%%%%%#====*%%+====+*%%%%#**#%%%%%%%%%****#%%%#+====+%%#==--*%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%%*====#%%++++++++++***#%%+-:-+%%#***++++++++++#%%+====#%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%%#+====%%%#++++******#@%#::::.*%%#***+++++++*%%%+=+==#%%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%%%#++++=*@@%%##**##%%@@@%+:::-%%%@%%##***##%%%#=++++*%%%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%%%%%*++++=+%@@@@@@@@*=+@@%=:-%@@*=+%@@%@@@@@*=+++++#%%%%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%%%%%%#*++***+======++++*@@%+#@%*+++++=====++*++++*%%%%%%%%%%%%%%%%%%%%%%#@
@#%%%%%%%%%%%%%%%%%%%%%%%#@%%%#*************#%@@@%%%%%#**++++++*+++**#%%%%#%%%%%%%%%%%%%%%%%%%#@
@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
`;
// Create a banner with the version number
const version = process.env.npm_package_version || '0.15.0';
const banner = `
${chalk.blue(logo)}
${chalk.yellow('š MongoDB RAG CLI')} ${chalk.green(`v${version}`)}
${chalk.cyan('Transform your data into intelligent insights with vector search\n')}`;
console.log(banner);
}
```
# cli/celebration.js
```js
// src/cli/celebration.js
import chalk from 'chalk';
const frames = [
`
š
\\|/
|
/ \\
`,
`
š
/|\\
|
/ \\
`,
`
āØ
\\|/
|
/ \\
`
];
// Create a multi-color border effect
function colorBorder(message) {
const border = '===============================';
return `
${chalk.blue(border)}
${chalk.green(message)}
${chalk.magenta(border)}
`;
}
export function celebrate(message) {
let frameIndex = 0;
const animation = setInterval(() => {
process.stdout.write('\x1B[2J\x1B[3J\x1B[H');
console.log(colorBorder(message));
console.log(chalk.cyan(frames[frameIndex]));
frameIndex = (frameIndex + 1) % frames.length;
}, 200);
// Stop after 2 seconds
setTimeout(() => {
clearInterval(animation);
process.stdout.write('\x1B[2J\x1B[3J\x1B[H');
console.log(colorBorder(message));
}, 2000);
}
```
# cli/createRagApp.js
```js
// src/cli/createRagApp.js
import fs from 'fs';
import path from 'path';
import { execSync } from 'child_process';
import chalk from 'chalk';
import { displayLogo } from './asciiLogo.js';
import MongoSpinner from './spinner.js';
import { celebrate } from './celebration.js';
import FunProgressBar from './progressBar.js';
function generateReadme(projectName) {
return `# ${projectName}
This RAG (Retrieval Augmented Generation) application was created with [mongodb-rag](https://npmjs.com/package/mongodb-rag).
## Getting Started
1. Configure your environment variables in \`.env\`:
- Set your MongoDB connection string
- Configure your embedding provider (OpenAI or Ollama)
- Adjust other settings as needed
2. Install dependencies:
\`\`\`bash
npm install
\`\`\`
3. Start the development server:
\`\`\`bash
npm run dev
\`\`\`
## Available CLI Commands
The mongodb-rag CLI provides several commands to manage your RAG application:
### Configuration
- \`npx mongodb-rag init\` - Initialize configuration
- \`npx mongodb-rag test-connection\` - Test provider connection
- \`npx mongodb-rag show-config\` - Display current configuration
- \`npx mongodb-rag edit-config\` - Modify configuration
### Vector Search Index Management
- \`npx mongodb-rag create-index\` - Create vector search index
- \`npx mongodb-rag show-indexes\` - List all indexes
- \`npx mongodb-rag delete-index\` - Remove an index
### Document Management
- \`npx mongodb-rag ingest --file <path>\` - Ingest a single file
- \`npx mongodb-rag ingest --directory <path>\` - Ingest a directory of files
- Options:
- \`--recursive\` - Process subdirectories
- \`--chunk-size <number>\` - Tokens per chunk
- \`--chunk-overlap <number>\` - Overlap between chunks
- \`--chunk-method <method>\` - Chunking strategy (fixed/recursive/semantic)
### Search
- \`npx mongodb-rag search "your query"\` - Search documents
- Options:
- \`--maxResults <number>\` - Maximum results (default: 5)
- \`--minScore <number>\` - Minimum similarity score (default: 0.7)
## API Endpoints
- POST \`/ingest\` - Ingest documents
- GET \`/search?query=<text>\` - Search documents
- DELETE \`/delete/:id\` - Delete a document
## Documentation
For detailed documentation, visit:
- [MongoDB RAG Documentation](https://mongodb-developer.github.io/mongodb-rag)
- [MongoDB Vector Search](https://www.mongodb.com/docs/atlas/atlas-vector-search/)
## License
This project is licensed under the MIT License.
`;
}
function createBackendFiles(projectPath) {
// Create backend directory structure
fs.mkdirSync(path.join(projectPath, 'backend'), { recursive: true });
fs.mkdirSync(path.join(projectPath, 'backend', 'config'), { recursive: true });
// Create backend package.json
fs.writeFileSync(path.join(projectPath, 'backend', 'package.json'), JSON.stringify({
name: `${path.basename(projectPath)}-backend`,
version: "1.0.0",
type: "module",
scripts: {
"start": "node server.js",
"dev": "nodemon server.js"
},
dependencies: {
"express": "^4.18.2",
"mongodb": "^6.3.0",
"dotenv": "^16.0.3",
"cors": "^2.8.5",
"mongodb-rag": "latest",
"nodemon": "^3.0.2"
}
}, null, 2));
// Create backend config
fs.writeFileSync(path.join(projectPath, 'backend', 'config', 'dbConfig.js'), `
import dotenv from 'dotenv';
dotenv.config();
export const config = {
mongoUrl: process.env.MONGODB_URI,
database: "mongorag",
collection: "documents",
embedding: {
provider: process.env.EMBEDDING_PROVIDER || "openai",
apiKey: process.env.EMBEDDING_API_KEY,
model: process.env.EMBEDDING_MODEL || "text-embedding-3-small",
dimensions: 1536
},
indexName: process.env.VECTOR_INDEX || "default"
};
`);
// Create backend server.js
fs.writeFileSync(path.join(projectPath, 'backend', 'server.js'), `
import express from 'express';
import cors from 'cors';
import { MongoRAG } from 'mongodb-rag';
import { config } from './config/dbConfig.js';
const app = express();
app.use(express.json());
app.use(cors());
const rag = new MongoRAG(config);
// Ingest Documents
app.post('/api/ingest', async (req, res) => {
try {
const { documents } = req.body;
const result = await rag.ingestBatch(documents);
res.json(result);
} catch (error) {
res.status(500).json({ error: error.message });
}
});
// Search Documents
app.get('/api/search', async (req, res) => {
try {
const { query } = req.query;
const results = await rag.search(query, { maxResults: 5 });
res.json(results);
} catch (error) {
res.status(500).json({ error: error.message });
}
});
// Delete a document
app.delete('/api/documents/:id', async (req, res) => {
try {
const col = await rag._getCollection();
await col.deleteOne({ documentId: req.params.id });
res.json({ message: 'Deleted successfully' });
} catch (error) {
res.status(500).json({ error: error.message });
}
});
const PORT = process.env.PORT || 5000;
app.listen(PORT, () => {
console.log(\`š Server running on port \${PORT}\`);
});
`);
}
function createFrontendFiles(projectPath) {
// Create frontend directory structure
fs.mkdirSync(path.join(projectPath, 'frontend'), { recursive: true });
fs.mkdirSync(path.join(projectPath, 'frontend', 'src'), { recursive: true });
fs.mkdirSync(path.join(projectPath, 'frontend', 'src', 'components'), { recursive: true });
fs.mkdirSync(path.join(projectPath, 'frontend', 'public'), { recursive: true });
// Create frontend package.json using Vite
fs.writeFileSync(path.join(projectPath, 'frontend', 'package.json'), JSON.stringify({
name: `${path.basename(projectPath)}-frontend`,
version: "1.0.0",
type: "module",
scripts: {
"dev": "vite",
"build": "vite build",
"preview": "vite preview"
},
dependencies: {
"react": "^18.2.0",
"react-dom": "^18.2.0",
"axios": "^1.6.2"
},
devDependencies: {
"@vitejs/plugin-react": "^4.2.1",
"vite": "^5.0.8"
}
}, null, 2));
// Create frontend components
const componentFiles = {
'Header.jsx': `
export function Header() {
return (
<header className="header">
<h1>MongoDB RAG Application</h1>
</header>
);
}`,
'Chatbot.jsx': `
import { useState } from 'react';
import axios from 'axios';
export function Chatbot() {
const [query, setQuery] = useState('');
const [results, setResults] = useState([]);
const [loading, setLoading] = useState(false);
const handleSearch = async (e) => {
e.preventDefault();
setLoading(true);
try {
const response = await axios.get(\`http://localhost:5000/api/search?query=\${query}\`);
setResults(response.data);
} catch (error) {
console.error('Search failed:', error);
}
setLoading(false);
};
return (
<div className="chatbot">
<form onSubmit={handleSearch}>
<input
type="text"
value={query}
onChange={(e) => setQuery(e.target.value)}
placeholder="Ask a question..."
/>
<button type="submit" disabled={loading}>
{loading ? 'Searching...' : 'Search'}
</button>
</form>
<div className="results">
{results.map((result, index) => (
<div key={index} className="result">
<p>{result.content}</p>
<small>Score: {result.score}</small>
</div>
))}
</div>
</div>
);
}`,
'Footer.jsx': `
export function Footer() {
return (
<footer className="footer">
<p>Built with <a href="https://npmjs.com/package/mongodb-rag">mongodb-rag</a></p>
</footer>
);
}`
};
// Create each component file
Object.entries(componentFiles).forEach(([filename, content]) => {
fs.writeFileSync(
path.join(projectPath, 'frontend', 'src', 'components', filename),
content
);
});
// Create main App.jsx
fs.writeFileSync(path.join(projectPath, 'frontend', 'src', 'App.jsx'), `
import { Header } from './components/Header';
import { Chatbot } from './components/Chatbot';
import { Footer } from './components/Footer';
import './styles.css';
function App() {
return (
<div className="app">
<Header />
<main>
<Chatbot />
</main>
<Footer />
</div>
);
}
export default App;
`);
// Create styles.css
fs.writeFileSync(path.join(projectPath, 'frontend', 'src', 'styles.css'), `
.app {
max-width: 800px;
margin: 0 auto;
padding: 2rem;
}
.header {
text-align: center;
margin-bottom: 2rem;
}
.chatbot {
margin: 2rem 0;
}
.chatbot form {
display: flex;
gap: 1rem;
margin-bottom: 1rem;
}
.chatbot input {
flex: 1;
padding: 0.5rem;
font-size: 1rem;
}
.chatbot button {
padding: 0.5rem 1rem;
background: #00684A;
color: white;
border: none;
border-radius: 4px;
cursor: pointer;
}
.chatbot button:disabled {
background: #ccc;
}
.results {
margin-top: 2rem;
}
.result {
padding: 1rem;
border: 1px solid #ddd;
margin-bottom: 1rem;
border-radius: 4px;
}
.footer {
text-align: center;
margin-top: 2rem;
padding-top: 1rem;
border-top: 1px solid #ddd;
}
`);
// Create index.html
fs.writeFileSync(path.join(projectPath, 'frontend', 'index.html'), `
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>MongoDB RAG App</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.jsx"></script>
</body>
</html>
`);
// Create main.jsx
fs.writeFileSync(path.join(projectPath, 'frontend', 'src', 'main.jsx'), `
import React from 'react'
import ReactDOM from 'react-dom/client'
import App from './App'
ReactDOM.createRoot(document.getElementById('root')).render(
<React.StrictMode>
<App />
</React.StrictMode>,
)
`);
}
export async function createRagApp(projectName) {
const spinner = new MongoSpinner();
const progressBar = new FunProgressBar();
// Display the ASCII logo
displayLogo();
const projectPath = path.resolve(process.cwd(), projectName);
if (fs.existsSync(projectPath)) {
console.error(chalk.red(`Error: Directory "${projectName}" already exists.`));
process.exit(1);
}
console.log(chalk.green(`\nš Creating a new RAG app in ${projectPath}\n`));
// Start the creation process with spinner
spinner.start('Initializing your RAG app');
fs.mkdirSync(projectPath, { recursive: true });
await new Promise(resolve => setTimeout(resolve, 1000));
spinner.stop(true);
// Show progress while creating files
console.log(chalk.cyan('\nPreparing your MongoDB RAG environment...'));
let currentProgress = 0;
progressBar.update(currentProgress);
// Create backend
console.log(chalk.blue('\nš Creating backend...'));
createBackendFiles(projectPath);
// Create frontend
console.log(chalk.blue('\nš Creating frontend...'));
createFrontendFiles(projectPath);
// Create root package.json for workspace
fs.writeFileSync(path.join(projectPath, 'package.json'), JSON.stringify({
name: projectName,
version: "1.0.0",
private: true,
workspaces: ["frontend", "backend"],
scripts: {
"dev": "concurrently \"npm run dev -w frontend\" \"npm run dev -w backend\"",
"build": "npm run build -w frontend",
"start": "npm run start -w backend"
},
devDependencies: {
"concurrently": "^8.2.2"
}
}, null, 2));
// Create root .env
fs.writeFileSync(path.join(projectPath, '.env'), `
MONGODB_URI=mongodb+srv://your_user:your_password@your-cluster.mongodb.net/mongorag
PORT=5000
# Embedding Configuration
EMBEDDING_PROVIDER=openai
EMBEDDING_API_KEY=your-embedding-api-key
EMBEDDING_MODEL=text-embedding-3-small
# MongoDB Vector Search Index
VECTOR_INDEX=default
`);
// Install dependencies
console.log(chalk.blue('\nš¦ Installing dependencies...'));
execSync(`cd ${projectPath} && npm install`, { stdio: 'inherit' });
// Create README
fs.writeFileSync(
path.join(projectPath, 'README.md'),
generateReadme(projectName)
);
// Show success message
celebrate('Full-Stack RAG App Created Successfully! š');
console.log(chalk.green('\nā
Project created successfully!'));
console.log(chalk.yellow('\nNext steps:'));
console.log(chalk.cyan(` 1. cd ${projectName}`));
console.log(chalk.cyan(' 2. Update .env with your MongoDB and API credentials'));
console.log(chalk.cyan(' 3. npm run dev # This will start both frontend and backend'));
}
```
# cli/playground.js
```js
// src/cli/playground.js
import { execSync } from 'child_process';
import fs from 'fs';
import path from 'path';
import express from 'express';
import { fileURLToPath } from 'url';
import open from 'open';
import { MongoRAG } from '../index.js';
import { createServer } from 'http';
import { Server } from 'socket.io';
import multer from 'multer';
import cors from 'cors';
import dotenv from 'dotenv';
import pdfParse from 'pdf-parse/lib/pdf-parse.js';
import detect from 'detect-port'; // Library to find available ports
dotenv.config();
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
// Properly resolve playground UI paths based on how the package is installed
const findPlaygroundUiPath = () => {
// Possible locations for playground-ui based on different installation scenarios
const possiblePaths = [
// When installed as a dependency in node_modules
path.resolve(__dirname, '../../src/playground-ui'),
// When running from source/development environment
path.resolve(__dirname, '../playground-ui'),
// When globally installed
path.resolve(process.env.npm_config_prefix || '/usr/local', 'lib/node_modules/mongodb-rag/src/playground-ui')
];
for (const potentialPath of possiblePaths) {
if (fs.existsSync(potentialPath)) {
// console.log(`ā
Found playground UI at: ${potentialPath}`);
return potentialPath;
}
}
console.warn('ā ļø Could not locate playground-ui directory');
return null;
};
const PLAYGROUND_UI_PATH = findPlaygroundUiPath();
const DEFAULT_BACKEND_PORT = 4000;
const DEFAULT_FRONTEND_PORT = 3000;
// Use environment variables or fallback
let backendPort = process.env.BACKEND_PORT || DEFAULT_BACKEND_PORT;
let playgroundPort = process.env.PLAYGROUND_PORT || DEFAULT_FRONTEND_PORT;
// Function to find an available port if the default is in use
const findAvailablePort = async (preferredPort, defaultPort) => {
const availablePort = await detect(preferredPort);
return availablePort === preferredPort ? preferredPort : await detect(defaultPort);
};
console.log('Current directory:', __dirname);
console.log('Process working directory:', process.cwd());
const upload = multer({ storage: multer.memoryStorage() });
export async function startPlayground() {
const app = express();
const server = createServer(app);
const io = new Server(server, { cors: { origin: '*' } });
app.use(express.json());
app.use(cors());
let mongodbUrl = process.env.MONGODB_URI;
let databaseName = 'playground';
let collectionName = 'documents';
const configPath = path.join(process.cwd(), '.mongodb-rag.json');
console.log("š Looking for config file at:", configPath);
console.log("š Current Working Directory:", process.cwd());
if (fs.existsSync(configPath)) {
try {
const config = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
console.log("ā
Loaded Configuration from File:", config);
mongodbUrl = config.mongoUrl || mongodbUrl;
databaseName = config.database || databaseName;
collectionName = config.collection || collectionName;
console.log("š Loaded MongoDB URL:", mongodbUrl);
console.log("š Loaded Database Name:", databaseName);
console.log("š Loaded Collection Name:", collectionName);
} catch (error) {
console.error("ā Error reading .mongodb-rag.json:", error.message);
}
} else {
console.warn("šØ Config file not found at:", configPath);
}
console.log("š Final MongoDB URL:", mongodbUrl);
console.log("š Final Database Name:", databaseName);
console.log("š Final Collection Name:", collectionName);
// Declare rag here to ensure it's in scope for all routes
let rag = null;
// Initialize RAG with configuration
try {
console.log("š¢ Before initializing MongoRAG:");
console.log(" š MongoDB URL:", mongodbUrl);
console.log(" š Database Name:", databaseName);
console.log(" š Collection Name:", collectionName);
rag = new MongoRAG({
mongoUrl: mongodbUrl,
database: databaseName,
collection: collectionName,
embedding: {
provider: process.env.EMBEDDING_PROVIDER || 'ollama',
apiKey: process.env.EMBEDDING_API_KEY,
model: process.env.EMBEDDING_MODEL || 'llama3',
dimensions: 1536,
baseUrl: process.env.EMBEDDING_BASE_URL || 'http://localhost:11434'
}
});
console.log("ā
MongoRAG Final Config:", JSON.stringify(rag.config, null, 2));
console.log("ā
After initializing MongoRAG:");
console.log(" š Database in rag.config:", rag.config.database);
console.log(" š Collection in rag.config:", rag.config.collection);
await rag.connect();
console.log('ā
Successfully connected to MongoDB');
} catch (error) {
console.error("ā ļø MongoDB Connection Error:", error.message);
console.info("ā¹ļø Playground will start with limited functionality");
}
// API endpoints...
app.post('/api/save-config', (req, res) => {
fs.writeFileSync('.mongodb-rag.json', JSON.stringify(req.body, null, 2));
res.json({ message: "Configuration saved successfully!" });
});
app.get("/api/config", (req, res) => {
if (!rag) {
return res.status(503).json({ error: "MongoDB connection not available" });
}
res.json({
mongoUrl: rag.config.mongoUrl || "Unknown",
database: rag.config.database || "Unknown",
collection: rag.config.collection || "Unknown",
provider: rag.config.embedding.provider || "Unknown",
apiKey: rag.config.embedding.apiKey || "Unknown",
model: rag.config.embedding.model || "Unknown",
dimensions: rag.config.embedding.dimensions || 1536,
batchSize: rag.config.embedding.batchSize || 100,
maxResults: rag.config.search.maxResults || 5,
minScore: rag.config.search.minScore || 0.7,
indexName: rag.config.indexName || "vector_index",
embeddingFieldPath: rag.config.embeddingFieldPath || "embedding"
});
});
app.post("/api/config", (req, res) => {
const newConfig = req.body;
// Update rag.config
rag.config.database = newConfig.database;
rag.config.collection = newConfig.collection;
rag.config.embeddingFieldPath = newConfig.embeddingFieldPath || "embedding";
rag.config.indexName = newConfig.indexName;
// Save to file (so changes persist)
fs.writeFileSync(".mongodb-rag.json", JSON.stringify(newConfig, null, 2));
res.json(newConfig);
});
// Add this new endpoint for creating vector search indexes
app.post('/api/indexes/create', async (req, res) => {
const existingIndexes = await collection.listIndexes().toArray();
if (existingIndexes.some(idx => idx.name === name)) {
return res.status(400).json({ error: "Index already exists" });
}
if (!rag) {
return res.status(503).json({
error: "MongoDB connection not available",
message: "Please check your MongoDB connection settings and try again"
});
}
try {
const { name, dimensions, path, similarity } = req.body;
const client = await rag.getClient();
const collection = client.db(rag.database).collection(rag.collection);
const indexConfig = {
name: name || "vector_index",
type: "vectorSearch",
definition: {
fields: [{
type: "vector",
path: path || "embedding",
numDimensions: dimensions || 1536,
similarity: similarity || "cosine"
}]
}
};
const result = await collection.createSearchIndex(indexConfig);
res.json({
success: true,
message: "Vector search index created successfully",
indexName: indexConfig.name,
result
});
} catch (error) {
console.error('Error creating index:', error);
res.status(500).json({ error: error.message });
}
});
// Add this endpoint to fetch documents
app.get('/api/documents', async (req, res) => {
if (!rag) {
return res.status(503).json({ error: "MongoDB connection not available" });
}
try {
const client = await rag.getClient();
const collection = client.db(rag.config.database).collection(rag.config.collection);
const documents = await collection.find({}).toArray();
res.json({ documents });
} catch (error) {
console.error('Error fetching documents:', error);
res.status(500).json({ error: error.message });
}
});
// Add this endpoint to fetch indexes
app.get('/api/indexes', async (req, res) => {
if (!rag) {
return res.status(503).json({ error: "MongoDB connection not available" });
}
try {
const client = await rag.getClient();
console.log("Using database for indexes:", rag.config.database);
console.log("Using collection for indexes:", rag.config.collection);
const collection = client.db(rag.config.database).collection(rag.config.collection);
// Fetch regular indexes
console.log("Fetching regular indexes...");
const regularIndexes = await collection.listIndexes().toArray();
console.log("Regular indexes fetched:", regularIndexes);
// Fetch search indexes
console.log("Fetching search indexes...");
const searchIndexes = await collection.aggregate([
{ $listSearchIndexes: {} }
]).toArray();
console.log("Search indexes fetched:", searchIndexes);
res.json({ regularIndexes, searchIndexes });
} catch (error) {
console.error('Error fetching indexes:', error);
res.status(500).json({ error: error.message });
}
});
// Add this endpoint to handle search queries
app.post('/api/search', async (req, res) => {
if (!rag) {
return res.status(503).json({ error: "MongoDB connection not available" });
}
const { query } = req.body;
try {
const client = await rag.getClient();
const collection = client.db(rag.config.database).collection(rag.config.collection);
// Perform a search using the query
const results = await collection.find({ $text: { $search: query } }).toArray();
res.json({ results });
} catch (error) {
console.error('Error performing search:', error);
res.status(500).json({ error: error.message });
}
});
io.on('connection', (socket) => {
socket.on('disconnect', () => {
console.log('User disconnected');
});
});
// Start the backend server
backendPort = await findAvailablePort(backendPort, DEFAULT_BACKEND_PORT);
server.listen(backendPort, () => {
console.log(`š Playground backend running at http://localhost:${backendPort}`);
});
// Serve the React UI with proper path resolution and build process
if (PLAYGROUND_UI_PATH) {
const frontendBuildPath = path.join(PLAYGROUND_UI_PATH, 'build');
const frontendDistPath = path.join(PLAYGROUND_UI_PATH, 'dist');
// Determine which directory to use (build or dist)
let uiBuildPath = fs.existsSync(frontendBuildPath) ? frontendBuildPath :
(fs.existsSync(frontendDistPath) ? frontendDistPath : null);
if (!uiBuildPath) {
console.warn("ā ļø Frontend build not found. Attempting to build...");
try {
// Check if package.json exists in the playground-ui directory
const packageJsonPath = path.join(PLAYGROUND_UI_PATH, 'package.json');
if (!fs.existsSync(packageJsonPath)) {
throw new Error("package.json not found in playground-ui directory");
}
// Attempt to install dependencies and build
execSync(`cd "${PLAYGROUND_UI_PATH}" && npm install && npm run build`, {
stdio: 'inherit',
timeout: 300000 // 5 minute timeout for build process
});
// Re-check build paths after building
uiBuildPath = fs.existsSync(frontendBuildPath) ? frontendBuildPath :
(fs.existsSync(frontendDistPath) ? frontendDistPath : null);
if (!uiBuildPath) {
throw new Error("Build completed but build directory not found");
}
} catch (error) {
console.error(`ā ļø Failed to build frontend: ${error.message}`);
console.info("ā¹ļø Starting in API-only mode");
return; // Exit frontend setup but keep backend running
}
}
const uiApp = express();
uiApp.use(express.static(uiBuildPath));
uiApp.get('*', (req, res) => {
res.sendFile(path.join(uiBuildPath, 'index.html'));
});
// Start the frontend server
playgroundPort = await findAvailablePort(playgroundPort, DEFAULT_FRONTEND_PORT);
uiApp.listen(playgroundPort, () => {
console.log(`š Playground UI running at http://localhost:${playgroundPort}`);
open(`http://localhost:${playgroundPort}`);
});
} else {
console.warn("ā ļø Playground UI components not found. Running in API-only mode.");
console.info("ā¹ļø You can still use the API endpoints at http://localhost:" + backendPort);
}
}
```
# cli/progressBar.js
```js
// src/cli/progressBar.js
import chalk from 'chalk';
const funFacts = [
"Did you know? Vector search helps find similar items even if they use different words!",
"MongoDB Atlas Vector Search uses cosine similarity by default š",
"RAG helps combine the power of vector search with your own data š",
"Vector embeddings can capture semantic meaning beyond keywords šÆ",
"MongoDB can handle billions of vectors efficiently! š",
"Vector search is like giving your database a human-like understanding š§ "
];
class FunProgressBar {
constructor() {
this.width = 40;
this.currentFact = 0;
}
update(progress) {
const filled = Math.round(this.width * progress);
const empty = this.width - filled;
const filledBar = 'ā'.repeat(filled);
const emptyBar = 'ā'.repeat(empty);
process.stdout.clearLine();
process.stdout.cursorTo(0);
const percentage = Math.round(progress * 100);
process.stdout.write(
chalk.blue(`[${filledBar}${emptyBar}] ${percentage}%\n`) +
chalk.yellow(`Fun Fact: ${funFacts[this.currentFact]}\n`)
);
this.currentFact = (this.currentFact + 1) % funFacts.length;
}
}
export default FunProgressBar;
```
# cli/spinner.js
```js
// src/cli/spinner.js
import chalk from 'chalk';
class MongoSpinner {
constructor() {
this.frames = [
'š ā ā ā ',
'š ā ā ā ',
'š ā ā ā ',
'š ā ā ā '
];
this.messages = [
'Preparing vector magic...',
'Calculating embeddings...',
'Optimizing search space...',
'Almost there...'
];
this.frameIndex = 0;
this.interval = null;
this.currentMessage = '';
this.isEnabled = this._checkEnabled();
}
_checkEnabled() {
// Check for required properties and methods
const requiredMethods = {
clearLine: process.stdout.clearLine,
cursorTo: process.stdout.cursorTo,
write: process.stdout.write
};
// Verify all methods exist and are functions
const hasRequiredMethods = Object.values(requiredMethods)
.every(method => method && typeof method === 'function');
return Boolean(
process.stdout.isTTY &&
!process.env.NO_SPINNER &&
!process.env.CI &&
hasRequiredMethods
);
}
start(text = '') {
this.currentMessage = text;
if (!this.isEnabled) {
// In non-interactive mode, just log once
this._fallbackLog(this.messages[0], text);
return;
}
process.stdout.write('\n');
this.interval = setInterval(() => {
const frame = this.frames[this.frameIndex];
const message = this.messages[this.frameIndex];
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(
chalk.blue(frame) + ' ' +
chalk.cyan(message) + ' ' +
chalk.yellow(this.currentMessage)
);
this.frameIndex = (this.frameIndex + 1) % this.frames.length;
}, 800);
}
updateMessage(text) {
this.currentMessage = text;
if (!this.isEnabled) {
this._fallbackLog(text);
return;
}
}
stop(success = true) {
if (this.interval) {
clearInterval(this.interval);
this.interval = null;
}
if (this.isEnabled) {
process.stdout.clearLine();
process.stdout.cursorTo(0);
if (success) {
process.stdout.write(
chalk.green('āØ Vector magic complete! ') +
chalk.blue('š\n')
);
} else {
process.stdout.write(
chalk.red('ā Operation failed ') +
chalk.yellow('š¢\n')
);
}
}
}
_fallbackLog(...messages) {
// Use plain console.log for non-interactive environments
console.log(...messages);
}
}
export default MongoSpinner;
```
# core/BatchProcessor.js
```js
import debug from 'debug';
import EventEmitter from 'events';
const log = debug('mongodb-rag:batch');
/**
* Handles batch processing of items with retry logic and concurrency control
* @extends EventEmitter
* @fires BatchProcessor#progress
* @fires BatchProcessor#batchError
*/
class BatchProcessor extends EventEmitter {
/**
* Creates a new batch processor instance
* @param {Object} options - Configuration options
* @param {number} [options.batchSize=100] - Number of items to process in each batch
* @param {number} [options.concurrency=2] - Number of batches to process concurrently
* @param {number} [options.retryAttempts=3] - Maximum number of retry attempts per batch
* @param {number} [options.retryDelay=1000] - Delay between retries in milliseconds
*/
constructor(options = {}) {
super();
this.options = {
batchSize: options.batchSize || 100,
concurrency: options.concurrency || 2,
retryAttempts: options.retryAttempts || 3,
retryDelay: options.retryDelay || 1000,
...options
};
this.stats = {
processed: 0,
failed: 0,
retried: 0,
total: 0
};
}
/**
* Processes a single batch of items with retry logic
* @param {Array} items - Items to process in this batch
* @param {Function} processor - Function to process the items
* @returns {Promise<{results: Array, errors: Array}>} Results and errors from processing
* @fires BatchProcessor#batchError
*/
async processBatch(items, processor) {
const results = [];
const errors = [];
try {
log(`Processing batch of ${items.length} items`);
for (let attempt = 1; attempt <= this.options.retryAttempts; attempt++) {
try {
const batchResults = await processor(items);
results.push(...batchResults);
this.stats.processed += items.length;
break;
} catch (error) {
if (attempt === this.options.retryAttempts) {
log(`Batch failed after ${attempt} attempts`);
this.stats.failed += items.length;
errors.push({ items, error });
throw error;
}
this.stats.retried += items.length;
log(`Retry attempt ${attempt} after error: ${error.message}`);
await this._sleep(this.options.retryDelay * attempt);
}
}
} catch (error) {
this.emit('batchError', { items, error });
}
return { results, errors };
}
/**
* Processes all items in batches with concurrency control
* @param {Array} items - All items to process
* @param {Function} processor - Function to process each batch
* @returns {Promise<{results: Array, errors: Array}>} Combined results and errors from all batches
* @fires BatchProcessor#progress
*/
async process(items, processor) {
this.stats.total = items.length;
const batches = this._createBatches(items);
const results = [];
const errors = [];
// Process batches with concurrency control
for (let i = 0; i < batches.length; i += this.options.concurrency) {
const batchPromises = batches
.slice(i, i + this.options.concurrency)
.map(batch => this.processBatch(batch, processor));
const batchResults = await Promise.allSettled(batchPromises);
batchResults.forEach((result, index) => {
if (result.status === 'fulfilled') {
results.push(...result.value.results);
errors.push(...result.value.errors);
} else {
const failedBatch = batches[i + index];
errors.push({
items: failedBatch,
error: result.reason
});
}
});
/**
* Progress event
* @event BatchProcessor#progress
* @type {Object}
* @property {number} processed - Number of successfully processed items
* @property {number} failed - Number of failed items
* @property {number} total - Total number of items
* @property {number} percent - Percentage of completion
*/
this.emit('progress', {
processed: this.stats.processed,
failed: this.stats.failed,
total: this.stats.total,
percent: Math.round((this.stats.processed + this.stats.failed) / this.stats.total * 100)
});
}
return { results, errors };
}
/**
* Splits array of items into batches
* @private
* @param {Array} items - Items to split into batches
* @returns {Array<Array>} Array of batches
*/
_createBatches(items) {
const batches = [];
for (let i = 0; i < items.length; i += this.options.batchSize) {
batches.push(items.slice(i, i + this.options.batchSize));
}
return batches;
}
/**
* Utility function to pause execution
* @private
* @param {number} ms - Milliseconds to sleep
* @returns {Promise<void>}
*/
_sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Returns current processing statistics
* @returns {Object} Current processing stats including success rate
* @property {number} processed - Number of successfully processed items
* @property {number} failed - Number of failed items
* @property {number} retried - Number of retried items
* @property {number} total - Total number of items
* @property {string} successRate - Percentage of successful processing
*/
getStats() {
return {
...this.stats,
successRate: this.stats.total ?
(this.stats.processed / this.stats.total * 100).toFixed(2) + '%' :
'0%'
};
}
}
export default BatchProcessor;
```
# core/CacheManager.js
```js
import debug from 'debug';
const log = debug('mongodb-rag:cache');
/**
* Manages an in-memory cache with TTL and size limits
* Provides caching functionality with automatic expiration and eviction
*/
class CacheManager {
/**
* Creates a new cache manager instance
* @param {Object} options - Configuration options
* @param {number} [options.ttl=3600] - Time-to-live in seconds (default 1 hour)
* @param {number} [options.maxSize=1000] - Maximum number of entries in cache
*/
constructor(options = {}) {
this.options = {
ttl: options.ttl || 3600, // 1 hour default TTL
maxSize: options.maxSize || 1000,
...options
};
this.cache = new Map();
this.keyTimestamps = new Map();
this.stats = {
hits: 0,
misses: 0,
evictions: 0
};
}
/**
* Retrieves a value from cache
* @param {string} key - Cache key to lookup
* @returns {Promise<*|null>} Cached value or null if not found/expired
*/
async get(key) {
const entry = this.cache.get(key);
if (!entry) {
this.stats.misses++;
return null;
}
if (this._isExpired(key)) {
this.delete(key);
this.stats.misses++;
return null;
}
this.stats.hits++;
return entry.value;
}
/**
* Stores a value in cache
* @param {string} key - Cache key
* @param {*} value - Value to store
* @returns {Promise<boolean>} True if value was stored successfully
*/
async set(key, value) {
if (this.cache.size >= this.options.maxSize) {
this._evictOldest();
}
this.cache.set(key, {
value,
timestamp: Date.now()
});
this.keyTimestamps.set(key, Date.now());
return true;
}
/**
* Removes an entry from cache
* @param {string} key - Cache key to remove
* @returns {Promise<void>}
*/
async delete(key) {
this.cache.delete(key);
this.keyTimestamps.delete(key);
}
/**
* Clears all entries from cache and resets statistics
* @returns {Promise<void>}
*/
async clear() {
this.cache.clear();
this.keyTimestamps.clear();
this.stats = {
hits: 0,
misses: 0,
evictions: 0
};
}
/**
* Returns current cache statistics
* @returns {Object} Cache statistics
* @property {number} hits - Number of cache hits
* @property {number} misses - Number of cache misses
* @property {number} evictions - Number of evicted entries
* @property {number} size - Current number of entries
* @property {number} maxSize - Maximum allowed entries
* @property {number} hitRate - Cache hit rate (0-1)
*/
getStats() {
return {
...this.stats,
size: this.cache.size,
maxSize: this.options.maxSize,
hitRate: this._calculateHitRate()
};
}
/**
* Checks if a cache entry has expired
* @private
* @param {string} key - Cache key to check
* @returns {boolean} True if entry has expired
*/
_isExpired(key) {
const timestamp = this.keyTimestamps.get(key);
if (!timestamp) return true;
const age = Date.now() - timestamp;
return age > this.options.ttl * 1000;
}
/**
* Removes the oldest entry from cache
* @private
*/
_evictOldest() {
const oldestKey = Array.from(this.keyTimestamps.entries())
.sort(([, a], [, b]) => a - b)[0]?.[0];
if (oldestKey) {
this.delete(oldestKey);
this.stats.evictions++;
}
}
/**
* Calculates the cache hit rate
* @private
* @returns {number} Hit rate between 0 and 1
*/
_calculateHitRate() {
const total = this.stats.hits + this.stats.misses;
return total === 0 ? 0 : this.stats.hits / total;
}
/**
* Creates a deterministic cache key from a value
* @static
* @param {string} type - Type prefix for the key
* @param {*} value - Value to create key from
* @returns {string} Base64-encoded cache key
*/
static createKey(type, value) {
const hash = Buffer.from(JSON.stringify(value))
.toString('base64')
.replace(/[/+=]/g, '_');
return `${type}:${hash}`;
}
}
export default CacheManager;
```
# core/chunker.js
```js
// chunker.js
import debug from 'debug';
import natural from 'natural';
const log = debug('mongodb-rag:chunker');
class Chunker {
constructor(options = {}) {
this.options = {
strategy: options.strategy || 'sliding',
maxChunkSize: options.maxChunkSize || 500,
overlap: options.overlap || 50,
splitter: options.splitter || 'sentence'
};
}
async chunkDocument(document) {
log(`Chunking document ${document.id} using ${this.options.strategy} strategy`);
switch (this.options.strategy) {
case 'sliding':
return this.slidingWindowChunk(document);
case 'semantic':
return this.semanticChunk(document);
case 'recursive':
return this.recursiveChunk(document);
default:
throw new Error(`Unknown chunking strategy: ${this.options.strategy}`);
}
}
slidingWindowChunk(document) {
const text = document.content;
const chunks = [];
const sentences = this.splitIntoSentences(text);
let currentChunk = [];
let currentLength = 0;
for (const sentence of sentences) {
const sentenceLength = sentence.length;
if (currentLength + sentenceLength > this.options.maxChunkSize && currentChunk.length > 0) {
chunks.push(this.createChunk(document, currentChunk.join(' ')));
const overlapSentences = this.calculateOverlap(currentChunk);
currentChunk = overlapSentences;
currentLength = overlapSentences.join(' ').length;
}
currentChunk.push(sentence);
currentLength += sentenceLength;
}
if (currentChunk.length > 0) {
chunks.push(this.createChunk(document, currentChunk.join(' ')));
}
log(`Created ${chunks.length} chunks for document ${document.id}`);
return chunks;
}
semanticChunk(document) {
const text = document.content;
const tokenizer = new natural.SentenceTokenizer();
const sentences = tokenizer.tokenize(text);
const chunks = [];
let currentChunk = [];
let currentLength = 0;
for (const sentence of sentences) {
const sentenceLength = sentence.length;
if (currentLength + sentenceLength > this.options.maxChunkSize && currentChunk.length > 0) {
chunks.push(this.createChunk(document, currentChunk.join(' ')));
currentChunk = [];
currentLength = 0;
}
currentChunk.push(sentence);
currentLength += sentenceLength;
}
if (currentChunk.length > 0) {
chunks.push(this.createChunk(document, currentChunk.join(' ')));
}
log(`Created ${chunks.length} semantic chunks for document ${document.id}`);
return chunks;
}
recursiveChunk(document) {
const text = document.content;
const chunks = [];
const sections = text.split(/\n\s*\n/); // Split based on paragraphs
for (const section of sections) {
if (section.length <= this.options.maxChunkSize) {
chunks.push(this.createChunk(document, section));
} else {
// If a section is too large, break it into sentences
const sentences = this.splitIntoSentences(section);
let currentChunk = [];
let currentLength = 0;
for (const sentence of sentences) {
const sentenceLength = sentence.length;
if (currentLength + sentenceLength > this.options.maxChunkSize && currentChunk.length > 0) {
chunks.push(this.createChunk(document, currentChunk.join(' ')));
currentChunk = [];
currentLength = 0;
}
currentChunk.push(sentence);
currentLength += sentenceLength;
}
if (currentChunk.length > 0) {
chunks.push(this.createChunk(document, currentChunk.join(' ')));
}
}
}
log(`Created ${chunks.length} recursive chunks for document ${document.id}`);
return chunks;
}
splitIntoSentences(text) {
return text
.replace(/([.!?])\s+/g, '$1\n')
.split('\n')
.map(s => s.trim())
.filter(s => s.length > 0);
}
calculateOverlap(sentences) {
const overlapTokenCount = Math.ceil(sentences.length * (this.options.overlap / 100));
return sentences.slice(-overlapTokenCount);
}
createChunk(document, content) {
return {
documentId: document.id,
content: content,
metadata: {
...document.metadata,
chunkIndex: Date.now(),
strategy: this.options.strategy
}
};
}
}
export default Chunker;
```
# core/IndexManager.js
```js
// src/core/IndexManager.js
import debug from 'debug';
const log = debug('mongodb-rag:index');
class IndexManager {
constructor(collection, config = {}) {
this.collection = collection;
this.options = {
indexName: config.indexName || 'vector_index',
dimensions: config.embedding?.dimensions || 1536,
similarity: config.search?.similarity || 'cosine',
embeddingPath: config.embeddingFieldPath || 'embedding',
...config
};
}
async ensureIndexes() {
try {
console.log('Checking existing indexes...');
const existingIndexes = await this.collection.listIndexes().toArray();
const hasVectorIndex = existingIndexes.some(index => index.name === this.options.indexName);
if (!hasVectorIndex) {
console.log('Creating missing vector search index...');
const indexDefinition = {
name: this.options.indexName,
type: 'vectorSearch',
definition: {
fields: [
{
type: 'vector',
path: this.options.embeddingPath,
numDimensions: this.options.dimensions,
similarity: this.options.similarity,
quantization: 'none'
}
]
}
};
await this.collection.createSearchIndex(indexDefinition);
console.log(`Vector search index '${this.options.indexName}' created successfully.`);
} else {
console.log(`Vector search index '${this.options.indexName}' already exists.`);
}
// Ensure supporting metadata indexes exist
await this.createSupportingIndexes();
console.log('All indexes verified.');
} catch (error) {
throw new Error(`Failed to ensure indexes: ${error.message}`);
}
}
async ensureIndex() {
return this.ensureIndexes();
}
async createSupportingIndexes() {
try {
await this.collection.createIndex(
{ documentId: 1 },
{ name: 'document_lookup' }
);
await this.collection.createIndex(
{ 'metadata.source': 1, 'metadata.category': 1 },
{ name: 'metadata_filter', sparse: true }
);
log('Supporting indexes created successfully');
} catch (error) {
throw new Error(`Failed to create supporting indexes: ${error.message}`);
}
}
buildSearchQuery(embedding, filter = {}, options = {}) {
const vectorSearchQuery = {
index: this.options.indexName,
path: this.options.embeddingPath,
queryVector: embedding,
numCandidates: options.numCandidates || 100,
limit: options.maxResults || this.options.search?.maxResults || 10,
exact: options.exact || false
};
if (!vectorSearchQuery.exact) {
vectorSearchQuery.numCandidates = options.numCandidates || 100;
}
const pipeline = [
{ $vectorSearch: vectorSearchQuery },
{
$project: {
documentId: 1,
content: 1,
metadata: 1,
score: { $meta: "vectorSearchScore" }
}
}
];
if (Object.keys(filter).length > 0) {
pipeline[0].$vectorSearch.filter = filter;
}
log('Generated search pipeline:', JSON.stringify(pipeline, null, 2));
return pipeline;
}
async getIndexStats() {
try {
const stats = await this.collection.stats();
const indexes = await this.collection.listIndexes().toArray();
return {
documentCount: stats.count,
indexSize: stats.totalIndexSize,
indexes: indexes.map(index => ({
name: index.name,
fields: Object.keys(index.key),
size: stats.indexSizes ? stats.indexSizes[index.name] : 'N/A'
}))
};
} catch (error) {
throw new Error(`Failed to get index stats: ${error.message}`);
}
}
}
export default IndexManager;
```
# core/MongoRAG.js
```js
// src/core/MongoRAG.js
import { MongoClient } from 'mongodb';
import debug from 'debug';
import IndexManager from './IndexManager.js';
import OpenAIEmbeddingProvider from '../providers/OpenAIEmbeddingProvider.js';
import OllamaEmbeddingProvider from '../providers/OllamaEmbeddingProvider.js';
const log = debug('mongodb-rag:core');
/**
* MongoRAG - A Retrieval-Augmented Generation (RAG) library
* for performing semantic search and vector-based retrieval using MongoDB.
*/
class MongoRAG {
constructor(config) {
// Validate required embedding configuration
const apiKey = config.embedding?.apiKey || process.env.EMBEDDING_API_KEY;
if (!apiKey && config.embedding?.provider !== 'ollama') {
console.log("API Key failing: ", apiKey);
throw new Error('Embedding API key is required unless using Ollama.');
}
// Clone config to prevent mutation
const safeConfig = { ...config };
// Ensure database and collection exist
safeConfig.database = safeConfig.database || "helpdesk"; // Provide a fallback
safeConfig.collection = safeConfig.collection || "articles"; // Provide a fallback
// Set up internal config structure
this.config = {
mongoUrl: safeConfig.mongoUrl,
database: safeConfig.database,
collection: safeConfig.collection,
indexName: safeConfig.indexName || "vector_index",
embeddingFieldPath: safeConfig.embeddingFieldPath || "embedding",
embedding: {
provider: safeConfig.embedding.provider,
apiKey: apiKey, // Use the resolved apiKey
model: safeConfig.embedding.model || 'text-embedding-3-small',
baseUrl: safeConfig.embedding.baseUrl || 'http://localhost:11434',
batchSize: safeConfig.embedding.batchSize || 100,
dimensions: safeConfig.embedding.dimensions || 1536
},
search: {
similarityMetric: safeConfig.search?.similarityMetric || 'cosine',
minScore: safeConfig.search?.minScore || 0.7,
maxResults: safeConfig.search?.maxResults || 5
}
};
console.log("ā
MongoRAG Final Config:", JSON.stringify(this.config, null, 2));
this.client = null;
this.indexManager = null;
this.provider = this._createEmbeddingProvider(this.config.embedding);
}
_createEmbeddingProvider(config) {
const { provider, apiKey, baseUrl, ...options } = config;
log(`Creating embedding provider: ${provider}`);
switch (provider) {
case 'openai':
return new OpenAIEmbeddingProvider({
apiKey,
model: options.model,
dimensions: options.dimensions
});
case 'ollama':
if (!baseUrl) {
throw new Error("Ollama base URL is missing from the config");
}
return new OllamaEmbeddingProvider({
baseUrl,
model: options.model
});
default:
throw new Error(`Unknown embedding provider: ${provider}`);
}
}
async connect() {
if (!this.client) {
try {
log('Initializing MongoDB client...');
this.client = new MongoClient(this.config.mongoUrl);
} catch (error) {
console.error('Error initializing MongoDB client:', error);
throw error;
}
}
if (!this.client.topology || !this.client.topology.isConnected()) {
try {
log('Connecting to MongoDB...');
await this.client.connect();
log('Connected to MongoDB');
} catch (error) {
console.error('MongoDB Connection Error:', error);
throw error;
}
}
}
async _getCollection(database, collection) {
await this.connect();
const dbName = database || this.config.database; // Fixed here
const colName = collection || this.config.collection; // Fixed here
console.log("š Using database:", dbName);
console.log("š Using collection:", colName);
if (!dbName || !colName) {
throw new Error('Database and collection must be specified.');
}
return this.client.db(dbName).collection(colName);
}
async ingestBatch(documents, options = {}) {
const { database, collection } = options;
const col = await this._getCollection(database, collection);
log(`Ingesting into ${database || this.config.defaultDatabase}.${collection || this.config.defaultCollection} - ${documents.length} documents`);
try {
const embeddedDocs = await this._embedDocuments(documents);
await col.insertMany(embeddedDocs);
log('Documents inserted successfully');
return { processed: documents.length, failed: 0 };
} catch (error) {
console.error('Batch Ingestion Error:', error);
return { processed: 0, failed: documents.length };
}
}
async search(query, options = {}) {
try {
console.log('[DEBUG] Starting search with query:', query);
console.log('[DEBUG] Search options:', options);
const { database, collection, maxResults = 5, skip = 0 } = options;
const col = await this._getCollection(database, collection);
const embedding = query ? await this.getEmbedding(query) : null;
const indexManager = new IndexManager(col, this.config);
// Check if the index exists without trying to create it
const existingIndexes = await col.listIndexes().toArray();
const indexName = options.indexName || this.config.indexName || 'vector_index';
const hasIndex = existingIndexes.some(index => index.name === indexName);
if (!hasIndex) {
throw new Error(`Vector search index '${indexName}' does not exist. Please create it using 'npx mongodb-rag init'.`);
}
// Construct the vector search query using the $vectorSearch operator
const aggregation = query
? [{
$vectorSearch: {
exact: false, // or true, depending on your requirements
filter: {}, // Add any filter specifications if needed
index: this.config.indexName,
limit: maxResults,
numCandidates: 100, // Adjust based on your needs
path: this.config.embeddingFieldPath,
queryVector: embedding
}
}]
: [{ $skip: skip }, { $limit: maxResults }]; // Simple aggregation for all documents
console.log('[DEBUG] Aggregation query:', JSON.stringify(aggregation, null, 2));
log(`Running vector search in ${database || this.config.defaultDatabase}.${collection || this.config.defaultCollection}`);
const results = await col.aggregate(aggregation).toArray();
console.log('[DEBUG] Search results:', results);
return results.map(r => ({
content: r.content,
documentId: r.documentId,
metadata: r.metadata,
score: r.score
}));
} catch (error) {
console.error('[DEBUG] Search error:', error);
throw error;
}
}
async _embedDocuments(documents) {
await this._initializeEmbeddingProvider();
const texts = documents.map(doc => doc.content);
const embeddings = await this.getEmbeddings(texts);
return documents.map((doc, i) => ({
...doc,
embedding: embeddings[i]
}));
}
async _initializeEmbeddingProvider() {
if (!this.provider) {
const { provider, apiKey, baseUrl, ...options } = this.config.embedding;
log(`Initializing embedding provider: ${provider}`);
switch (provider) {
case 'openai':
this.provider = new OpenAIEmbeddingProvider({ apiKey, ...options });
break;
case 'ollama':
if (!baseUrl) {
throw new Error("Ollama base URL is missing from the config. Run 'npx mongodb-rag edit-config' to set it.");
}
this.provider = new OllamaEmbeddingProvider({
provider: 'ollama',
baseUrl,
model: options.model
});
break;
default:
throw new Error(`Unknown embedding provider: ${provider}`);
}
}
}
async getEmbedding(text) {
if (!this.provider) {
throw new Error('Embedding provider not initialized');
}
return await this.provider.getEmbedding(text);
}
async getEmbeddings(texts) {
if (!this.provider) {
throw new Error('Embedding provider not initialized');
}
return await this.provider.getEmbeddings(texts);
}
async close() {
if (this.client) {
await this.client.close();
log('MongoDB connection closed');
}
}
async listDocuments({ limit = 10, skip = 0, database, collection } = {}) {
try {
// Use an empty query or a special query to fetch all documents
const query = ""; // or use a wildcard query if supported
const options = { database, collection, maxResults: limit, skip };
// Call the search method with the query
const results = await this.search(query, options);
// Return the results, applying skip manually if needed
return results.slice(skip, skip + limit);
} catch (error) {
console.error('Error listing documents:', error);
throw error;
}
}
getClient() {
return this.client;
}
}
export default MongoRAG;
```
# index.js
```js
import MongoRAG from './core/MongoRAG.js';
import BatchProcessor from './core/BatchProcessor.js';
import CacheManager from './core/CacheManager.js';
import IndexManager from './core/IndexManager.js';
import Chunker from './core/chunker.js';
export { MongoRAG as default };
export { MongoRAG, BatchProcessor, CacheManager, Chunker };
```
# playground-ui/.gitignore
```
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
# dependencies
/node_modules
/.pnp
.pnp.js
# testing
/coverage
# production
/build
# misc
.DS_Store
.env.local
.env.development.local
.env.test.local
.env.production.local
npm-debug.log*
yarn-debug.log*
yarn-error.log*
```
# playground-ui/config-overrides.js
```js
const { override, addWebpackAlias } = require('customize-cra');
const path = require('path');
module.exports = override(
addWebpackAlias({
'path': 'path-browserify'
})
);
```
# playground-ui/package.json
```json
{
"name": "playground-ui",
"version": "0.1.0",
"private": true,
"resolutions": {
"path": "path-browserify"
},
"browser": {
"path": "path-browserify"
},
"dependencies": {
"@emotion/react": "^11.14.0",
"@emotion/styled": "^11.14.0",
"@mui/icons-material": "^6.4.4",
"@mui/material": "^6.4.4",
"@mui/x-data-grid": "^7.27.0",
"@testing-library/dom": "^10.4.0",
"@testing-library/jest-dom": "^6.6.3",
"@testing-library/react": "^16.2.0",
"@testing-library/user-event": "^13.5.0",
"customize-cra": "^1.0.0",
"detect-port": "^2.1.0",
"path-browserify": "^1.0.1",
"pdf-parse": "^1.1.1",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-scripts": "5.0.1",
"socket.io-client": "^4.8.1",
"web-vitals": "^2.1.4"
},
"scripts": {
"start": "PORT=3000 react-app-rewired start",
"build": "react-app-rewired build",
"test": "react-app-rewired test",
"eject": "react-app-rewired eject"
},
"eslintConfig": {
"extends": [
"react-app",
"react-app/jest"
]
},
"browserslist": {
"production": [
">0.2%",
"not dead",
"not op_mini all"
],
"development": [
"last 1 chrome version",
"last 1 firefox version",
"last 1 safari version"
]
},
"devDependencies": {
"react-app-rewired": "^2.2.1"
}
}
```
# playground-ui/public/favicon.ico
This is a binary file of the type: Binary
# playground-ui/public/index.html
```html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="theme-color" content="#000000" />
<meta
name="description"
content="Web site created using create-react-app"
/>
<link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
<!--
manifest.json provides metadata used when your web app is installed on a
user's mobile device or desktop. See https://developers.google.com/web/fundamentals/web-app-manifest/
-->
<link rel="manifest" href="%PUBLIC_URL%/manifest.json" />
<!--
Notice the use of %PUBLIC_URL% in the tags above.
It will be replaced with the URL of the `public` folder during the build.
Only files inside the `public` folder can be referenced from the HTML.
Unlike "/favicon.ico" or "favicon.ico", "%PUBLIC_URL%/favicon.ico" will
work correctly both with client-side routing and a non-root public URL.
Learn how to configure a non-root public URL by running `npm run build`.
-->
<title>React App</title>
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div>
<!--
This HTML file is a template.
If you open it directly in the browser, you will see an empty page.
You can add webfonts, meta tags, or analytics to this file.
The build step will place the bundled scripts into the <body> tag.
To begin the development, run `npm start` or `yarn start`.
To create a production bundle, use `npm run build` or `yarn build`.
-->
</body>
</html>
```
# playground-ui/public/logo.svg
This is a file of the type: SVG Image
# playground-ui/public/logo192.png
This is a binary file of the type: Image
# playground-ui/public/logo512.png
This is a binary file of the type: Image
# playground-ui/public/manifest.json
```json
{
"short_name": "React App",
"name": "Create React App Sample",
"icons": [
{
"src": "favicon.ico",
"sizes": "64x64 32x32 24x24 16x16",
"type": "image/x-icon"
},
{
"src": "logo192.png",
"type": "image/png",
"sizes": "192x192"
},
{
"src": "logo512.png",
"type": "image/png",
"sizes": "512x512"
}
],
"start_url": ".",
"display": "standalone",
"theme_color": "#000000",
"background_color": "#ffffff"
}
```
# playground-ui/public/owl.png
This is a binary file of the type: Image
# playground-ui/public/robots.txt
```txt
# https://www.robotstxt.org/robotstxt.html
User-agent: *
Disallow:
```
# playground-ui/README.md
```md
# Getting Started with Create React App
This project was bootstrapped with [Create React App](https://github.com/facebook/create-react-app).
## Available Scripts
In the project directory, you can run:
### `npm start`
Runs the app in the development mode.\
Open [http://localhost:3000](http://localhost:3000) to view it in your browser.
The page will reload when you make changes.\
You may also see any lint errors in the console.
### `npm test`
Launches the test runner in the interactive watch mode.\
See the section about [running tests](https://facebook.github.io/create-react-app/docs/running-tests) for more information.
### `npm run build`
Builds the app for production to the `build` folder.\
It correctly bundles React in production mode and optimizes the build for the best performance.
The build is minified and the filenames include the hashes.\
Your app is ready to be deployed!
See the section about [deployment](https://facebook.github.io/create-react-app/docs/deployment) for more information.
### `npm run eject`
**Note: this is a one-way operation. Once you `eject`, you can't go back!**
If you aren't satisfied with the build tool and configuration choices, you can `eject` at any time. This command will remove the single build dependency from your project.
Instead, it will copy all the configuration files and the transitive dependencies (webpack, Babel, ESLint, etc) right into your project so you have full control over them. All of the commands except `eject` will still work, but they will point to the copied scripts so you can tweak them. At this point you're on your own.
You don't have to ever use `eject`. The curated feature set is suitable for small and middle deployments, and you shouldn't feel obligated to use this feature. However we understand that this tool wouldn't be useful if you couldn't customize it when you are ready for it.
## Learn More
You can learn more in the [Create React App documentation](https://facebook.github.io/create-react-app/docs/getting-started).
To learn React, check out the [React documentation](https://reactjs.org/).
### Code Splitting
This section has moved here: [https://facebook.github.io/create-react-app/docs/code-splitting](https://facebook.github.io/create-react-app/docs/code-splitting)
### Analyzing the Bundle Size
This section has moved here: [https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size](https://facebook.github.io/create-react-app/docs/analyzing-the-bundle-size)
### Making a Progressive Web App
This section has moved here: [https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app](https://facebook.github.io/create-react-app/docs/making-a-progressive-web-app)
### Advanced Configuration
This section has moved here: [https://facebook.github.io/create-react-app/docs/advanced-configuration](https://facebook.github.io/create-react-app/docs/advanced-configuration)
### Deployment
This section has moved here: [https://facebook.github.io/create-react-app/docs/deployment](https://facebook.github.io/create-react-app/docs/deployment)
### `npm run build` fails to minify
This section has moved here: [https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify](https://facebook.github.io/create-react-app/docs/troubleshooting#npm-run-build-fails-to-minify)
```
# playground-ui/src/App.css
```css
.App {
text-align: center;
}
.App-logo {
height: 40vmin;
pointer-events: none;
}
@media (prefers-reduced-motion: no-preference) {
.App-logo {
animation: App-logo-spin infinite 20s linear;
}
}
.App-header {
background-color: #282c34;
min-height: 100vh;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
font-size: calc(10px + 2vmin);
color: white;
}
.App-link {
color: #61dafb;
}
@keyframes App-logo-spin {
from {
transform: rotate(0deg);
}
to {
transform: rotate(360deg);
}
}
```
# playground-ui/src/App.js
```js
// src/playground-ui/App.js
import React, { useState, useEffect } from 'react';
import io from 'socket.io-client';
import { Container, Typography, Button, TextField, CircularProgress, Snackbar, Alert, Box, Tabs, Tab, Dialog, DialogTitle, DialogContent, DialogActions, Card, CardContent, Accordion, AccordionSummary, AccordionDetails, Grid, ThemeProvider, createTheme, Paper, Divider, IconButton, Tooltip, AppBar, Toolbar, MenuItem } from '@mui/material';
import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
import { CloudUpload as UploadIcon, Search as SearchIcon, Settings as SettingsIcon, GitHub as GitHubIcon, Refresh as RefreshIcon, Download as DownloadIcon, Storage as StorageIcon, Add as AddIcon } from '@mui/icons-material';
import './App.css';
import logo from './owl.png';
import { DataGrid } from '@mui/x-data-grid';
import ConfigurationBanner from "./components/ConfigurationBanner";
const socket = io('http://localhost:4000');
const BACKEND_URL = process.env.REACT_APP_BACKEND_URL || 'http://localhost:4000';
const theme = createTheme({
palette: {
primary: {
main: '#12753c', // MongoDB green
},
secondary: {
main: '#001E2B', // MongoDB dark
},
background: {
default: '#F9FBFA',
paper: '#FFFFFF',
}
},
shape: {
borderRadius: 8,
},
components: {
MuiButton: {
styleOverrides: {
root: {
textTransform: 'none',
fontWeight: 600,
},
},
},
MuiTab: {
styleOverrides: {
root: {
textTransform: 'none',
fontWeight: 600,
},
},
},
},
});
function App() {
const [query, setQuery] = useState('');
const [results, setResults] = useState([]);
const [uploading, setUploading] = useState(false);
const [searching, setSearching] = useState(false);
const [status, setStatus] = useState('');
const [openWelcome, setOpenWelcome] = useState(!document.cookie.includes("hideWelcomePopup=true"));
const [openSnackbar, setOpenSnackbar] = useState(false);
const [config, setConfig] = useState({
mongoUrl: '',
database: '',
collection: '',
provider: '',
apiKey: '',
model: '',
dimensions: 1536,
batchSize: 100,
maxResults: 5,
minScore: 0.7,
indexName: 'vector_index'
});
const [tabValue, setTabValue] = useState(0);
const [formConfig, setFormConfig] = useState(config);
const [documents, setDocuments] = useState([]);
const [loadingDocuments, setLoadingDocuments] = useState(false);
const [indexes, setIndexes] = useState({ searchIndexes: [], regularIndexes: [] });
const [loadingIndexes, setLoadingIndexes] = useState(false);
const [createIndexDialog, setCreateIndexDialog] = useState(false);
const [newIndexConfig, setNewIndexConfig] = useState({
name: '',
dimensions: 1536,
path: 'embedding',
similarity: 'cosine'
});
useEffect(() => {
socket.on('update', (data) => {
setStatus(data.message);
setOpenSnackbar(true);
});
const loadConfig = async () => {
try {
const response = await fetch(`${BACKEND_URL}/api/config`);
const data = await response.json();
console.log("Config data:", data);
setConfig(data);
setFormConfig(data);
} catch (error) {
console.error('Error loading config:', error);
setStatus('Failed to load configuration');
setOpenSnackbar(true);
}
};
const loadInitialDocuments = async () => {
try {
const response = await fetch(`${BACKEND_URL}/api/documents`);
const data = await response.json();
if (data.documents) {
setDocuments(data.documents);
}
} catch (error) {
console.error('Error loading documents:', error);
}
};
loadConfig();
loadInitialDocuments();
return () => {
socket.off('update');
};
}, []);
useEffect(() => {
console.log("Form config:", formConfig);
}, [formConfig]);
const reloadConfig = async () => {
const response = await fetch(`${BACKEND_URL}/api/config`);
console.log("Reloading config:", response);
const data = await response.json();
console.log("Reloaded config:", data);
setConfig(data);
setFormConfig(data);
setStatus('Configuration reloaded');
setOpenSnackbar(true);
};
const handleTabChange = (event, newValue) => {
setTabValue(newValue);
};
const handleUpload = async (event) => {
const file = event.target.files[0];
if (!file) return;
setUploading(true);
const formData = new FormData();
formData.append('file', file);
try {
const response = await fetch(`${BACKEND_URL}/api/ingest`, {
method: 'POST',
body: formData,
});
const result = await response.json();
setStatus(result.message || 'Upload successful!');
setOpenSnackbar(true);
} catch (error) {
console.error('Error uploading file:', error);
}
setUploading(false);
};
const handleSearch = async () => {
if (!query.trim()) return;
setSearching(true);
try {
console.log("Searching for:", query);
const response = await fetch(`${BACKEND_URL}/api/search`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ query })
});
if (!response.ok) {
throw new Error(`Search failed: ${response.statusText}`);
}
const data = await response.json();
console.log('Search results:', data);
setResults(data.results || []);
} catch (error) {
console.error('Search error:', error);
setStatus(`Search failed: ${error.message}`);
setOpenSnackbar(true);
setResults([]);
} finally {
setSearching(false);
}
};
const handleConfigSubmit = async () => {
try {
await fetch(`${BACKEND_URL}/api/save-config`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(formConfig)
});
await reloadConfig();
setStatus('Configuration saved successfully!');
setOpenSnackbar(true);
} catch (error) {
console.error('Error saving config:', error);
setStatus('Failed to save configuration');
setOpenSnackbar(true);
}
};
const handleDownload = async (type) => {
try {
const response = await fetch(`${BACKEND_URL}/api/download/${type}`);
if (!response.ok) {
throw new Error(`Failed to download ${type} file`);
}
const blob = await response.blob();
const url = window.URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = type === 'config' ? '.mongodb-rag.json' : '.env';
document.body.appendChild(a);
a.click();
window.URL.revokeObjectURL(url);
document.body.removeChild(a);
setStatus(`Successfully downloaded ${type === 'config' ? '.mongodb-rag.json' : '.env'} file`);
setOpenSnackbar(true);
} catch (error) {
console.error('Download error:', error);
setStatus(`Failed to download ${type} file`);
setOpenSnackbar(true);
}
};
const formatDocumentField = (value) => {
if (value === undefined) {
return ''; // Return an empty string or a default message if value is undefined
}
if (Array.isArray(value)) {
return value.slice(0, 10).map(item =>
typeof item === 'object' ? JSON.stringify(item) : item
).join(', ') + (value.length > 10 ? '...' : '');
}
if (typeof value === 'object' && value !== null) {
return Object.keys(value).length > 0 ? '{...}' : '{}';
}
return String(value);
};
const DocumentCard = ({ document }) => {
const excludeFields = ['embedding'];
const fields = Object.entries(document)
.filter(([key]) => !excludeFields.includes(key));
return (
<Card sx={{ mb: 2, width: '100%' }}>
<CardContent>
<Accordion>
<AccordionSummary expandIcon={<ExpandMoreIcon />}>
<Typography>
{document.title || document.content?.slice(0, 100) || 'Document'}...
</Typography>
</AccordionSummary>
<AccordionDetails>
<Grid container spacing={1}>
{fields.map(([key, value]) => (
<Grid item xs={12} key={key}>
<Typography variant="body2" color="textSecondary" component="div">
<strong>{key}:</strong> {formatDocumentField(value)}
</Typography>
</Grid>
))}
</Grid>
</AccordionDetails>
</Accordion>
</CardContent>
</Card>
);
};
const loadDocuments = async () => {
setLoadingDocuments(true);
try {
const response = await fetch(`${BACKEND_URL}/api/documents`);
const data = await response.json();
if (data.documents) {
setDocuments(data.documents);
}
} catch (error) {
console.error('Error loading documents:', error);
setStatus('Failed to load documents');
setOpenSnackbar(true);
}
setLoadingDocuments(false);
};
const loadIndexes = async () => {
setLoadingIndexes(true);
try {
const response = await fetch(`${BACKEND_URL}/api/indexes`);
if (!response.ok) {
throw new Error(`Failed to fetch indexes: ${response.statusText}`);
}
const data = await response.json();
setIndexes({
searchIndexes: Array.isArray(data?.searchIndexes) ? data.searchIndexes : [],
regularIndexes: Array.isArray(data?.regularIndexes) ? data.regularIndexes : []
});
} catch (error) {
console.error('Error loading indexes:', error);
setStatus('Failed to load indexes');
setOpenSnackbar(true);
setIndexes({ searchIndexes: [], regularIndexes: [] }); // Ensure empty state
}
setLoadingIndexes(false);
};
const handleCreateIndex = async () => {
try {
const response = await fetch(`${BACKEND_URL}/api/indexes/create`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(newIndexConfig)
});
if (!response.ok) {
throw new Error(`Failed to create index: ${response.statusText}`);
}
const result = await response.json();
setStatus('Vector search index created successfully!');
setOpenSnackbar(true);
setCreateIndexDialog(false);
loadIndexes(); // Refresh the index list
} catch (error) {
console.error('Error creating index:', error);
setStatus(`Failed to create index: ${error.message}`);
setOpenSnackbar(true);
}
};
const IndexesView = () => {
const searchIndexColumns = [
{ field: 'name', headerName: 'Name', flex: 1 },
{ field: 'type', headerName: 'Type', width: 130 },
{ field: 'status', headerName: 'Status', width: 130 },
];
const regularIndexColumns = [
{ field: 'name', headerName: 'Name1', flex: 1 },
{
field: 'key',
headerName: 'Key',
flex: 1,
valueGetter: (params) => {
if (!params || !params.row) return "{}"; // Ensures params is not undefined
return params.row.key ? JSON.stringify(params.row.key) : "{}";
}
},
{
field: 'unique',
headerName: 'Unique',
width: 130,
valueGetter: (params) => {
if (!params || !params.row) return "No"; // Prevents accessing undefined
return params.row.unique ? "Yes" : "No";
}
},
];
// Ensure indexes are always arrays to prevent map errors
const searchIndexRows = Array.isArray(indexes.searchIndexes) ? indexes.searchIndexes.map((idx, i) => ({
id: i,
name: idx.name,
type: idx.type || 'vector',
status: idx.status || 'ready'
})) : [];
const regularIndexRows = Array.isArray(indexes.regularIndexes) ? indexes.regularIndexes.map((idx, i) => ({
id: i,
name: idx.name,
key: idx.key,
unique: idx.unique || false
})) : [];
return (
<div>
<Box sx={{ mt: 4 }}>
<Paper variant="outlined" sx={{ p: 3, mb: 3 }}>
<Box sx={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', mb: 2 }}>
<Typography variant="h6">
Indexes for {indexes.database}/{indexes.collection}
</Typography>
<Box sx={{ display: 'flex', gap: 2 }}>
<Button
variant="contained"
color="primary"
startIcon={<AddIcon />}
onClick={() => setCreateIndexDialog(true)}
>
Create Vector Index
</Button>
<Button
variant="outlined"
startIcon={<RefreshIcon />}
onClick={loadIndexes}
disabled={loadingIndexes}
>
Refresh
</Button>
</Box>
</Box>
<Typography variant="subtitle1" sx={{ mt: 3, mb: 2 }}>
Vector Search Indexes
</Typography>
<DataGrid
rows={searchIndexRows}
columns={searchIndexColumns}
autoHeight
hideFooter={searchIndexRows.length <= 10}
sx={{ mb: 4 }}
/>
<Typography variant="subtitle1" sx={{ mt: 3, mb: 2 }}>
Regular Indexes
</Typography>
<DataGrid
rows={regularIndexRows}
columns={regularIndexColumns}
autoHeight
hideFooter={regularIndexRows.length <= 10}
/>
</Paper>
<Dialog
open={createIndexDialog}
onClose={() => setCreateIndexDialog(false)}
maxWidth="sm"
fullWidth
>
<DialogTitle>Create Vector Search Index</DialogTitle>
<DialogContent>
<Box sx={{ mt: 2 }}>
<TextField
fullWidth
label="Index Name"
value={newIndexConfig.name}
onChange={(e) => setNewIndexConfig({ ...newIndexConfig, name: e.target.value })}
sx={{ mb: 2 }}
helperText="Name for the new vector search index"
/>
<TextField
fullWidth
label="Dimensions"
type="number"
value={newIndexConfig.dimensions}
onChange={(e) => setNewIndexConfig({ ...newIndexConfig, dimensions: parseInt(e.target.value) })}
sx={{ mb: 2 }}
helperText="Number of dimensions for the vector field"
/>
<TextField
fullWidth
label="Field Path"
value={newIndexConfig.path}
onChange={(e) => setNewIndexConfig({ ...newIndexConfig, path: e.target.value })}
sx={{ mb: 2 }}
helperText="Path to the vector field in your documents"
/>
<TextField
fullWidth
select
label="Similarity Metric"
value={newIndexConfig.similarity}
onChange={(e) => setNewIndexConfig({ ...newIndexConfig, similarity: e.target.value })}
helperText="Similarity metric for vector comparison"
>
<MenuItem value="cosine">Cosine Similarity</MenuItem>
<MenuItem value="dotProduct">Dot Product</MenuItem>
<MenuItem value="euclidean">Euclidean Distance</MenuItem>
</TextField>
</Box>
</DialogContent>
<DialogActions>
<Button onClick={() => setCreateIndexDialog(false)}>Cancel</Button>
<Button
onClick={handleCreateIndex}
variant="contained"
color="primary"
disabled={!newIndexConfig.name || !newIndexConfig.dimensions}
>
Create Index
</Button>
</DialogActions>
</Dialog>
</Box>
</div>
);
};
return (
<ThemeProvider theme={theme}>
<Box sx={{
minHeight: '90vh',
bgcolor: 'background.default',
pb: 4
}}>
<AppBar position="static" color="secondary" elevation={0}>
<Toolbar>
<Box display="flex" alignItems="center" sx={{ width: '100%', maxWidth: 1200, mx: 'auto' }}>
<img src={logo} alt="Playground Logo" style={{ height: '40px', marginRight: '15px' }} />
<Box flex={1}>
<Typography variant="h6" sx={{ fontWeight: 600 }}>
MongoDB-RAG Playground
</Typography>
<Box sx={{ display: 'flex', gap: 2, mt: 0.5 }}>
<a
href="https://npmjs.com/package/mongodb-rag"
target="_blank"
rel="noopener noreferrer"
style={{ textDecoration: 'none' }}
>
<img
src="https://img.shields.io/npm/v/mongodb-rag?color=blue&label=npm&logo=npm"
alt="NPM Version"
/>
</a>
<a
href="https://github.com/mongodb-developer/mongodb-rag"
target="_blank"
rel="noopener noreferrer"
style={{ textDecoration: 'none' }}
>
<img
src="https://img.shields.io/github/stars/mongodb-developer/mongodb-rag?style=social"
alt="GitHub Stars"
/>
</a>
<a
href="https://github.com/mongodb-developer/mongodb-rag"
target="_blank"
rel="noopener noreferrer"
style={{ textDecoration: 'none' }}
>
<img
src="https://img.shields.io/github/license/mongodb-developer/mongodb-rag?color=blue"
alt="License"
/>
</a>
</Box>
</Box>
<Tooltip title="View on GitHub">
<IconButton
color="inherit"
href="https://github.com/mongodb-developer/mongodb-rag"
target="_blank"
rel="noopener noreferrer"
>
<GitHubIcon />
</IconButton>
</Tooltip>
</Box>
</Toolbar>
</AppBar>
<Container maxWidth="md" sx={{ mt: 4 }}>
<ConfigurationBanner />
<Paper elevation={0} sx={{ p: 3, mb: 3 }}>
<Tabs
value={tabValue}
onChange={handleTabChange}
centered
sx={{
borderBottom: 1,
borderColor: 'divider',
'& .MuiTab-root': {
minWidth: 120,
px: 3
}
}}
>
<Tab icon={<SearchIcon />} iconPosition="start" label="Documents" />
<Tab icon={<UploadIcon />} iconPosition="start" label="Upload" />
<Tab icon={<StorageIcon />} iconPosition="start" label="Indexes" />
<Tab icon={<SettingsIcon />} iconPosition="start" label="Configure" />
</Tabs>
{tabValue === 1 && (
<Box sx={{ mt: 4, textAlign: 'center' }}>
<input
type="file"
accept=".json, .md, .pdf"
onChange={handleUpload}
style={{ display: 'none' }}
id="file-upload"
/>
<label htmlFor="file-upload">
<Button
variant="outlined"
component="span"
startIcon={<UploadIcon />}
size="large"
disabled={uploading}
>
{uploading ? 'Uploading...' : 'Choose File'}
</Button>
</label>
</Box>
)}
{tabValue === 0 && (
<Box sx={{ mt: 4 }}>
<Paper variant="outlined" sx={{ p: 3, mb: 3 }}>
<TextField
fullWidth
label="Search documents"
placeholder="Enter your search query..."
variant="outlined"
value={query}
onChange={(e) => setQuery(e.target.value)}
onKeyPress={(e) => e.key === 'Enter' && handleSearch()}
InputProps={{
endAdornment: (
<Box sx={{ display: 'flex', gap: 1 }}>
<Tooltip title="Search">
<IconButton
onClick={handleSearch}
disabled={searching}
color="primary"
>
<SearchIcon />
</IconButton>
</Tooltip>
<Divider orientation="vertical" flexItem />
<Tooltip title="Load All Documents">
<IconButton
onClick={loadDocuments}
disabled={loadingDocuments}
>
<RefreshIcon />
</IconButton>
</Tooltip>
</Box>
),
}}
/>
</Paper>
{(results.length > 0 || documents.length > 0) && (
<Box>
<Typography variant="h6" sx={{ mb: 2 }}>
{query ? `Found ${results.length} matching documents` : `Showing ${documents.length} documents`}
</Typography>
<Box sx={{ maxHeight: '600px', overflow: 'auto' }}>
{(query ? results : documents).map((doc, index) => (
<DocumentCard key={index} document={doc} />
))}
</Box>
</Box>
)}
{query && results.length === 0 && !searching && (
<Typography variant="body1" color="textSecondary" sx={{ mt: 2 }}>
No documents found matching your query
</Typography>
)}
{!query && documents.length === 0 && !loadingDocuments && (
<Typography variant="body1" color="textSecondary" sx={{ mt: 2 }}>
No documents found. Try uploading some documents first.
</Typography>
)}
</Box>
)}
{tabValue === 2 && <IndexesView />}
{tabValue === 3 && (
<Box sx={{ mt: 4 }}>
<Paper variant="outlined" sx={{ p: 3, mb: 3 }}>
<Box sx={{ display: 'flex', gap: 2, mb: 3 }}>
<Button
variant="outlined"
startIcon={<DownloadIcon />}
onClick={() => handleDownload('config')}
>
Download Config
</Button>
<Button
variant="outlined"
startIcon={<DownloadIcon />}
onClick={() => handleDownload('env')}
>
Download .env
</Button>
<Button
variant="outlined"
startIcon={<RefreshIcon />}
onClick={reloadConfig}
>
Refresh
</Button>
</Box>
<Grid container spacing={3}>
<Grid item xs={12} md={6}>
<TextField fullWidth label="MongoDB URI" variant="outlined" value={formConfig.mongoUrl ? '********' : ''} onChange={(e) => setFormConfig({ ...formConfig, mongoUrl: e.target.value })} sx={{ mt: 2 }} />
</Grid>
<Grid item xs={12} md={6}>
<TextField fullWidth label="Database" variant="outlined" value={formConfig.database} onChange={(e) => setFormConfig({ ...formConfig, database: e.target.value })} sx={{ mt: 2 }} />
</Grid>
<Grid item xs={12} md={6}>
<TextField fullWidth label="Collection" variant="outlined" value={formConfig.collection} onChange={(e) => setFormConfig({ ...formConfig, collection: e.target.value })} sx={{ mt: 2 }} />
</Grid>
<Grid item xs={12} md={6}>
<TextField fullWidth label="Provider" variant="outlined" value={formConfig.provider} onChange={(e) => setFormConfig({ ...formConfig, provider: e.target.value })} sx={{ mt: 2 }} />
</Grid>
<Grid item xs={12} md={6}>
<TextField fullWidth label="API Key" variant="outlined" value={formConfig.apiKey} onChange={(e) => setFormConfig({ ...formConfig, apiKey: e.target.value })} sx={{ mt: 2 }} />
</Grid>
<Grid item xs={12} md={6}>
<TextField fullWidth label="Model" variant="outlined" value={formConfig.model} onChange={(e) => setFormConfig({ ...formConfig, model: e.target.value })} sx={{ mt: 2 }} />
</Grid>
<Grid item xs={12} md={6}>
<TextField fullWidth label="Dimensions" variant="outlined" type="number" value={formConfig.dimensions} onChange={(e) => setFormConfig({ ...formConfig, dimensions: e.target.value })} sx={{ mt: 2 }} />
</Grid>
<Grid item xs={12} md={6}>
<TextField fullWidth label="Batch Size" variant="outlined" type="number" value={formConfig.batchSize} onChange={(e) => setFormConfig({ ...formConfig, batchSize: e.target.value })} sx={{ mt: 2 }} />
</Grid>
<Grid item xs={12} md={6}>
<TextField fullWidth label="Max Results" variant="outlined" type="number" value={formConfig.maxResults} onChange={(e) => setFormConfig({ ...formConfig, maxResults: e.target.value })} sx={{ mt: 2 }} />
</Grid>
<Grid item xs={12} md={6}>
<TextField fullWidth label="Min Score" variant="outlined" type="number" step="0.1" value={formConfig.minScore} onChange={(e) => setFormConfig({ ...formConfig, minScore: e.target.value })} sx={{ mt: 2 }} />
</Grid>
<Grid item xs={12} md={6}>
<TextField fullWidth label="Index Name" variant="outlined" value={formConfig.indexName} onChange={(e) => setFormConfig({ ...formConfig, indexName: e.target.value })} sx={{ mt: 2 }} />
</Grid>
</Grid>
<Button variant="contained" color="primary" onClick={handleConfigSubmit} sx={{ mt: 2 }}>Save Configuration</Button>
</Paper>
</Box>
)}
</Paper>
</Container>
</Box>
<Dialog open={openWelcome} onClose={() => setOpenWelcome(false)}>
<DialogTitle>Welcome to MongoDB-RAG Playground</DialogTitle>
<DialogContent>
<Typography variant="body1" sx={{ mb: 2 }}>
This playground allows you to test MongoDB Atlas Vector Search by uploading documents, running searches, and managing your settings.
</Typography>
<Typography variant="body2">
<strong>Current Settings:</strong>
</Typography>
<Typography variant="body2">
MongoDB URI: {config.mongoUrl ? '********' : 'Not Set'}
</Typography>
<Typography variant="body2">
Database: {config.database || 'Not Set'}
</Typography>
<Typography variant="body2">
Collection: {config.collection || 'Not Set'}
</Typography>
<Typography variant="body2">
Provider: {config.provider || 'Not Set'}
</Typography>
<Typography variant="body2">
Model: {config.model || 'Not Set'}
</Typography>
<Typography variant="body2">
Dimensions: {config.dimensions || 1536}
</Typography>
<Typography variant="body2">
Batch Size: {config.batchSize || 100}
</Typography>
<Typography variant="body2">
Max Results: {config.maxResults || 5}
</Typography>
<Typography variant="body2">
Min Score: {config.minScore || 0.7}
</Typography>
<Typography variant="body2">
Index Name: {config.indexName || 'Not Set'}
</Typography>
<Typography variant="body1" sx={{ mt: 2 }}>
You can update these settings in the Configuration tab.
</Typography>
<Box sx={{ display: 'flex', alignItems: 'center', mt: 2 }}>
<input type="checkbox" id="dontShowAgain" onChange={(e) => {
if (e.target.checked) {
document.cookie = "hideWelcomePopup=true; path=/; max-age=" + 60 * 60 * 24 * 30; // 30 days
}
}} />
<label htmlFor="dontShowAgain" style={{ marginLeft: "8px" }}>Don't show this again</label>
</Box>
</DialogContent>
<DialogActions>
<Button onClick={() => setOpenWelcome(false)} variant="contained" color="primary">Got it</Button>
</DialogActions>
</Dialog>
<Snackbar open={openSnackbar} autoHideDuration={3000} onClose={() => setOpenSnackbar(false)}>
<Alert onClose={() => setOpenSnackbar(false)} severity="info">
{status}
</Alert>
</Snackbar>
</ThemeProvider>
);
}
export default App;
```
# playground-ui/src/App.test.js
```js
import { render, screen } from '@testing-library/react';
import App from './App';
test('renders learn react link', () => {
render(<App />);
const linkElement = screen.getByText(/learn react/i);
expect(linkElement).toBeInTheDocument();
});
```
# playground-ui/src/components/ConfigurationBanner.jsx
```jsx
// src/components/ConfigurationBanner.jsx
import React, { useState, useEffect } from "react";
const ConfigurationBanner = () => {
const [config, setConfig] = useState(null);
const [editing, setEditing] = useState(false);
const [updatedConfig, setUpdatedConfig] = useState({});
useEffect(() => {
fetch("http://localhost:4000/api/config")
.then((res) => res.json())
.then((data) => setConfig(data))
.catch((err) => console.error("Error fetching config:", err));
}, []);
const handleEditClick = () => {
setEditing(true);
setUpdatedConfig(config);
};
const handleSave = () => {
fetch("http://localhost:4000/api/config", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(updatedConfig),
})
.then((res) => res.json())
.then((data) => {
setConfig(data);
setEditing(false);
})
.catch((err) => console.error("Error saving config:", err));
};
const handleCancel = () => {
setEditing(false); // Exit edit mode without saving
};
if (!config) return <p>Loading config...</p>;
return (
<div style={styles.banner}>
{editing ? (
<div style={styles.formContainer}>
<label style={styles.label}>Database:</label>
<input
style={styles.input}
value={updatedConfig.database}
onChange={(e) => setUpdatedConfig({ ...updatedConfig, database: e.target.value })}
/>
<label style={styles.label}>Collection:</label>
<input
style={styles.input}
value={updatedConfig.collection}
onChange={(e) => setUpdatedConfig({ ...updatedConfig, collection: e.target.value })}
/>
<label style={styles.label}>Embedding Field:</label>
<input
style={styles.input}
value={updatedConfig.embeddingFieldPath || "embedding"}
onChange={(e) => setUpdatedConfig({ ...updatedConfig, embeddingFieldPath: e.target.value })}
/>
<label style={styles.label}>Index Name:</label>
<input
style={styles.input}
value={updatedConfig.indexName}
onChange={(e) => setUpdatedConfig({ ...updatedConfig, indexName: e.target.value })}
/>
<div style={styles.buttonGroup}>
<button style={styles.saveButton} onClick={handleSave}>Save</button>
<button style={styles.cancelButton} onClick={handleCancel}>Cancel</button>
</div>
</div>
) : (
<div style={styles.displayContainer}>
<span><strong>Database:</strong> {config.database}</span>
<span><strong>Collection:</strong> {config.collection}</span>
<span><strong>Embedding Field:</strong> {config.embeddingFieldPath || "embedding"}</span>
<span><strong>Index Name:</strong> {config.indexName}</span>
<button style={styles.editButton} onClick={handleEditClick}>Edit</button>
</div>
)}
</div>
);
};
const styles = {
banner: {
background: "#f4f4f4",
padding: "12px 20px",
borderBottom: "2px solid #ccc",
display: "flex",
justifyContent: "space-between",
alignItems: "center",
fontFamily: "Arial, sans-serif",
},
displayContainer: {
display: "flex",
gap: "15px",
alignItems: "center",
},
formContainer: {
display: "flex",
flexWrap: "wrap",
gap: "10px",
alignItems: "center",
},
label: {
fontWeight: "bold",
marginRight: "5px",
},
input: {
padding: "5px",
border: "1px solid #ccc",
borderRadius: "4px",
minWidth: "150px",
},
buttonGroup: {
display: "flex",
gap: "10px",
marginLeft: "10px",
},
editButton: {
background: "#007bff",
color: "white",
padding: "5px 10px",
border: "none",
borderRadius: "4px",
cursor: "pointer",
},
saveButton: {
background: "#28a745",
color: "white",
padding: "5px 10px",
border: "none",
borderRadius: "4px",
cursor: "pointer",
},
cancelButton: {
background: "#dc3545",
color: "white",
padding: "5px 10px",
border: "none",
borderRadius: "4px",
cursor: "pointer",
},
};
export default ConfigurationBanner;
```
# playground-ui/src/index.css
```css
body {
margin: 0;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
sans-serif;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
}
code {
font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
monospace;
}
```
# playground-ui/src/index.js
```js
import React from 'react';
import ReactDOM from 'react-dom/client';
import './index.css';
import App from './App';
import reportWebVitals from './reportWebVitals';
const root = ReactDOM.createRoot(document.getElementById('root'));
root.render(
<React.StrictMode>
<App />
</React.StrictMode>
);
// If you want to start measuring performance in your app, pass a function
// to log results (for example: reportWebVitals(console.log))
// or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals
reportWebVitals();
```
# playground-ui/src/logo.svg
This is a file of the type: SVG Image
# playground-ui/src/owl.png
This is a binary file of the type: Image
# playground-ui/src/reportWebVitals.js
```js
const reportWebVitals = onPerfEntry => {
if (onPerfEntry && onPerfEntry instanceof Function) {
import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => {
getCLS(onPerfEntry);
getFID(onPerfEntry);
getFCP(onPerfEntry);
getLCP(onPerfEntry);
getTTFB(onPerfEntry);
});
}
};
export default reportWebVitals;
```
# playground-ui/src/setupTests.js
```js
// jest-dom adds custom jest matchers for asserting on DOM nodes.
// allows you to do things like:
// expect(element).toHaveTextContent(/react/i)
// learn more: https://github.com/testing-library/jest-dom
import '@testing-library/jest-dom';
```
# providers/AnthropicEmbeddingProvider.js
```js
// src/providers/AnthropicEmbeddingProvider.js
import BaseEmbeddingProvider from './BaseEmbeddingProvider.js';
import axios from 'axios';
import debug from 'debug';
const log = debug('mongodb-rag:embedding:anthropic');
class AnthropicEmbeddingProvider extends BaseEmbeddingProvider {
constructor(options = {}) {
super(options);
if (!options.apiKey) {
throw new Error('Anthropic API key is required');
}
this.apiKey = options.apiKey;
this.model = options.model || 'claude-3'; // Ensure model is supported
this.client = axios.create({
baseURL: 'https://api.anthropic.com/v1/messages', // ā
Update if necessary
headers: {
'x-api-key': this.apiKey,
'Content-Type': 'application/json'
}
});
log('Anthropic embedding provider initialized');
}
async _embedBatch(texts) {
try {
log(`Getting embeddings for batch of ${texts.length} texts`);
// ā
Use Claude to generate a textual embedding-friendly representation
const response = await this.client.post('', {
model: this.model,
max_tokens: 128, // Limit the response length
messages: texts.map(text => ({
role: "user",
content: `Summarize this for embedding: "${text}"`
}))
});
if (!response.data || !response.data.content) {
throw new Error(`Unexpected response from Anthropic API: ${JSON.stringify(response.data)}`);
}
// ā
Convert response text to embeddings using OpenAI
const openAIResponse = await axios.post('https://api.openai.com/v1/embeddings', {
model: "text-embedding-3-small",
input: response.data.content.map(msg => msg.content)
}, {
headers: { 'Authorization': `Bearer ${process.env.OPENAI_API_KEY}` }
});
return openAIResponse.data.data.map(item => item.embedding);
} catch (error) {
if (error.response?.data) {
throw new Error(`Anthropic API error: ${error.response.data.error?.message || JSON.stringify(error.response.data)}`);
}
throw error;
}
}
}
export default AnthropicEmbeddingProvider;
```
# providers/BaseEmbeddingProvider.js
```js
// src/providers/BaseEmbeddingProvider.js
import debug from 'debug';
const log = debug('mongodb-rag:embedding');
/**
* Base class for embedding providers that handles common functionality
* like batching, retries, and error handling.
* @class BaseEmbeddingProvider
*/
class BaseEmbeddingProvider {
/**
* Creates a new embedding provider instance
* @param {Object} options - Configuration options
* @param {number} [options.batchSize=100] - Number of texts to process in each batch
* @param {number} [options.maxRetries=3] - Maximum number of retry attempts
* @param {number} [options.retryDelay=1000] - Delay between retries in milliseconds
* @param {string} [options.provider] - Provider name ('openai', 'ollama', etc.)
* @param {string} [options.apiKey] - API key for the provider (required except for Ollama)
* @throws {Error} If API key is missing for non-Ollama providers
*/
constructor(options = {}) {
this.options = {
batchSize: options.batchSize || 100,
maxRetries: options.maxRetries || 3,
retryDelay: options.retryDelay || 1000,
...options
};
if (options.provider !== 'ollama' && !options.apiKey) {
throw new Error('API key is required');
}
}
/**
* Generates embeddings for one or more texts
* @param {string|string[]} texts - Text(s) to generate embeddings for
* @returns {Promise<number[][]>} Array of embedding vectors
*/
async getEmbeddings(texts) {
if (!Array.isArray(texts)) {
texts = [texts];
}
if (texts.length === 0) {
return [];
}
const batches = this._createBatches(texts);
const results = [];
for (const batch of batches) {
let embeddings = await this._processWithRetry(batch);
results.push(...embeddings);
}
return results;
}
/**
* Processes a batch of texts with retry logic
* @private
* @param {string[]} batch - Batch of texts to process
* @returns {Promise<number[][]>} Embeddings for the batch
* @throws {Error} If all retry attempts fail
*/
async _processWithRetry(batch) {
let lastError;
for (let attempt = 1; attempt <= this.options.maxRetries; attempt++) {
try {
return await this._embedBatch(batch);
} catch (error) {
lastError = error;
if (attempt === this.options.maxRetries) {
log(`Failed after ${attempt} attempts: ${error.message}`);
throw error;
}
log(`Retry ${attempt}/${this.options.maxRetries} after error: ${error.message}`);
await this._sleep(this.options.retryDelay * attempt);
}
}
}
/**
* Splits an array of texts into batches
* @private
* @param {string[]} texts - Array of texts to batch
* @returns {string[][]} Array of text batches
*/
_createBatches(texts) {
const batches = [];
for (let i = 0; i < texts.length; i += this.options.batchSize) {
batches.push(texts.slice(i, i + this.options.batchSize));
}
return batches;
}
/**
* Utility function to pause execution
* @private
* @param {number} ms - Milliseconds to sleep
* @returns {Promise<void>}
*/
async _sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
/**
* Abstract method to be implemented by provider classes
* @abstract
* @protected
* @param {string[]} texts - Batch of texts to embed
* @returns {Promise<number[][]>} Array of embedding vectors
* @throws {Error} If not implemented by provider class
*/
async _embedBatch(texts) {
throw new Error('_embedBatch must be implemented by the provider');
}
}
export default BaseEmbeddingProvider;
```
# providers/DeepSeekEmbeddingProvider.js
```js
// src/providers/DeepSeekEmbeddingProvider.js
import BaseEmbeddingProvider from './BaseEmbeddingProvider.js';
import axios from 'axios';
import debug from 'debug';
const log = debug('mongodb-rag:embedding:deepseek');
/**
* DeepSeek Embedding Provider implementation
* Generates embeddings using the DeepSeek API
* @extends BaseEmbeddingProvider
*/
class DeepSeekEmbeddingProvider extends BaseEmbeddingProvider {
/**
* Creates a new DeepSeek embedding provider instance
* @param {Object} options - Configuration options
* @param {string} options.apiKey - DeepSeek API key
* @param {string} [options.model='deepseek-embedding'] - Model to use for embeddings
* @throws {Error} If API key is not provided
*/
constructor(options = {}) {
super(options);
if (!options.apiKey) {
throw new Error('DeepSeek API key is required');
}
this.apiKey = options.apiKey;
this.model = options.model || 'deepseek-embedding';
this.client = axios.create({
baseURL: 'https://api.deepseek.com/v1',
headers: {
'Authorization': `Bearer ${this.apiKey}`,
'Content-Type': 'application/json'
}
});
log('DeepSeek embedding provider initialized');
}
/**
* Generates embeddings for a batch of texts using DeepSeek API
* @protected
* @param {string[]} texts - Array of texts to embed
* @returns {Promise<number[][]>} Array of embedding vectors
* @throws {Error} If the API request fails or returns unexpected response
*/
async _embedBatch(texts) {
try {
log(`Getting embeddings for batch of ${texts.length} texts`);
const response = await this.client.post('/embeddings', {
model: this.model,
input: texts
});
if (!response.data || !response.data.data) {
throw new Error(`Unexpected response from DeepSeek API: ${JSON.stringify(response.data)}`);
}
return response.data.data.map(item => item.embedding);
} catch (error) {
if (error.response?.data) {
throw new Error(`DeepSeek API error: ${error.response.data.error?.message || JSON.stringify(error.response.data)}`);
}
throw error;
}
}
}
// ā
Ensure the class is properly exported as default
export default DeepSeekEmbeddingProvider;
```
# providers/OllamaEmbeddingProvider.js
```js
// src/providers/OllamaEmbeddingProvider.js
import fetch from 'node-fetch';
import debug from 'debug';
const log = debug('mongodb-rag:ollama');
class OllamaEmbeddingProvider {
constructor(config) {
if (!config.baseUrl) {
throw new Error('Ollama base URL is required');
}
if (!config.model) {
throw new Error('Ollama model name is required');
}
this.baseUrl = config.baseUrl;
this.model = config.model;
this.dimensions = config.dimensions || 1536;
log(`Initialized Ollama provider with model: ${this.model}`);
}
async getEmbedding(text) {
if (!text || typeof text !== 'string') {
throw new Error('Input text must be a non-empty string');
}
try {
const response = await fetch(`${this.baseUrl}/api/embeddings`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model: this.model,
prompt: text
})
});
if (!response.ok) {
const error = await response.text();
throw new Error(`Ollama API error: ${error}`);
}
const data = await response.json();
if (!data.embedding || !Array.isArray(data.embedding)) {
throw new Error('Invalid response from Ollama API');
}
return data.embedding;
} catch (error) {
log('Error getting embedding from Ollama:', error);
throw error;
}
}
async getEmbeddings(texts) {
if (!Array.isArray(texts) || texts.length === 0) {
return [];
}
// Validate all inputs are strings
if (!texts.every(text => typeof text === 'string' && text.length > 0)) {
throw new Error('All inputs must be non-empty strings');
}
// Ollama doesn't support batch processing, so we need to process sequentially
try {
const embeddings = await Promise.all(texts.map(text => this.getEmbedding(text)));
return embeddings;
} catch (error) {
log('Error getting embeddings from Ollama:', error);
throw error;
}
}
async verifyModel() {
try {
const response = await fetch(`${this.baseUrl}/api/tags`);
if (!response.ok) {
throw new Error(`Failed to list models: ${response.statusText}`);
}
const data = await response.json();
return data.models?.some(model => model.name === this.model) || false;
} catch (error) {
throw new Error(`Failed to verify model ${this.model}: ${error.message}`);
}
}
}
export default OllamaEmbeddingProvider;
```
# providers/OpenAIEmbeddingProvider.js
```js
// src/providers/OpenAIEmbeddingProvider.js
import OpenAI from 'openai';
import debug from 'debug';
const log = debug('mongodb-rag:openai');
class OpenAIEmbeddingProvider {
constructor(config) {
if (!config.apiKey) {
throw new Error('OpenAI API key is required');
}
this.client = new OpenAI({ apiKey: config.apiKey });
this.model = config.model || 'text-embedding-3-small';
this.dimensions = config.dimensions || 1536;
log(`Initialized OpenAI provider with model: ${this.model}`);
}
async getEmbedding(text) {
if (!text || typeof text !== 'string') {
throw new Error('Input text must be a non-empty string');
}
try {
const response = await this.client.embeddings.create({
model: this.model,
input: text
});
if (!response.data || !response.data[0]?.embedding) {
throw new Error('Invalid response from OpenAI API');
}
return response.data[0].embedding;
} catch (error) {
log('Error getting embedding from OpenAI:', error);
throw error;
}
}
async getEmbeddings(texts) {
if (!Array.isArray(texts) || texts.length === 0) {
return [];
}
// Validate all inputs are strings
if (!texts.every(text => typeof text === 'string' && text.length > 0)) {
throw new Error('All inputs must be non-empty strings');
}
try {
const response = await this.client.embeddings.create({
model: this.model,
input: texts
});
if (!response.data || !Array.isArray(response.data)) {
throw new Error('Invalid response from OpenAI API');
}
return response.data.map(item => item.embedding);
} catch (error) {
log('Error getting embeddings from OpenAI:', error);
throw error;
}
}
}
export default OpenAIEmbeddingProvider;
```