docusaurus-openai-search
Version:
AI-powered search plugin for Docusaurus - extends Algolia search with intelligent keyword generation and RAG-based answers
777 lines (776 loc) • 30.9 kB
JavaScript
import { getLogger } from './logger';
import { addRecaptchaHeader } from './recaptcha';
export class SearchOrchestrator {
constructor(config, onProgress) {
this.logger = getLogger();
// P3-002: AbortController for canceling pending operations
this.abortController = null;
this.pendingOperations = new Set();
this.isDestroyed = false;
// Week 6: Session management
this.currentSessionId = null;
this.config = config;
this.onProgress = onProgress;
this.recaptchaSiteKey = config.recaptcha?.siteKey;
// Initialize abort controller for this orchestrator instance
this.abortController = new AbortController();
// Week 6: Initialize session if conversational memory is enabled
if (config.features?.conversationalMemory?.enabled) {
this.initializeSession();
}
}
/**
* P3-002: Cancel all pending operations and cleanup resources
*/
cancelAllOperations() {
this.isDestroyed = true;
if (this.abortController) {
this.abortController.abort();
this.abortController = null;
}
// Clear pending operations
this.pendingOperations.clear();
this.logger.log('SearchOrchestrator: All operations cancelled');
}
/**
* P3-002: Check if operations should be aborted due to race conditions
*/
checkAborted() {
if (this.isDestroyed || !this.abortController || this.abortController.signal.aborted) {
throw new Error('Operation cancelled');
}
}
/**
* P3-002: Track pending operation and handle completion
*/
async trackOperation(operation, operationName) {
this.checkAborted();
this.pendingOperations.add(operation);
try {
const result = await operation;
this.checkAborted(); // Check again after completion
return result;
}
catch (error) {
if (error instanceof Error && error.name === 'AbortError') {
this.logger.log(`${operationName} operation aborted`);
throw new Error('Operation cancelled');
}
throw error;
}
finally {
this.pendingOperations.delete(operation);
}
}
/**
* Main orchestration method that performs AI-powered search
* Week 2 Enhancement: Returns validation data along with answer and documents
*/
async performAISearch(query, algoliaClient, algoliaIndex) {
try {
// P3-002: Check if operation should proceed
this.checkAborted();
// Step 1: Request keywords from backend
this.updateProgress({
step: 'requesting-keywords',
message: 'Analyzing your question...',
progress: 10,
});
const keywords = await this.trackOperation(this.getKeywordsFromBackend(query), 'getKeywords');
this.updateProgress({
step: 'keywords-received',
message: 'Search strategy identified',
progress: 20,
details: { keywords }
});
// P3-002: Check abort before continuing
this.checkAborted();
// Step 2: Search for each keyword
this.updateProgress({
step: 'searching',
message: 'Searching documentation...',
progress: 30,
});
const allDocuments = [];
const documentLinks = [];
for (let i = 0; i < keywords.length; i++) {
// P3-002: Check abort before each search iteration
this.checkAborted();
const keyword = keywords[i];
this.updateProgress({
step: 'searching',
message: `Searching for: "${keyword}" (${i + 1}/${keywords.length})`,
progress: 30 + (i * 30 / keywords.length),
details: {
keywords,
documentsFound: allDocuments.length,
documentLinks
}
});
const results = await this.trackOperation(this.performSingleSearch(keyword, algoliaClient, algoliaIndex), `search-${keyword}`);
const documents = this.extractDocuments(results);
// Add unique documents
documents.forEach(doc => {
if (!documentLinks.includes(doc.url)) {
allDocuments.push(doc);
documentLinks.push(doc.url);
}
});
}
this.updateProgress({
step: 'documents-found',
message: `Found ${allDocuments.length} relevant documents`,
progress: 70,
details: {
keywords,
documentsFound: allDocuments.length,
documentLinks
}
});
// P3-002: Check abort before answer generation
this.checkAborted();
// Step 3: Generate answer using RAG
this.updateProgress({
step: 'generating-answer',
message: 'Generating comprehensive answer...',
progress: 80,
details: {
keywords,
documentsFound: allDocuments.length,
documentLinks
}
});
const result = await this.trackOperation(this.generateAnswerFromBackend(query, allDocuments), 'generateAnswer');
this.updateProgress({
step: 'complete',
message: 'Answer ready!',
progress: 100,
details: {
keywords,
documentsFound: allDocuments.length,
documentLinks
}
});
return {
answer: result.answer,
documents: allDocuments,
validation: result.validation, // Week 2: Pass validation data through
queryAnalysis: result.queryAnalysis, // Week 3: Pass query analysis data through
enhancement: result.enhancement // Stage 3: Pass recursive enhancement data through
};
}
catch (error) {
this.logger.logError('SearchOrchestrator', error);
throw error;
}
}
/**
* P3-002: Enhanced get search keywords from backend with AbortController
*/
async getKeywordsFromBackend(query) {
this.checkAborted();
let headers = {
'Content-Type': 'application/json',
};
// Add reCAPTCHA token if configured
if (this.recaptchaSiteKey) {
headers = await addRecaptchaHeader(headers, this.recaptchaSiteKey, 'keywords');
}
// P3-002: Check abort after potentially async reCAPTCHA operation
this.checkAborted();
const response = await fetch(`${this.config.backend.url}/api/keywords`, {
method: 'POST',
headers,
body: JSON.stringify({
query,
systemContext: this.config.context?.systemContext,
maxKeywords: this.config.maxSearchQueries || 5
}),
// P3-002: Add abort signal to fetch
signal: this.abortController?.signal,
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error?.message || 'Failed to get keywords');
}
const data = await response.json();
return data.keywords;
}
/**
* P3-002: Enhanced generate answer from backend using RAG with AbortController
* Week 2 Enhancement: Returns both answer and validation data
*/
async generateAnswerFromBackend(query, documents) {
this.checkAborted();
let headers = {
'Content-Type': 'application/json',
};
// Add reCAPTCHA token if configured
if (this.recaptchaSiteKey) {
headers = await addRecaptchaHeader(headers, this.recaptchaSiteKey, 'generate_answer');
}
// P3-002: Check abort after potentially async reCAPTCHA operation
this.checkAborted();
const response = await fetch(`${this.config.backend.url}/api/generate-answer`, {
method: 'POST',
headers,
body: JSON.stringify({
query,
documents: documents.slice(0, 10), // Backend will handle the limit
systemContext: this.config.context?.systemContext,
}),
// P3-002: Add abort signal to fetch
signal: this.abortController?.signal,
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error?.message || 'Failed to generate answer');
}
const data = await response.json();
return {
answer: data.answer,
validation: data.validation, // Week 2: Capture validation data
queryAnalysis: data.queryAnalysis, // Week 3: Capture query analysis data
enhancement: data.enhancement // Stage 3: Capture recursive enhancement data
};
}
/**
* Expand query with variations and synonyms for better search results
*/
expandQuery(query) {
const variations = [query];
// Remove special characters version
const cleanQuery = query.replace(/[^a-zA-Z0-9\s]/g, ' ').trim();
if (cleanQuery !== query) {
variations.push(cleanQuery);
}
// Common substitutions
const substitutions = [
['use', 'usage'],
['config', 'configuration'],
['auth', 'authentication'],
['docs', 'documentation'],
['api', 'API'],
['intro', 'introduction'],
['dev', 'development'],
['prod', 'production']
];
substitutions.forEach(([from, to]) => {
if (query.toLowerCase().includes(from)) {
variations.push(query.replace(new RegExp(from, 'gi'), to));
}
});
return [...new Set(variations)].slice(0, 3); // Max 3 variations
}
/**
* P3-002: Enhanced perform a single search query with query expansion and abort checking
*/
async performSingleSearch(query, algoliaClient, indexName, hitsPerPage = 10 // Increased from 5
) {
this.checkAborted();
const allHits = [];
const seenUrls = new Set();
// Try query variations
const queries = this.expandQuery(query);
for (const q of queries) {
try {
const response = await algoliaClient.search([{
indexName,
query: q,
params: {
hitsPerPage,
attributesToRetrieve: ['*'],
attributesToHighlight: ['*'],
highlightPreTag: '<mark>',
highlightPostTag: '</mark>',
// Add these for better results
removeWordsIfNoResults: 'allOptional',
queryType: 'prefixLast'
}
}]);
const hits = response.results[0]?.hits || [];
// Deduplicate by URL
hits.forEach((hit) => {
if (hit.url && !seenUrls.has(hit.url)) {
seenUrls.add(hit.url);
allHits.push(hit);
}
});
}
catch (error) {
this.logger.log(`Search failed for query variant "${q}":`, error);
}
}
return allHits.slice(0, hitsPerPage); // Return top N unique results
}
/**
* Extract document content from search results with enhanced context
*/
extractDocuments(searchResults) {
return searchResults.map(result => {
let content = '';
// Build complete hierarchy path for context
const hierarchyPath = [];
const levels = ['lvl0', 'lvl1', 'lvl2', 'lvl3', 'lvl4', 'lvl5'];
levels.forEach(level => {
const value = result.hierarchy[level];
if (value && !hierarchyPath.includes(value)) {
hierarchyPath.push(value);
}
});
// Add section path as context
if (hierarchyPath.length > 0) {
content += `Section: ${hierarchyPath.join(' > ')}\n\n`;
}
// Add main content
if (result.content) {
content += `${result.content}\n\n`;
}
// Add highlighted snippets
if (result._highlightResult?.content?.value) {
const highlighted = result._highlightResult.content.value
.replace(/<mark>/g, '')
.replace(/<\/mark>/g, '')
.replace(/<[^>]*>/g, '');
// Only add if it's different from main content
if (!content.includes(highlighted)) {
content += `Key excerpt: ${highlighted}\n\n`;
}
}
// Extract any code blocks from snippets
if (result._snippetResult?.content?.value) {
const snippet = result._snippetResult.content.value;
const codeMatch = snippet.match(/```[\s\S]*?```/g);
if (codeMatch) {
content += `Code example found:\n${codeMatch[0]}\n\n`;
}
}
const title = hierarchyPath[hierarchyPath.length - 1] ||
result.hierarchy?.lvl1 ||
'Documentation';
return {
url: result.url,
title: title.replace(/<[^>]*>/g, ''),
content: content.trim()
};
}).filter(doc => doc.content); // Remove empty documents
}
/**
* Update progress
*/
updateProgress(step) {
if (this.onProgress) {
this.onProgress(step);
}
this.logger.log(`Search step: ${step.step} - ${step.message}`);
}
/**
* Stage 2: Enhanced AI search with multi-source capabilities
*/
async performMultiSourceAISearch(query, searchClient, indexName, multiSourceConfig) {
try {
// P3-002: Check if operation should proceed
this.checkAborted();
// Step 1: Request keywords from backend
this.updateProgress({
step: 'requesting-keywords',
message: 'Analyzing your question...',
progress: 5,
});
const keywords = await this.trackOperation(this.getKeywordsFromBackend(query), 'getKeywords');
this.updateProgress({
step: 'keywords-received',
message: 'Search strategy identified',
progress: 10,
details: { keywords }
});
// P3-002: Check abort before continuing
this.checkAborted();
// Step 2: Search documentation (traditional search)
this.updateProgress({
step: 'searching',
message: 'Searching documentation...',
progress: 20,
});
const allDocuments = [];
const documentLinks = [];
for (let i = 0; i < keywords.length; i++) {
// P3-002: Check abort before each search iteration
this.checkAborted();
const keyword = keywords[i];
this.updateProgress({
step: 'searching',
message: `Searching documentation: "${keyword}" (${i + 1}/${keywords.length})`,
progress: 20 + (i * 20 / keywords.length),
details: {
keywords,
documentsFound: allDocuments.length,
documentLinks
}
});
const results = await this.trackOperation(this.performSingleSearch(keyword, searchClient, indexName), `search-${keyword}`);
const documents = this.extractDocuments(results);
// Add unique documents
documents.forEach(doc => {
if (!documentLinks.includes(doc.url)) {
allDocuments.push(doc);
documentLinks.push(doc.url);
}
});
}
this.updateProgress({
step: 'documents-found',
message: `Found ${allDocuments.length} documentation sources`,
progress: 40,
details: {
keywords,
documentsFound: allDocuments.length,
documentLinks
}
});
// P3-002: Check abort before multi-source search
this.checkAborted();
// Step 3: Multi-source search
this.updateProgress({
step: 'multi-source-search',
message: 'Searching additional sources...',
progress: 50,
});
const multiSourceResult = await this.trackOperation(this.performMultiSourceSearch(query, allDocuments, multiSourceConfig), 'multiSourceSearch');
this.updateProgress({
step: 'aggregating-results',
message: 'Aggregating results from all sources...',
progress: 80,
details: {
keywords,
documentsFound: allDocuments.length,
documentLinks,
sourcesFound: multiSourceResult.aggregationMetrics.sourceBreakdown
}
});
// P3-002: Check abort before completion
this.checkAborted();
this.updateProgress({
step: 'complete',
message: 'Search completed successfully',
progress: 100,
details: {
keywords,
documentsFound: allDocuments.length,
documentLinks,
sourcesFound: multiSourceResult.aggregationMetrics.sourceBreakdown
}
});
return multiSourceResult;
}
catch (error) {
// P3-002: Handle cancellation gracefully
if (error.message === 'Operation cancelled') {
this.logger.log('Multi-source AI search cancelled');
throw error;
}
this.logger.error('Multi-source AI search failed:', error);
// Fallback to basic search
const fallbackResult = await this.performAISearch(query, searchClient, indexName);
return {
answer: fallbackResult.answer,
sources: fallbackResult.documents.map(doc => ({
source: 'documentation',
title: doc.title,
url: doc.url,
content: doc.content,
metadata: {
weight: 0.5,
type: 'documentation'
}
})),
aggregationMetrics: {
totalSources: fallbackResult.documents.length,
sourceBreakdown: {
documentation: fallbackResult.documents.length,
github: 0,
blog: 0,
changelog: 0
},
confidenceScore: 50
},
validation: fallbackResult.validation
};
}
}
/**
* Stage 2: Multi-source search backend integration
*/
async performMultiSourceSearch(query, documents, multiSourceConfig) {
const requestBody = {
query,
documents,
systemContext: this.config.systemContext,
config: multiSourceConfig || {}
};
const requestOptions = {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'User-Agent': 'docusaurus-ai-search/1.0',
},
body: JSON.stringify(requestBody),
signal: this.abortController?.signal
};
// Add reCAPTCHA header if enabled
if (this.recaptchaSiteKey) {
await addRecaptchaHeader(requestOptions.headers, this.recaptchaSiteKey, 'multi_source_search');
}
const response = await fetch(`${this.config.backend.url}/api/multi-source-search`, requestOptions);
if (!response.ok) {
const errorData = await response.json().catch(() => ({}));
throw new Error(`Multi-source search failed: ${errorData.error?.message || response.statusText}`);
}
const data = await response.json();
return {
answer: data.answer,
sources: data.sources || [],
aggregationMetrics: data.aggregationMetrics || {
totalSources: 0,
sourceBreakdown: { documentation: 0, github: 0, blog: 0, changelog: 0 },
confidenceScore: 0
},
validation: data.validation
};
}
/**
* Week 6: Initialize a new conversation session
*/
async initializeSession() {
try {
const response = await fetch(`${this.config.backend.url}/api/session/create`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({
systemContext: this.config.context?.systemContext
}),
signal: this.abortController?.signal
});
if (response.ok) {
const data = await response.json();
this.currentSessionId = data.sessionId;
this.logger.log(`Session initialized: ${this.currentSessionId}`);
}
}
catch (error) {
this.logger.error('Failed to initialize session:', error);
// Continue without session if creation fails
}
}
/**
* Week 6: Get current session ID
*/
getSessionId() {
return this.currentSessionId;
}
/**
* Week 6: Get conversation history for current session
*/
async getConversationHistory() {
if (!this.currentSessionId) {
return [];
}
try {
const response = await fetch(`${this.config.backend.url}/api/session/${this.currentSessionId}/history`, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
},
signal: this.abortController?.signal
});
if (response.ok) {
const data = await response.json();
return data.turns || [];
}
}
catch (error) {
this.logger.error('Failed to get conversation history:', error);
}
return [];
}
/**
* Week 6: Enhanced AI search with conversational memory
*/
async performConversationalAISearch(query, algoliaClient, algoliaIndex) {
try {
// P3-002: Check if operation should proceed
this.checkAborted();
// Step 1: Request keywords from backend
this.updateProgress({
step: 'requesting-keywords',
message: 'Analyzing your question...',
progress: 10,
});
const keywords = await this.trackOperation(this.getKeywordsFromBackend(query), 'getKeywords');
this.updateProgress({
step: 'keywords-received',
message: 'Search strategy identified',
progress: 20,
details: { keywords }
});
// P3-002: Check abort before continuing
this.checkAborted();
// Step 2: Search for each keyword
this.updateProgress({
step: 'searching',
message: 'Searching documentation...',
progress: 30,
});
const allDocuments = [];
const documentLinks = [];
for (let i = 0; i < keywords.length; i++) {
// P3-002: Check abort before each search iteration
this.checkAborted();
const keyword = keywords[i];
this.updateProgress({
step: 'searching',
message: `Searching for: "${keyword}" (${i + 1}/${keywords.length})`,
progress: 30 + (i * 30 / keywords.length),
details: {
keywords,
documentsFound: allDocuments.length,
documentLinks
}
});
const results = await this.trackOperation(this.performSingleSearch(keyword, algoliaClient, algoliaIndex), `search-${keyword}`);
const documents = this.extractDocuments(results);
// Add unique documents
documents.forEach(doc => {
if (!documentLinks.includes(doc.url)) {
allDocuments.push(doc);
documentLinks.push(doc.url);
}
});
}
this.updateProgress({
step: 'documents-found',
message: `Found ${allDocuments.length} relevant documents`,
progress: 70,
details: {
keywords,
documentsFound: allDocuments.length,
documentLinks
}
});
// P3-002: Check abort before answer generation
this.checkAborted();
// Step 3: Generate answer with conversational memory
this.updateProgress({
step: 'generating-answer',
message: 'Generating comprehensive answer with context...',
progress: 80,
});
const answerResult = await this.trackOperation(this.generateAnswerWithMemory(query, allDocuments), 'generateAnswerWithMemory');
this.updateProgress({
step: 'complete',
message: 'Search completed successfully',
progress: 100,
details: {
keywords,
documentsFound: allDocuments.length,
documentLinks
}
});
return {
answer: answerResult.answer,
documents: allDocuments,
validation: answerResult.validation,
queryAnalysis: answerResult.queryAnalysis,
followUpQuestions: answerResult.followUpQuestions,
sessionId: answerResult.sessionId || this.currentSessionId || undefined
};
}
catch (error) {
// P3-002: Handle cancellation gracefully
if (error.message === 'Operation cancelled') {
this.logger.log('Conversational AI search cancelled');
throw error;
}
this.logger.error('Conversational AI search failed:', error);
// Fallback to basic search
const fallbackResult = await this.performAISearch(query, algoliaClient, algoliaIndex);
return {
...fallbackResult,
followUpQuestions: [],
sessionId: this.currentSessionId || undefined
};
}
}
/**
* Week 6: Generate answer with conversational memory
*/
async generateAnswerWithMemory(query, documents) {
const requestBody = {
query,
documents,
systemContext: this.config.context?.systemContext,
sessionId: this.currentSessionId
};
const requestOptions = {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'User-Agent': 'docusaurus-ai-search/1.0',
},
body: JSON.stringify(requestBody),
signal: this.abortController?.signal
};
// Add reCAPTCHA if enabled
if (this.recaptchaSiteKey) {
await addRecaptchaHeader(requestOptions.headers, this.recaptchaSiteKey, 'generate_answer_with_memory');
}
const response = await fetch(`${this.config.backend.url}/api/generate-answer-with-memory`, requestOptions);
if (!response.ok) {
const errorData = await response.json().catch(() => ({}));
throw new Error(`Answer generation failed: ${errorData.error?.message || response.statusText}`);
}
const data = await response.json();
// Update session ID if returned from backend
if (data.sessionId && !this.currentSessionId) {
this.currentSessionId = data.sessionId;
}
return {
answer: data.answer,
validation: data.validation,
queryAnalysis: data.queryAnalysis,
followUpQuestions: data.followUpQuestions || [],
sessionId: data.sessionId
};
}
/**
* Week 6: Generate follow-up questions for a given query and answer
*/
async generateFollowUpQuestions(query, answer, queryAnalysis) {
try {
const requestBody = {
sessionId: this.currentSessionId,
query,
answer,
queryAnalysis
};
const response = await fetch(`${this.config.backend.url}/api/follow-up-questions`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(requestBody),
signal: this.abortController?.signal
});
if (response.ok) {
const data = await response.json();
return data.followUpQuestions || [];
}
}
catch (error) {
this.logger.error('Failed to generate follow-up questions:', error);
}
return [];
}
}