@vfarcic/dot-ai
Version:
AI-powered development productivity platform that enhances software development workflows through intelligent automation and AI-driven assistance
1,001 lines (997 loc) • 59.1 kB
JavaScript
"use strict";
/**
* Remediate Tool - AI-powered Kubernetes issue analysis and remediation
*/
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
var desc = Object.getOwnPropertyDescriptor(m, k);
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
desc = { enumerable: true, get: function() { return m[k]; } };
}
Object.defineProperty(o, k2, desc);
}) : (function(o, m, k, k2) {
if (k2 === undefined) k2 = k;
o[k2] = m[k];
}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __importStar = (this && this.__importStar) || (function () {
var ownKeys = function(o) {
ownKeys = Object.getOwnPropertyNames || function (o) {
var ar = [];
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
return ar;
};
return ownKeys(o);
};
return function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
__setModuleDefault(result, mod);
return result;
};
})();
Object.defineProperty(exports, "__esModule", { value: true });
exports.REMEDIATE_TOOL_INPUT_SCHEMA = exports.REMEDIATE_TOOL_DESCRIPTION = exports.REMEDIATE_TOOL_NAME = void 0;
exports.parseAIFinalAnalysis = parseAIFinalAnalysis;
exports.handleRemediateTool = handleRemediateTool;
const zod_1 = require("zod");
const error_handling_1 = require("../core/error-handling");
const ai_provider_factory_1 = require("../core/ai-provider-factory");
const mcp_client_registry_1 = require("../core/mcp-client-registry");
const generic_session_manager_1 = require("../core/generic-session-manager");
const index_1 = require("../core/index");
const visualization_1 = require("../core/visualization");
const plugin_registry_1 = require("../core/plugin-registry");
const fs = __importStar(require("fs"));
const path = __importStar(require("path"));
const request_context_1 = require("../interfaces/request-context");
const rbac_1 = require("../core/rbac");
const session_events_1 = require("../core/session-events");
const internal_tools_1 = require("../core/internal-tools");
// PRD #143 Milestone 1: Hybrid approach - AI can use kubectl_api_resources tool OR continue with JSON dataRequests
// Tool metadata for direct MCP registration
exports.REMEDIATE_TOOL_NAME = 'remediate';
exports.REMEDIATE_TOOL_DESCRIPTION = 'AI-powered Kubernetes issue analysis that provides root cause identification and actionable remediation steps. Unlike basic kubectl commands, this tool performs multi-step investigation, correlates cluster data, and generates intelligent solutions. Use when users want to understand WHY something is broken, not just see raw status. Ideal for: troubleshooting failures, diagnosing performance issues, analyzing pod problems, investigating networking/storage issues, or any "what\'s wrong" questions.';
// Zod schema for MCP registration
exports.REMEDIATE_TOOL_INPUT_SCHEMA = {
issue: zod_1.z
.string()
.min(1)
.max(2000)
.describe('Issue description that needs to be analyzed and remediated')
.optional(),
mode: zod_1.z
.enum(['manual', 'automatic'])
.optional()
.default('manual')
.describe('Execution mode: manual requires user approval, automatic executes based on thresholds'),
confidenceThreshold: zod_1.z
.number()
.min(0)
.max(1)
.optional()
.default(0.8)
.describe('For automatic mode: minimum confidence required for execution (default: 0.8)'),
maxRiskLevel: zod_1.z
.enum(['low', 'medium', 'high'])
.optional()
.default('low')
.describe('For automatic mode: maximum risk level allowed for execution (default: low)'),
executeChoice: zod_1.z
.number()
.min(1)
.max(2)
.optional()
.describe('Execute a previously generated choice (1=Execute via MCP, 2=Execute via agent)'),
sessionId: zod_1.z
.string()
.optional()
.describe('Session ID from previous remediate call when executing a choice'),
executedCommands: zod_1.z
.array(zod_1.z.string())
.optional()
.describe('Commands that were executed to remediate the issue'),
interaction_id: zod_1.z
.string()
.optional()
.describe('INTERNAL ONLY - Do not populate. Used for evaluation dataset generation.'),
};
/** Kubectl and Helm tool names for investigation and validation */
const KUBECTL_INVESTIGATION_TOOL_NAMES = [
'kubectl_get',
'kubectl_describe',
'kubectl_logs',
'kubectl_events',
'kubectl_api_resources',
'kubectl_get_crd_schema',
'kubectl_get_resource_json',
// Helm investigation tools (PRD #251: Helm Day-2 operations)
'helm_list',
'helm_status',
'helm_history',
'helm_get_values',
// Dry-run tools for validation
'kubectl_patch_dryrun',
'kubectl_apply_dryrun',
'kubectl_delete_dryrun',
];
/**
* AI-driven investigation - uses toolLoop for single-phase investigation and analysis
*
* PRD #343: Kubectl tools are routed through the plugin system.
* PRD #359: Uses unified plugin registry for tool access.
* No fallback to direct execution - MCP server has no RBAC.
*/
async function conductInvestigation(session, sessionManager, aiProvider, logger, requestId, isValidation = false, interactionId) {
if (!(0, plugin_registry_1.isPluginInitialized)()) {
throw new Error('Plugin system not initialized');
}
const pluginManager = (0, plugin_registry_1.getPluginManager)();
const maxIterations = 25;
logger.info('Starting AI investigation with toolLoop', {
requestId,
sessionId: session.sessionId,
issue: session.data.issue,
});
try {
// Load investigation system prompt (static, cacheable)
const promptPath = path.join(__dirname, '..', '..', 'prompts', 'remediate-system.md');
const systemPrompt = fs.readFileSync(promptPath, 'utf8');
// PRD #343: Get kubectl tools from plugin
const kubectlTools = pluginManager
.getDiscoveredTools()
.filter(t => KUBECTL_INVESTIGATION_TOOL_NAMES.includes(t.name));
if (kubectlTools.length === 0) {
throw new Error('No kubectl tools available from plugin. Ensure agentic-tools plugin is running.');
}
// PRD #407: Combine plugin tools (allowlisted) with internal tools (git_clone, fs_list, fs_read)
// PRD #358: Get MCP server tools attached to remediate
const mcpTools = (0, mcp_client_registry_1.isMcpClientInitialized)()
? (0, mcp_client_registry_1.getMcpClientManager)().getToolsForOperation('remediate')
: [];
const allTools = [...kubectlTools, ...(0, internal_tools_1.getInternalTools)(), ...mcpTools];
logger.debug('Starting toolLoop with investigation tools', {
requestId,
sessionId: session.sessionId,
toolCount: allTools.length,
mcpToolCount: mcpTools.length,
tools: allTools.map(t => t.name),
});
// PRD #407: Clean up old clone directories (non-blocking)
(0, internal_tools_1.cleanupOldClones)();
// PRD #407: Combined executor routes plugin tools to plugin, internal tools to local handlers
// PRD #358: Chain MCP executor with plugin executor as fallback
const internalExecutor = (0, internal_tools_1.createInternalToolExecutor)(session.sessionId);
const pluginExecutor = pluginManager.createToolExecutor(internalExecutor);
const toolExecutor = (0, mcp_client_registry_1.isMcpClientInitialized)()
? (0, mcp_client_registry_1.getMcpClientManager)().createToolExecutor(pluginExecutor)
: pluginExecutor;
// Use toolLoop for AI-driven investigation with all tools (kubectl + internal + MCP)
// System prompt is static (cached), issue description is dynamic (userMessage)
const operationName = isValidation
? 'remediate-validation'
: 'remediate-investigation';
const result = await aiProvider.toolLoop({
systemPrompt: systemPrompt,
userMessage: `Investigate this Kubernetes issue: ${session.data.issue}`,
tools: allTools,
toolExecutor: toolExecutor,
maxIterations: maxIterations,
operation: operationName,
evaluationContext: {
user_intent: session.data.issue,
},
interaction_id: interactionId,
});
logger.info('Investigation completed by toolLoop', {
requestId,
sessionId: session.sessionId,
iterations: result.iterations,
toolCallsExecuted: result.toolCallsExecuted.length,
responseLength: result.finalMessage.length,
});
// Guard: if the AI call did not succeed, surface the real error instead of trying to parse
if (result.status && result.status !== 'success') {
throw new Error(`Remediation investigation ${result.status}: ${result.finalMessage}`);
}
// Parse final response as JSON (AI returns final analysis in JSON format)
const finalAnalysis = parseAIFinalAnalysis(result.finalMessage);
// Build RemediateOutput from parsed analysis
const output = {
status: finalAnalysis.issueStatus === 'active'
? 'awaiting_user_approval'
: 'success',
sessionId: session.sessionId,
investigation: {
iterations: result.iterations,
dataGathered: result.toolCallsExecuted.map((tc, i) => `${tc.tool} (call ${i + 1})`),
},
analysis: {
rootCause: finalAnalysis.rootCause,
confidence: finalAnalysis.confidence,
factors: finalAnalysis.factors,
},
remediation: finalAnalysis.remediation,
validationIntent: finalAnalysis.validationIntent,
executed: false,
mode: session.data.mode,
};
// Add guidance based on issue status
if (finalAnalysis.issueStatus === 'resolved' ||
finalAnalysis.issueStatus === 'non_existent') {
const statusMessage = finalAnalysis.issueStatus === 'resolved'
? 'Issue has been successfully resolved'
: 'No issues found - system is healthy';
output.guidance = `✅ ${statusMessage.toUpperCase()}: ${finalAnalysis.remediation.summary}`;
output.agentInstructions = `1. Show user that the ${finalAnalysis.issueStatus === 'resolved' ? 'issue has been resolved' : 'no issues were found'}\n2. Display the analysis and confidence level\n3. Explain the current healthy state\n4. No further action required`;
output.message = `${statusMessage} with ${Math.round(finalAnalysis.confidence * 100)}% confidence.`;
}
else {
// Active issue - generate execution options
const commandsSummary = finalAnalysis.remediation.actions.length === 1
? `The following kubectl command will be executed:\n${finalAnalysis.remediation.actions[0].command}`
: `The following ${finalAnalysis.remediation.actions.length} kubectl commands will be executed:\n${finalAnalysis.remediation.actions.map((action, i) => `${i + 1}. ${action.command}`).join('\n')}`;
const highRiskActions = finalAnalysis.remediation.actions.filter(a => a.risk === 'high');
const mediumRiskActions = finalAnalysis.remediation.actions.filter(a => a.risk === 'medium');
const riskSummary = [
...(highRiskActions.length > 0
? [
`${highRiskActions.length} HIGH RISK actions require careful review`,
]
: []),
...(mediumRiskActions.length > 0
? [
`${mediumRiskActions.length} MEDIUM RISK actions should be executed with monitoring`,
]
: []),
'All actions are designed to be safe kubectl operations (no destructive commands)',
].join('. ');
output.guidance = `🔴 CRITICAL: Present the kubectl commands to the user and ask them to choose execution method. DO NOT execute commands without user approval.\n\n${commandsSummary}\n\nRisk Assessment: ${riskSummary}`;
output.agentInstructions = `1. Show the user the root cause analysis and confidence level\n2. Display the kubectl commands that will be executed\n3. Explain the risk assessment\n4. Present the two execution choices and wait for user selection\n5. When user selects option 1 or 2, call the remediate tool again with: executeChoice: [1 or 2], sessionId: "${session.sessionId}", mode: "${session.data.mode}"\n6. DO NOT automatically execute any commands until user makes their choice\n7. Before executing, the user can run impact_analysis to understand the blast radius and downstream dependencies of the remediation actions`;
output.nextAction = 'remediate';
output.message = `AI analysis identified the root cause with ${Math.round(finalAnalysis.confidence * 100)}% confidence. ${finalAnalysis.remediation.actions.length} remediation actions are recommended.`;
}
// Update session with final analysis
sessionManager.updateSession(session.sessionId, {
finalAnalysis: output,
status: 'analysis_complete',
});
(0, session_events_1.getSessionEventBus)().publish(session_events_1.SESSION_EVENTS.SESSION_UPDATED, {
sessionId: session.sessionId,
toolName: 'remediate',
status: 'analysis_complete',
issue: session.data.issue,
timestamp: new Date().toISOString(),
});
logger.info('Investigation and analysis completed', {
requestId,
sessionId: session.sessionId,
rootCause: output.analysis.rootCause,
recommendedActions: output.remediation.actions.length,
});
return output;
}
catch (error) {
logger.error('Investigation failed', error, {
requestId,
sessionId: session.sessionId,
});
// Mark session as failed
sessionManager.updateSession(session.sessionId, { status: 'failed' });
(0, session_events_1.getSessionEventBus)().publish(session_events_1.SESSION_EVENTS.SESSION_UPDATED, {
sessionId: session.sessionId,
toolName: 'remediate',
status: 'failed',
issue: session.data.issue,
timestamp: new Date().toISOString(),
});
throw error_handling_1.ErrorHandler.createError(error_handling_1.ErrorCategory.AI_SERVICE, error_handling_1.ErrorSeverity.HIGH, `Investigation failed: ${error instanceof Error ? error.message : 'Unknown error'}`, {
operation: 'investigation_loop',
component: 'RemediateTool',
input: { sessionId: session.sessionId },
});
}
}
/**
* Parse AI final analysis response
*/
function parseAIFinalAnalysis(aiResponse) {
try {
// Try to extract JSON from the response
// Use non-greedy match and try to parse incrementally to handle extra text after JSON
const firstBraceIndex = aiResponse.indexOf('{');
if (firstBraceIndex === -1) {
throw new Error('No JSON found in AI final analysis response');
}
// Try to find the end of the JSON object by tracking brace depth
let braceCount = 0;
let inString = false;
let escapeNext = false;
let jsonEndIndex = -1;
for (let i = firstBraceIndex; i < aiResponse.length; i++) {
const char = aiResponse[i];
if (escapeNext) {
escapeNext = false;
continue;
}
if (char === '\\') {
escapeNext = true;
continue;
}
if (char === '"') {
inString = !inString;
continue;
}
if (inString)
continue;
if (char === '{')
braceCount++;
if (char === '}') {
braceCount--;
if (braceCount === 0) {
jsonEndIndex = i + 1;
break;
}
}
}
if (jsonEndIndex === -1) {
throw new Error('Could not find complete JSON object in AI response');
}
const jsonString = aiResponse.substring(firstBraceIndex, jsonEndIndex);
const parsed = JSON.parse(jsonString);
// Validate required fields
if (!parsed.issueStatus ||
!parsed.rootCause ||
parsed.confidence === undefined ||
!Array.isArray(parsed.factors) ||
!parsed.remediation) {
throw new Error('Invalid AI final analysis response structure');
}
// Validate issueStatus field
if (!['active', 'resolved', 'non_existent'].includes(parsed.issueStatus)) {
throw new Error(`Invalid issue status: ${parsed.issueStatus}. Must be 'active', 'resolved', or 'non_existent'`);
}
if (!parsed.remediation.summary ||
!Array.isArray(parsed.remediation.actions) ||
!parsed.remediation.risk) {
throw new Error('Invalid remediation structure in AI final analysis response');
}
// Validate each remediation action
for (const action of parsed.remediation.actions) {
if (!action.description || !action.risk || !action.rationale) {
throw new Error('Invalid remediation action structure');
}
if (!['low', 'medium', 'high'].includes(action.risk)) {
throw new Error(`Invalid risk level: ${action.risk}`);
}
}
// Validate overall risk level
if (!['low', 'medium', 'high'].includes(parsed.remediation.risk)) {
throw new Error(`Invalid overall risk level: ${parsed.remediation.risk}`);
}
// Validate confidence is between 0 and 1
if (parsed.confidence < 0 || parsed.confidence > 1) {
throw new Error(`Invalid confidence value: ${parsed.confidence}. Must be between 0 and 1`);
}
return parsed;
}
catch (error) {
// Log the actual AI response content when parsing fails - critical for debugging
console.error('🚨 JSON PARSING FAILED - AI Response Content:', {
responseLength: aiResponse.length,
actualResponse: aiResponse,
errorMessage: error instanceof Error ? error.message : 'Unknown error',
firstChars: aiResponse.substring(0, 100),
lastChars: aiResponse.length > 100
? aiResponse.substring(aiResponse.length - 100)
: '',
});
throw new Error(`Failed to parse AI final analysis response: ${error instanceof Error ? error.message : 'Unknown error'}. Response content: "${aiResponse}"`, { cause: error });
}
}
/**
* Execute user choice from previous session
* PRD #359: Uses unified plugin registry
*/
async function executeUserChoice(sessionManager, sessionId, choice, logger, requestId, currentInteractionId) {
try {
// Load previous session
const session = sessionManager.getSession(sessionId);
if (!session) {
throw new Error(`Session file not found: ${sessionId}`);
}
if (!session.data.finalAnalysis) {
throw error_handling_1.ErrorHandler.createError(error_handling_1.ErrorCategory.VALIDATION, error_handling_1.ErrorSeverity.HIGH, 'Session does not have final analysis - cannot execute choice', { operation: 'choice_execution', component: 'RemediateTool', sessionId });
}
logger.info('Loaded session for choice execution', {
requestId,
sessionId,
choice,
actionCount: session.data.finalAnalysis.remediation.actions.length,
});
// Handle different choices
switch (choice) {
case 1: // Execute automatically via MCP
return await executeRemediationCommands(session, sessionManager, logger, requestId, currentInteractionId);
case 2: {
// Execute via agent
const actions = session.data.finalAnalysis.remediation.actions;
const gitSourceActions = actions.filter(a => a.gitSource && !a.command);
const kubectlActions = actions.filter(a => a.command && !a.gitSource);
if (gitSourceActions.length > 0 && kubectlActions.length === 0) {
return {
content: [
{
type: 'text',
text: JSON.stringify({
status: 'success',
sessionId: sessionId,
message: 'GitOps remediation detected - use automatic execution (choice 1) for PR creation',
remediation: session.data.finalAnalysis.remediation,
instructions: {
nextSteps: [
'This remediation requires GitOps PR creation which is handled automatically.',
'Please use choice 1 (Execute automatically) to create the PR.',
'Alternatively, manually apply the file changes from gitSource.files to your repository.',
],
},
}, null, 2),
},
],
};
}
const validationIntent = session.data.finalAnalysis.validationIntent ||
'Check the status of the affected resources to verify the issue has been resolved';
return {
content: [
{
type: 'text',
text: JSON.stringify({
status: 'success',
sessionId: sessionId,
message: 'Ready for agent execution',
remediation: session.data.finalAnalysis.remediation,
instructions: {
nextSteps: [
'STEP 1: Execute the kubectl commands shown in the remediation section using your Bash tool',
'STEP 2: After successful execution, call the remediation tool with validation using these parameters:',
`issue: "${validationIntent}"`,
`executedCommands: [list of commands you executed]`,
'STEP 3: The tool will perform fresh validation to confirm the issue is resolved',
],
},
}, null, 2),
},
],
};
}
default:
throw error_handling_1.ErrorHandler.createError(error_handling_1.ErrorCategory.VALIDATION, error_handling_1.ErrorSeverity.HIGH, `Invalid choice: ${choice}. Must be 1 or 2`, { operation: 'choice_validation', component: 'RemediateTool' });
}
}
catch (error) {
logger.error('Choice execution failed', error, {
requestId,
sessionId,
choice,
});
if (error instanceof Error &&
error.message.includes('Session file not found')) {
throw error_handling_1.ErrorHandler.createError(error_handling_1.ErrorCategory.OPERATION, error_handling_1.ErrorSeverity.HIGH, `Session not found: ${sessionId}. The session may have expired or been deleted.`, { operation: 'session_loading', component: 'RemediateTool' });
}
throw error;
}
}
/**
* Execute remediation commands via kubectl
* PRD #359: Uses unified plugin registry
*/
async function executeRemediationCommands(session, sessionManager, logger, requestId, currentInteractionId) {
const results = [];
const finalAnalysis = session.data.finalAnalysis;
let overallSuccess = true;
let executedCommandCount = 0;
let pullRequestInfo;
logger.info('Starting remediation command execution', {
requestId,
sessionId: session.sessionId,
commandCount: finalAnalysis.remediation.actions.length,
});
// Execute each remediation action
for (let i = 0; i < finalAnalysis.remediation.actions.length; i++) {
const action = finalAnalysis.remediation.actions[i];
const actionId = `action_${i + 1}`;
try {
// PRD #408: Handle gitSource actions — create PR instead of kubectl
if (action.gitSource && !action.command) {
logger.info('Processing gitSource remediation action', {
requestId,
sessionId: session.sessionId,
actionId,
repoURL: action.gitSource.repoURL,
repoPath: action.gitSource.repoPath,
});
if (!action.gitSource.repoPath) {
results.push({
action: `${actionId}: ${action.description} (failed: missing repoPath)`,
success: false,
output: 'Git-based remediation requires repoPath from investigation phase',
timestamp: new Date(),
});
overallSuccess = false;
continue;
}
const prInput = {
repoPath: action.gitSource.repoPath,
files: action.gitSource.files.map((f) => ({
path: f.path,
content: f.content,
})),
title: `fix: ${action.description}`,
body: `## Remediation\n\n${action.rationale}\n\n**Risk Level:** ${action.risk}`,
branchName: `remediate/${session.sessionId.slice(0, 12)}-${Date.now()}`,
baseBranch: action.gitSource.branch || 'main',
};
const prExecutor = (0, internal_tools_1.createInternalToolExecutor)(session.sessionId);
const prResult = (await prExecutor('git_create_pr', prInput));
if (prResult.success && 'prUrl' in prResult) {
const filesList = prResult.filesChanged && prResult.filesChanged.length > 0
? prResult.filesChanged.join(', ')
: 'none';
results.push({
action: `${actionId}: ${action.description} (PR created)`,
success: true,
output: `PR #${prResult.prNumber}: ${prResult.prUrl}\nBranch: ${prResult.branch}\nFiles changed: ${filesList}`,
timestamp: new Date(),
});
pullRequestInfo = {
url: prResult.prUrl,
number: prResult.prNumber,
branch: prResult.branch,
baseBranch: prResult.baseBranch,
filesChanged: prResult.filesChanged,
};
}
else if (prResult.success && 'error' in prResult) {
const filesList = prResult.filesChanged && prResult.filesChanged.length > 0
? prResult.filesChanged.join(', ')
: 'none';
results.push({
action: `${actionId}: ${action.description} (branch pushed, manual PR needed)`,
success: true,
output: `Branch: ${prResult.branch}\nFiles changed: ${filesList}\nNote: ${prResult.error}`,
timestamp: new Date(),
});
}
else {
overallSuccess = false;
results.push({
action: `${actionId}: ${action.description} (failed)`,
success: false,
output: prResult.error,
timestamp: new Date(),
});
}
continue;
}
logger.info('Executing remediation action', {
requestId,
sessionId: session.sessionId,
actionId,
command: action.command,
});
// PRD #359: Execute the command via unified plugin registry
// Clean up escape sequences that some AI models incorrectly add to JSON parameters
let fullCommand = action.command || '';
fullCommand = fullCommand.replace(/\\"/g, '"');
const response = await (0, plugin_registry_1.invokePluginTool)('agentic-tools', 'shell_exec', {
command: fullCommand,
});
if (!response.success) {
throw new Error(response.error?.message || 'Command execution failed');
}
// Check for nested error - plugin wraps command errors in { success: false, error: "..." }
if (typeof response.result === 'object' && response.result !== null) {
const result = response.result;
if (result.success === false) {
throw new Error(result.error || result.message || 'Command execution failed');
}
}
// Extract only the data field - never pass JSON wrapper
let output;
if (typeof response.result === 'object' && response.result !== null) {
const result = response.result;
if (result.data !== undefined) {
output = String(result.data);
}
else if (typeof result === 'string') {
output = result;
}
else {
throw new Error('Plugin returned unexpected response format - missing data field');
}
}
else {
output = String(response.result || '');
}
executedCommandCount++;
results.push({
action: `${actionId}: ${action.description}`,
success: true,
output: output,
timestamp: new Date(),
});
logger.info('Remediation action succeeded', {
requestId,
sessionId: session.sessionId,
actionId,
});
}
catch (error) {
const errorMessage = error instanceof Error ? error.message : 'Unknown error';
overallSuccess = false;
results.push({
action: `${actionId}: ${action.description}`,
success: false,
error: errorMessage,
timestamp: new Date(),
});
logger.error('Remediation action failed', error, {
requestId,
sessionId: session.sessionId,
actionId,
command: action.command,
});
}
}
// Run automatic post-execution validation if commands were executed and all succeeded
let validationResult = null;
if (overallSuccess && executedCommandCount > 0 && finalAnalysis.validationIntent) {
const validationIntent = finalAnalysis.validationIntent;
try {
// In automatic mode, wait for Kubernetes to apply changes before validating
// This gives time for deployments to roll out new pods, operators to reconcile, etc.
// Manual mode skips this delay since the user can verify interactively
if (session.data.mode === 'automatic') {
logger.info('Waiting for Kubernetes to apply changes before validation', {
requestId,
sessionId: session.sessionId,
delayMs: 30000,
});
await new Promise(resolve => setTimeout(resolve, 30000));
}
logger.info('Running post-execution validation', {
requestId,
sessionId: session.sessionId,
validationIntent: validationIntent,
});
// Run validation by calling main function recursively with validation intent
// Include original issue context so the AI understands what was fixed
const executedCommands = results.map(r => r.action);
const validationIssue = `POST-REMEDIATION VALIDATION
Original issue that was remediated: ${session.data.issue}
Commands that were executed to fix the issue:
${executedCommands.map((cmd, i) => `${i + 1}. ${cmd}`).join('\n')}
Validation task: ${validationIntent}
IMPORTANT: You MUST respond with the final JSON analysis format as specified in your instructions. Verify the remediation was successful and return your analysis as JSON with issueStatus set to "resolved" if fixed, or "active" if issues remain.`;
// Run validation via conductInvestigation() directly (not handleRemediateTool)
// to avoid creating a new session with mode:'manual' which always returns awaiting_user_approval
const validationSession = sessionManager.createSession({
toolName: 'remediate',
issue: validationIssue,
mode: session.data.mode,
status: 'investigating',
interaction_id: currentInteractionId || session.data.interaction_id,
});
const validationAiProvider = (0, ai_provider_factory_1.createAIProvider)();
const validationOutput = await conductInvestigation(validationSession, sessionManager, validationAiProvider, logger, requestId, true, // isValidation
currentInteractionId || session.data.interaction_id);
const validationResolved = validationOutput.status === 'success';
// If validation discovered remaining issues, enhance with execution context
if (!validationResolved) {
logger.info('Validation discovered remaining issues, enhancing response with execution context', {
requestId,
sessionId: session.sessionId,
validationStatus: validationOutput.status,
newIssueConfidence: validationOutput.analysis?.confidence,
});
const enhancedResponse = {
...validationOutput,
status: 'partially_resolved',
executed: true,
results: results,
executedCommands: results.map(r => r.action),
previousExecution: {
sessionId: session.sessionId,
summary: `Previously executed ${results.length} remediation actions`,
actions: finalAnalysis.remediation.actions,
},
validation: {
success: false,
summary: 'Validation found remaining issues after remediation',
},
pullRequest: pullRequestInfo,
};
return {
content: [
{
type: 'text',
text: JSON.stringify(enhancedResponse, null, 2),
},
],
};
}
// Validation confirmed issue is resolved - create success response
logger.info('Validation confirmed issue is resolved, creating success response', {
requestId,
sessionId: session.sessionId,
validationStatus: validationOutput.status,
});
const successResponse = {
status: 'success',
sessionId: session.sessionId,
executed: true,
results: results,
executedCommands: results.map(r => r.action),
analysis: validationOutput.analysis,
remediation: {
summary: `Successfully executed ${results.length} remediation actions. ${validationOutput.remediation?.summary || 'Issue resolved.'}`,
actions: finalAnalysis.remediation.actions,
risk: finalAnalysis.remediation.risk,
},
investigation: validationOutput.investigation,
validationIntent: validationOutput.validationIntent,
guidance: `✅ REMEDIATION COMPLETE: Issue has been successfully resolved through executed commands.`,
agentInstructions: `1. Show user that the issue has been successfully resolved\n2. Display the actual kubectl commands that were executed (from remediation.actions[].command field)\n3. Show execution results with success/failure status for each command\n4. Show the validation results confirming the fix worked\n5. No further action required`,
message: `Issue successfully resolved. Executed ${results.length} remediation actions and validated the fix.`,
validation: {
success: true,
summary: 'Validation confirmed issue resolution',
},
pullRequest: pullRequestInfo,
};
const content = [
{
type: 'text',
text: JSON.stringify(successResponse, null, 2),
},
];
return { content };
}
catch (error) {
logger.warn('Post-execution validation failed', {
requestId,
sessionId: session.sessionId,
error: error instanceof Error ? error.message : 'Unknown error',
});
validationResult = {
success: false,
error: error instanceof Error ? error.message : 'Validation failed',
summary: 'Validation could not be completed automatically',
};
}
}
// Update session with execution results
sessionManager.updateSession(session.sessionId, {
status: overallSuccess ? 'executed_successfully' : 'executed_with_errors',
executionResults: results,
});
(0, session_events_1.getSessionEventBus)().publish(session_events_1.SESSION_EVENTS.SESSION_UPDATED, {
sessionId: session.sessionId,
toolName: 'remediate',
status: overallSuccess ? 'executed_successfully' : 'executed_with_errors',
issue: session.data.issue,
timestamp: new Date().toISOString(),
});
const hasOnlyGitOps = executedCommandCount === 0 && pullRequestInfo !== undefined;
const prInfo = pullRequestInfo;
let nextSteps;
if (hasOnlyGitOps && prInfo) {
nextSteps = [
'Changes have been pushed to a Git branch for GitOps reconciliation:',
` PR: ${prInfo.url}`,
` Branch: ${prInfo.branch} → ${prInfo.baseBranch}`,
` Files changed: ${prInfo.filesChanged.join(', ')}`,
'',
'Next steps:',
' 1. Review and merge the PR in your Git repository',
' 2. Wait for Argo CD/Flux to sync the changes',
' 3. Verify the issue is resolved after reconciliation',
'',
`You can verify the fix by running: remediate("Verify that ${finalAnalysis.analysis.rootCause.toLowerCase()} has been resolved")`,
];
}
else if (overallSuccess) {
if (validationResult) {
nextSteps = [
'The following kubectl commands were executed to remediate the issue:',
...finalAnalysis.remediation.actions
.filter(a => a.command)
.map((action, index) => {
const resultIndex = finalAnalysis.remediation.actions.indexOf(action);
return ` ${index + 1}. ${action.command} ${results[resultIndex]?.success ? '✓' : '✗'}`;
}),
'Automatic validation has been completed - see validation results above',
'Monitor your cluster to ensure the issue remains resolved',
];
}
else {
nextSteps = [
'The following kubectl commands were executed to remediate the issue:',
...finalAnalysis.remediation.actions
.filter(a => a.command)
.map((action, index) => {
const resultIndex = finalAnalysis.remediation.actions.indexOf(action);
return ` ${index + 1}. ${action.command} ${results[resultIndex]?.success ? '✓' : '✗'}`;
}),
`You can verify the fix by running: remediate("Verify that ${finalAnalysis.analysis.rootCause.toLowerCase()} has been resolved")`,
'Monitor your cluster to ensure the issue is fully resolved',
];
}
}
else {
nextSteps = [
'The following kubectl commands were attempted:',
...finalAnalysis.remediation.actions
.filter(a => a.command)
.map((action, index) => {
const resultIndex = finalAnalysis.remediation.actions.indexOf(action);
return ` ${index + 1}. ${action.command} ${results[resultIndex]?.success ? '✓' : '✗'}`;
}),
'Some remediation commands failed - check the results above',
'Review the error messages and address any underlying issues',
'You may need to run additional commands or investigate further',
];
}
const response = {
status: overallSuccess ? 'success' : 'failed',
sessionId: session.sessionId,
executed: true,
results: results,
executedCommands: results.map(r => r.action),
message: overallSuccess
? hasOnlyGitOps
? `Successfully created PR for ${results.length} GitOps remediation action(s)`
: `Successfully executed ${results.length} remediation actions`
: `Executed ${results.length} actions with ${results.filter(r => !r.success).length} failures`,
validation: validationResult,
instructions: {
showExecutedCommands: !hasOnlyGitOps,
showActualKubectlCommands: !hasOnlyGitOps,
nextSteps,
},
investigation: finalAnalysis.investigation,
analysis: finalAnalysis.analysis,
remediation: finalAnalysis.remediation,
pullRequest: pullRequestInfo,
};
logger.info('Remediation execution completed', {
requestId,
sessionId: session.sessionId,
overallSuccess,
successfulActions: results.filter(r => r.success).length,
failedActions: results.filter(r => !r.success).length,
});
const content = [
{
type: 'text',
text: JSON.stringify(response, null, 2),
},
];
return { content };
}
/**
* Main tool handler for remediate tool
*
* PRD #343: All kubectl operations go through plugin.
* PRD #359: Uses unified plugin registry - no pluginManager parameter needed.
*/
async function handleRemediateTool(args) {
const requestId = `remediate_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
const logger = new error_handling_1.ConsoleLogger('RemediateTool');
try {
// Initialize session manager
const sessionManager = new generic_session_manager_1.GenericSessionManager('rem');
logger.debug('Session manager initialized', { requestId });
// Validate input
const validatedInput = validateRemediateInput(args);
// Handle choice execution if provided
if (validatedInput.executeChoice && validatedInput.sessionId) {
// PRD #392 Milestone 2: execution requires 'apply' verb
const identity = (0, request_context_1.getCurrentIdentity)();
const rbacResult = await (0, rbac_1.checkToolAccess)(identity, {
toolName: 'remediate',
verb: 'apply',
});
if (!rbacResult.allowed) {
return {
content: [
{
type: 'text',
text: JSON.stringify({
error: 'FORBIDDEN',
message: `Access denied: executing remediation commands requires 'apply' permission on 'remediate'. You can diagnose issues, but applying fixes requires additional authorization.`,
tool: 'remediate',
user: identity?.email,
}),
},
],
};
}
logger.info('Executing user choice from previous session', {
requestId,
choice: validatedInput.executeChoice,
sessionId: validatedInput.sessionId,
});
return await executeUserChoice(sessionManager, validatedInput.sessionId, validatedInput.executeChoice, logger, requestId, validatedInput.interaction_id);
}
// Validate that we have an issue for new investigations
if (!validatedInput.issue) {
throw error_handling_1.ErrorHandler.createError(error_handling_1.ErrorCategory.VALIDATION, error_handling_1.ErrorSeverity.HIGH, 'Issue description is required for new investigations', { operation: 'input_validation', component: 'RemediateTool' });
}
// Create initial session using session manager
// PRD #320: Include toolName for visualization prompt selection
const session = sessionManager.createSession({
toolName: 'remediate',
issue: validatedInput.issue,
mode: validatedInput.mode || 'manual',
interaction_id: validatedInput.interaction_id,
status: 'investigating',
});
(0, session_events_1.getSessionEventBus)().publish(session_events_1.SESSION_EVENTS.SESSION_CREATED, {
sessionId: session.sessionId,
toolName: 'remediate',
status: 'investigating',
issue: validatedInput.issue,
timestamp: session.createdAt,
});
logger.info('Investigation session created', {
requestId,
sessionId: session.sessionId,
});
// Initialize AI provider (will validate API key automatically)
const aiProvider = (0, ai_provider_factory_1.createAIProvider)();
// Conduct AI-driven investigation (detect if this is post-execution validation)
const isValidation = validatedInput.executedCommands &&
validatedInput.executedCommands.length > 0;
const finalAnalysis = await conductInvestigation(session, sessionManager, aiProvider, logger, requestId, isValidation, validatedInput.interaction_id);
logger.info('Remediation analysis completed', {
requestId,
sessionId: session.sessionId,
rootCause: finalAnalysis.analysis.rootCause,
actionCount: finalAnalysis.remediation.actions.length,
riskLevel: finalAnalysis.remediation.risk,
});
// For resolved/non-existent issues, return success immediately without execution decision
if (finalAnalysis.status === 'success') {
logger.info('Issue resolved/non-existent - returning success without execution decision', {
requestId,
sessionId: session.sessionId,
status: finalAnalysis.status,
});
// Generate visualization URL for resolved issues
const visualizationUrl = (0, visualization_1.getVisualizationUrl)(session.sessionId);
// Build response with visualization URL in JSON
const content = [
{
type: 'text',
text: JSON.stringify({
...finalAnalysis,
...(visualizationUrl ? { visualizationUrl } : {}),
}, null, 2),
},
];
// Add agent instruction block if visualization URL is present
const agentDisplayBlock = (0, index_1.buildAgentDisplayBlock)({ visualizationUrl });
if (agentDisplayBlock) {
content.push(agentDisplayBlock);
}
return { content };
}
// Make execution decision based on mode and thresholds
const executionDecision = makeExecutionDecision(validatedInput.mode || 'manual', finalAnalysis