remcode
Version:
Turn your AI assistant into a codebase expert. Intelligent code analysis, semantic search, and software engineering guidance through MCP integration.
318 lines (317 loc) • 12.7 kB
JavaScript
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.WorkflowOrchestrator = void 0;
exports.createWorkflowOrchestrator = createWorkflowOrchestrator;
const logger_1 = require("../utils/logger");
const monitor_1 = require("../workflows/monitor");
const service_1 = require("../notifications/service");
const state_manager_1 = require("../processing/state-manager");
const logger = (0, logger_1.getLogger)('WorkflowOrchestrator');
/**
* Orchestrates end-to-end workflow automation including monitoring,
* health checks, notifications, and auto-healing
*/
class WorkflowOrchestrator {
constructor(config, githubToken) {
this.isActive = false;
this.automationActions = [];
this.config = config;
this.monitor = new monitor_1.WorkflowMonitor(githubToken);
this.stateManager = new state_manager_1.StateManager();
if (config.notifications) {
this.notifications = new service_1.NotificationService(config.notifications);
}
logger.info('WorkflowOrchestrator initialized', {
repository: `${config.repository.owner}/${config.repository.repo}`,
hasNotifications: !!this.notifications,
automation: config.automation
});
}
/**
* Start the orchestration service
*/
async start() {
if (this.isActive) {
logger.warn('Orchestrator is already active');
return;
}
logger.info('Starting workflow orchestration');
this.isActive = true;
// Perform initial health check
await this.performHealthCheck();
// Schedule periodic health checks
if (this.config.monitoring.healthCheckInterval) {
const intervalMs = this.config.monitoring.healthCheckInterval * 60 * 1000;
this.healthCheckTimer = setInterval(() => {
this.performHealthCheck().catch(error => {
logger.error('Scheduled health check failed', error);
});
}, intervalMs);
logger.info(`Scheduled health checks every ${this.config.monitoring.healthCheckInterval} minutes`);
}
// Set up proactive monitoring if enabled
if (this.config.automation.enableProactiveMonitoring) {
await this.setupProactiveMonitoring();
}
logger.info('Workflow orchestration started successfully');
}
/**
* Stop the orchestration service
*/
async stop() {
if (!this.isActive) {
logger.warn('Orchestrator is not active');
return;
}
logger.info('Stopping workflow orchestration');
this.isActive = false;
if (this.healthCheckTimer) {
clearInterval(this.healthCheckTimer);
this.healthCheckTimer = undefined;
}
logger.info('Workflow orchestration stopped');
}
/**
* Get current orchestration status
*/
getStatus() {
const lastAction = this.automationActions[this.automationActions.length - 1];
return {
isActive: this.isActive,
lastHealthCheck: lastAction?.timestamp,
nextHealthCheck: this.healthCheckTimer && this.config.monitoring.healthCheckInterval ?
new Date(Date.now() + this.config.monitoring.healthCheckInterval * 60 * 1000).toISOString() :
undefined,
totalWorkflowsMonitored: this.automationActions.filter(a => a.action === 'health_check').length,
healthStatus: this.determineOverallHealth(),
automationActions: this.automationActions.slice(-10) // Last 10 actions
};
}
/**
* Perform comprehensive health check
*/
async performHealthCheck() {
try {
logger.info('Performing workflow health check');
const { owner, repo } = this.config.repository;
// Get workflow health status
const health = await this.monitor.monitorWorkflowHealth(owner, repo, {
maxFailureRate: 30,
maxConsecutiveFailures: 2,
alertOnSlowRuns: true,
maxDurationMinutes: 45
});
this.logAutomationAction('health_check', 'Scheduled health check completed', true);
// Take action based on health status
if (!health.healthy) {
await this.handleUnhealthyWorkflow(health);
}
// Send notifications if configured
if (this.notifications && !health.healthy) {
const notification = service_1.NotificationService.createHealthNotification(`${owner}/${repo}`, health.issues, health.recommendations);
await this.notifications.sendNotification(notification);
}
// Auto-healing if enabled
if (this.config.automation.enableAutoHealing && !health.healthy) {
await this.attemptAutoHealing(health);
}
}
catch (error) {
logger.error('Health check failed', error);
this.logAutomationAction('health_check', 'Health check failed', false);
}
}
/**
* Handle unhealthy workflow scenarios
*/
async handleUnhealthyWorkflow(health) {
const { owner, repo } = this.config.repository;
logger.warn('Unhealthy workflow detected', {
repository: `${owner}/${repo}`,
issues: health.issues,
recommendations: health.recommendations
});
// Check for consecutive failures that might need retry
if (this.config.monitoring.enableAutoRetry) {
const recentRuns = health.lastRuns || [];
const consecutiveFailures = this.countConsecutiveFailures(recentRuns);
if (consecutiveFailures >= 2 && consecutiveFailures <= (this.config.monitoring.maxRetryAttempts || 3)) {
await this.attemptAutoRetry(recentRuns[0]?.runId);
}
}
this.logAutomationAction('unhealthy_workflow_detected', `Issues: ${health.issues.join(', ')}`, true);
}
/**
* Attempt automatic retry of failed workflows
*/
async attemptAutoRetry(runId) {
if (!runId)
return;
try {
const { owner, repo } = this.config.repository;
logger.info(`Attempting auto-retry for run ${runId}`);
await this.monitor.retryWorkflowRun(owner, repo, runId, false);
this.logAutomationAction('auto_retry', `Retried workflow run ${runId}`, true);
// Wait for retry delay before next check
if (this.config.monitoring.retryDelayMinutes) {
await new Promise(resolve => setTimeout(resolve, this.config.monitoring.retryDelayMinutes * 60 * 1000));
}
}
catch (error) {
logger.error(`Auto-retry failed for run ${runId}`, error);
this.logAutomationAction('auto_retry', `Failed to retry run ${runId}`, false);
}
}
/**
* Attempt auto-healing actions
*/
async attemptAutoHealing(health) {
logger.info('Attempting auto-healing actions');
// Example auto-healing actions:
// 1. Clear workflow caches if performance issues detected
// 2. Restart workflows if they're stuck
// 3. Update workflow configurations based on recommendations
try {
// For now, we'll log the auto-healing attempt
// In a real implementation, you would implement specific healing actions
const healingActions = this.generateHealingActions(health);
for (const action of healingActions) {
logger.info(`Executing healing action: ${action.name}`);
// Execute the action
await action.execute();
this.logAutomationAction('auto_healing', action.name, true);
}
}
catch (error) {
logger.error('Auto-healing failed', error);
this.logAutomationAction('auto_healing', 'Auto-healing failed', false);
}
}
/**
* Generate healing actions based on health issues
*/
generateHealingActions(health) {
const actions = [];
// Example healing actions based on common issues
if (health.issues.some((issue) => issue.includes('High failure rate'))) {
actions.push({
name: 'Reset workflow environment',
execute: async () => {
// Simulate environment reset
await new Promise(resolve => setTimeout(resolve, 1000));
}
});
}
if (health.issues.some((issue) => issue.includes('consecutive failures'))) {
actions.push({
name: 'Clear workflow caches',
execute: async () => {
// Simulate cache clearing
await new Promise(resolve => setTimeout(resolve, 500));
}
});
}
return actions;
}
/**
* Set up proactive monitoring
*/
async setupProactiveMonitoring() {
logger.info('Setting up proactive monitoring');
// Monitor for trends that indicate potential issues
// This could include:
// - Gradually increasing failure rates
// - Performance degradation over time
// - Resource usage patterns
this.logAutomationAction('proactive_monitoring', 'Proactive monitoring enabled', true);
}
/**
* Count consecutive failures from recent runs
*/
countConsecutiveFailures(runs) {
let count = 0;
for (const run of runs) {
if (run.conclusion === 'failure') {
count++;
}
else {
break;
}
}
return count;
}
/**
* Determine overall health status
*/
determineOverallHealth() {
const recentActions = this.automationActions.slice(-5);
const healthChecks = recentActions.filter(a => a.action === 'health_check');
const failures = recentActions.filter(a => !a.success);
if (failures.length >= 3)
return 'critical';
if (failures.length >= 1)
return 'warning';
return 'healthy';
}
/**
* Log automation action
*/
logAutomationAction(action, reason, success) {
const entry = {
timestamp: new Date().toISOString(),
action,
reason,
success
};
this.automationActions.push(entry);
// Keep only last 100 actions to prevent memory issues
if (this.automationActions.length > 100) {
this.automationActions = this.automationActions.slice(-100);
}
logger.info('Automation action logged', entry);
}
/**
* Handle workflow completion notification
*/
async handleWorkflowCompletion(runId, status, duration, stats) {
if (!this.notifications)
return;
try {
const { owner, repo } = this.config.repository;
const notification = service_1.NotificationService.createWorkflowNotification(`${owner}/${repo}`, runId, status, duration, stats);
await this.notifications.sendNotification(notification);
this.logAutomationAction('notification_sent', `Workflow completion notification for run ${runId}`, true);
}
catch (error) {
logger.error('Failed to send workflow completion notification', error);
this.logAutomationAction('notification_failed', `Failed to notify for run ${runId}`, false);
}
}
}
exports.WorkflowOrchestrator = WorkflowOrchestrator;
/**
* Factory function to create orchestrator with common configurations
*/
function createWorkflowOrchestrator(repository, options = {}) {
const config = {
repository,
monitoring: {
healthCheckInterval: options.healthCheckMinutes || 30,
enableAutoRetry: options.enableAutoRetry ?? true,
maxRetryAttempts: 3,
retryDelayMinutes: 5
},
notifications: options.enableNotifications && options.slackWebhook ? {
slack: {
webhook: options.slackWebhook,
username: 'Remcode Orchestrator'
}
} : undefined,
automation: {
enableAutoHealing: options.enableAutoHealing ?? false,
enablePerformanceOptimization: true,
enableProactiveMonitoring: true
}
};
return new WorkflowOrchestrator(config, options.githubToken);
}