unifi-pro-mcp/docs/deployment.md

MCP Server for UniFi Network and Protect systems with multi-device support

# Production Deployment Guide

Comprehensive guide for deploying UniFi PRO MCP in production environments with enterprise-grade reliability and security.

## 🎯 Table of Contents

- [Deployment Overview](#deployment-overview)
- [Infrastructure Requirements](#infrastructure-requirements)
- [Container Deployment](#container-deployment)
- [Cloud Deployment](#cloud-deployment)
- [High Availability Setup](#high-availability-setup)
- [Monitoring & Observability](#monitoring--observability)
- [Backup & Recovery](#backup--recovery)
- [Performance Optimization](#performance-optimization)
- [CI/CD Pipeline](#cicd-pipeline)
- [Maintenance Procedures](#maintenance-procedures)

## 🏗️ Deployment Overview

### Deployment Architectures

```mermaid
graph TB
    subgraph "Production Environment"
        LB[Load Balancer]
        subgraph "Application Tier"
            MCP1[UniFi MCP Server 1]
            MCP2[UniFi MCP Server 2]
            MCP3[UniFi MCP Server 3]
        end
        subgraph "Data Tier"
            REDIS[(Redis Cache)]
            LOGS[(Log Storage)]
        end
        subgraph "UniFi Infrastructure"
            UC1[UniFi Controller 1]
            UC2[UniFi Controller 2]
            UP1[UniFi Protect 1]
        end
    end
    
    CLAUDE[Claude Desktop] --> LB
    LB --> MCP1
    LB --> MCP2
    LB --> MCP3
    MCP1 --> REDIS
    MCP2 --> REDIS
    MCP3 --> REDIS
    MCP1 --> UC1
    MCP2 --> UC2
    MCP3 --> UP1
    MCP1 --> LOGS
    MCP2 --> LOGS
    MCP3 --> LOGS
```

### Deployment Options

| Option | Use Case | Complexity | Cost | Scalability |
|--------|----------|------------|------|-------------|
| **Single Instance** | Development, Small Office | Low | Low | Limited |
| **Container (Docker)** | Medium Business | Medium | Medium | Good |
| **Kubernetes** | Enterprise | High | Medium | Excellent |
| **Cloud Managed** | Global Enterprise | Medium | High | Excellent |

## 🖥️ Infrastructure Requirements

### Minimum Production Requirements

**Single Instance:**
```yaml
Compute:
  CPU: 2 cores (3.0 GHz+)
  RAM: 4 GB
  Storage: 20 GB SSD
  Network: 1 Gbps

Operating System:
  - Ubuntu 22.04 LTS (recommended)
  - RHEL 8/9
  - CentOS Stream 9
  - macOS 12+ (development only)
```

**High Availability Setup:**
```yaml
Load Balancer:
  CPU: 2 cores
  RAM: 2 GB
  Network: 2x 1 Gbps (redundant)

Application Servers (3x):
  CPU: 4 cores each
  RAM: 8 GB each  
  Storage: 50 GB SSD each
  Network: 1 Gbps each

Shared Storage:
  Type: Network-attached (NFS/iSCSI)
  Capacity: 200 GB
  IOPS: 1000+ (logs and cache)
```

### Network Requirements

```yaml
Firewall Rules:
  Inbound:
    - TCP 443 (HTTPS API)
    - TCP 22 (SSH management)
  
  Outbound:
    - TCP 8443 (UniFi Controller)
    - TCP 7443 (UniFi Protect)
    - TCP 443 (HTTPS/GitHub)
    - TCP 53 (DNS)
    - TCP 123 (NTP)

Network Connectivity:
  - Direct access to UniFi Controllers
  - Internet access for updates
  - DNS resolution capability
  - NTP synchronization
```

## 🐳 Container Deployment

### Docker Production Setup

**1. Create Production Dockerfile:**

```dockerfile
# Production Dockerfile
FROM node:18-alpine AS builder

# Install build dependencies
RUN apk add --no-cache python3 make g++

WORKDIR /app
COPY package*.json ./
RUN npm ci --only=production && npm cache clean --force

# Production stage
FROM node:18-alpine AS production

# Security: Create non-root user
RUN addgroup -g 1001 -S unifi-mcp && \
    adduser -S unifi-mcp -u 1001

# Install production dependencies
RUN apk add --no-cache \
    dumb-init \
    curl \
    && rm -rf /var/cache/apk/*

WORKDIR /app

# Copy application files
COPY --from=builder /app/node_modules ./node_modules
COPY --chown=unifi-mcp:unifi-mcp . .

# Build application
RUN npm run build && \
    chown -R unifi-mcp:unifi-mcp /app

# Switch to non-root user
USER unifi-mcp

# Health check
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
    CMD curl -f http://localhost:3000/health || exit 1

# Security: Read-only root filesystem
VOLUME ["/tmp", "/app/logs"]

# Use init system for proper signal handling
ENTRYPOINT ["/usr/bin/dumb-init", "--"]
CMD ["node", "dist/index.js"]

# Metadata
LABEL maintainer="your-email@company.com"
LABEL version="1.0.0"
LABEL description="UniFi PRO MCP Server"
```

**2. Docker Compose Production Configuration:**

```yaml
version: '3.8'

services:
  unifi-mcp:
    build:
      context: .
      dockerfile: Dockerfile.production
    image: unifi-mcp:latest
    container_name: unifi-mcp-prod
    restart: unless-stopped
    
    # Resource limits
    deploy:
      resources:
        limits:
          cpus: '2.0'
          memory: 4G
        reservations:
          cpus: '1.0'
          memory: 2G
    
    # Environment configuration
    environment:
      - NODE_ENV=production
      - LOG_LEVEL=info
      - REDIS_URL=redis://redis:6379
    
    # Secrets management
    secrets:
      - unifi_credentials
      - ssl_certificates
    
    # Network configuration
    networks:
      - unifi_network
    
    # Port mapping
    ports:
      - "3000:3000"
    
    # Volume mounts
    volumes:
      - logs_volume:/app/logs
      - cache_volume:/app/cache
      - /etc/ssl/certs:/etc/ssl/certs:ro
    
    # Security context
    security_opt:
      - no-new-privileges:true
    cap_drop:
      - ALL
    cap_add:
      - NET_BIND_SERVICE
    
    # Health check
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:3000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 30s

  # Redis for caching
  redis:
    image: redis:7-alpine
    container_name: redis-cache
    restart: unless-stopped
    command: redis-server --appendonly yes
    volumes:
      - redis_data:/data
    networks:
      - unifi_network
    deploy:
      resources:
        limits:
          memory: 1G

  # Nginx reverse proxy
  nginx:
    image: nginx:alpine
    container_name: nginx-proxy
    restart: unless-stopped
    ports:
      - "80:80"
      - "443:443"
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
      - ssl_certs:/etc/ssl/certs:ro
    networks:
      - unifi_network
    depends_on:
      - unifi-mcp

  # Log aggregation
  fluentd:
    image: fluent/fluentd:v1.16
    container_name: log-collector
    volumes:
      - ./fluentd.conf:/fluentd/etc/fluent.conf:ro
      - logs_volume:/app/logs:ro
    networks:
      - unifi_network

# Network configuration
networks:
  unifi_network:
    driver: bridge
    ipam:
      driver: default
      config:
        - subnet: 172.20.0.0/16

# Volume configuration
volumes:
  logs_volume:
    driver: local
  cache_volume:
    driver: local  
  redis_data:
    driver: local
  ssl_certs:
    external: true

# Secrets configuration
secrets:
  unifi_credentials:
    file: ./secrets/credentials.env
  ssl_certificates:
    file: ./secrets/ssl_bundle.pem
```

**3. Production Nginx Configuration:**

```nginx
# nginx.conf
events {
    worker_connections 1024;
}

http {
    upstream unifi_mcp {
        server unifi-mcp:3000 max_fails=3 fail_timeout=30s;
        keepalive 32;
    }

    # Rate limiting
    limit_req_zone $binary_remote_addr zone=api:10m rate=10r/s;

    # SSL configuration
    ssl_protocols TLSv1.2 TLSv1.3;
    ssl_ciphers ECDHE-RSA-AES256-GCM-SHA512:DHE-RSA-AES256-GCM-SHA512;
    ssl_prefer_server_ciphers off;
    ssl_session_cache shared:SSL:10m;
    ssl_session_timeout 10m;

    server {
        listen 443 ssl http2;
        server_name unifi-mcp.company.com;

        # SSL certificates
        ssl_certificate /etc/ssl/certs/unifi-mcp.crt;
        ssl_certificate_key /etc/ssl/certs/unifi-mcp.key;

        # Security headers
        add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
        add_header X-Frame-Options DENY always;
        add_header X-Content-Type-Options nosniff always;
        add_header X-XSS-Protection "1; mode=block" always;

        # Rate limiting
        limit_req zone=api burst=20 nodelay;

        # Proxy configuration
        location / {
            proxy_pass http://unifi_mcp;
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            
            # Timeouts
            proxy_connect_timeout 30s;
            proxy_send_timeout 30s;
            proxy_read_timeout 30s;
            proxy_buffering off;
        }

        # Health check endpoint
        location /health {
            proxy_pass http://unifi_mcp/health;
            access_log off;
        }
    }

    # HTTP redirect to HTTPS
    server {
        listen 80;
        server_name unifi-mcp.company.com;
        return 301 https://$server_name$request_uri;
    }
}
```

## ☁️ Cloud Deployment

### AWS Deployment

**1. ECS with Fargate:**

```yaml
# aws-ecs-task-definition.json
{
  "family": "unifi-mcp",
  "networkMode": "awsvpc",
  "requiresCompatibilities": ["FARGATE"],
  "cpu": "1024",
  "memory": "2048",
  "executionRoleArn": "arn:aws:iam::ACCOUNT:role/ecsTaskExecutionRole",
  "taskRoleArn": "arn:aws:iam::ACCOUNT:role/unifi-mcp-task-role",
  
  "containerDefinitions": [
    {
      "name": "unifi-mcp",
      "image": "your-account.dkr.ecr.region.amazonaws.com/unifi-mcp:latest",
      "essential": true,
      "portMappings": [
        {
          "containerPort": 3000,
          "protocol": "tcp"
        }
      ],
      "environment": [
        {
          "name": "NODE_ENV",
          "value": "production"
        },
        {
          "name": "REDIS_URL",
          "value": "redis://unifi-cache.aws.com:6379"
        }
      ],
      "secrets": [
        {
          "name": "UNIFI_PASSWORD",
          "valueFrom": "arn:aws:secretsmanager:region:account:secret:unifi-credentials"
        }
      ],
      "logConfiguration": {
        "logDriver": "awslogs",
        "options": {
          "awslogs-group": "/aws/ecs/unifi-mcp",
          "awslogs-region": "us-west-2",
          "awslogs-stream-prefix": "ecs"
        }
      },
      "healthCheck": {
        "command": ["CMD-SHELL", "curl -f http://localhost:3000/health || exit 1"],
        "interval": 30,
        "timeout": 5,
        "retries": 3,
        "startPeriod": 60
      }
    }
  ]
}
```

**2. Infrastructure as Code (Terraform):**

```hcl
# main.tf
provider "aws" {
  region = var.aws_region
}

# VPC and networking
module "vpc" {
  source = "terraform-aws-modules/vpc/aws"
  version = "~> 3.0"

  name = "unifi-mcp-vpc"
  cidr = "10.0.0.0/16"

  azs             = ["${var.aws_region}a", "${var.aws_region}b"]
  private_subnets = ["10.0.1.0/24", "10.0.2.0/24"]
  public_subnets  = ["10.0.101.0/24", "10.0.102.0/24"]

  enable_nat_gateway = true
  enable_vpn_gateway = true
  single_nat_gateway = false

  tags = {
    Environment = "production"
    Project     = "unifi-mcp"
  }
}

# Application Load Balancer
resource "aws_lb" "main" {
  name               = "unifi-mcp-alb"
  internal           = false
  load_balancer_type = "application"
  security_groups    = [aws_security_group.alb.id]
  subnets           = module.vpc.public_subnets

  enable_deletion_protection = true

  tags = {
    Environment = "production"
  }
}

# ECS Cluster
resource "aws_ecs_cluster" "main" {
  name = "unifi-mcp-cluster"

  capacity_providers = ["FARGATE", "FARGATE_SPOT"]

  default_capacity_provider_strategy {
    capacity_provider = "FARGATE"
    weight           = 1
  }

  setting {
    name  = "containerInsights"
    value = "enabled"
  }

  tags = {
    Environment = "production"
  }
}

# ECS Service
resource "aws_ecs_service" "main" {
  name            = "unifi-mcp-service"
  cluster         = aws_ecs_cluster.main.id
  task_definition = aws_ecs_task_definition.main.arn
  desired_count   = 3
  launch_type     = "FARGATE"

  network_configuration {
    security_groups = [aws_security_group.ecs_tasks.id]
    subnets         = module.vpc.private_subnets
  }

  load_balancer {
    target_group_arn = aws_lb_target_group.main.arn
    container_name   = "unifi-mcp"
    container_port   = 3000
  }

  depends_on = [aws_lb_listener.main]

  tags = {
    Environment = "production"
  }
}

# ElastiCache Redis
resource "aws_elasticache_subnet_group" "main" {
  name       = "unifi-mcp-cache-subnet"
  subnet_ids = module.vpc.private_subnets
}

resource "aws_elasticache_cluster" "redis" {
  cluster_id           = "unifi-mcp-cache"
  engine              = "redis"
  node_type           = "cache.t3.micro"
  num_cache_nodes     = 1
  parameter_group_name = "default.redis7"
  port                = 6379
  subnet_group_name   = aws_elasticache_subnet_group.main.name
  security_group_ids  = [aws_security_group.redis.id]

  tags = {
    Environment = "production"
  }
}
```

### Google Cloud Platform

**1. Cloud Run Deployment:**

```yaml
# cloudrun-service.yaml
apiVersion: serving.knative.dev/v1
kind: Service
metadata:
  name: unifi-mcp
  namespace: default
  annotations:
    run.googleapis.com/ingress: all
    run.googleapis.com/execution-environment: gen2
spec:
  template:
    metadata:
      annotations:
        autoscaling.knative.dev/maxScale: "10"
        autoscaling.knative.dev/minScale: "1"
        run.googleapis.com/cpu-throttling: "false"
        run.googleapis.com/execution-environment: gen2
    spec:
      containerConcurrency: 100
      timeoutSeconds: 300
      containers:
      - image: gcr.io/PROJECT_ID/unifi-mcp:latest
        ports:
        - containerPort: 3000
        env:
        - name: NODE_ENV
          value: production
        - name: REDIS_URL
          valueFrom:
            secretKeyRef:
              name: redis-connection
              key: url
        resources:
          limits:
            cpu: "2"
            memory: "4Gi"
        livenessProbe:
          httpGet:
            path: /health
            port: 3000
          initialDelaySeconds: 30
          periodSeconds: 30
        readinessProbe:
          httpGet:
            path: /ready
            port: 3000
          initialDelaySeconds: 10
          periodSeconds: 10
```

**2. Kubernetes Deployment:**

```yaml
# k8s-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: unifi-mcp
  namespace: production
spec:
  replicas: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
      maxSurge: 1
  selector:
    matchLabels:
      app: unifi-mcp
  template:
    metadata:
      labels:
        app: unifi-mcp
    spec:
      securityContext:
        runAsNonRoot: true
        runAsUser: 1001
        fsGroup: 1001
      containers:
      - name: unifi-mcp
        image: unifi-mcp:1.0.0
        ports:
        - containerPort: 3000
        env:
        - name: NODE_ENV
          value: production
        - name: REDIS_URL
          valueFrom:
            configMapKeyRef:
              name: app-config
              key: redis-url
        - name: UNIFI_PASSWORD
          valueFrom:
            secretKeyRef:
              name: unifi-credentials
              key: password
        resources:
          requests:
            memory: "1Gi"
            cpu: "500m"
          limits:
            memory: "4Gi"
            cpu: "2"
        livenessProbe:
          httpGet:
            path: /health
            port: 3000
          initialDelaySeconds: 30
          periodSeconds: 30
        readinessProbe:
          httpGet:
            path: /ready
            port: 3000
          initialDelaySeconds: 10
          periodSeconds: 10
        securityContext:
          allowPrivilegeEscalation: false
          readOnlyRootFilesystem: true
          runAsNonRoot: true
        volumeMounts:
        - name: tmp-volume
          mountPath: /tmp
        - name: cache-volume
          mountPath: /app/cache
      volumes:
      - name: tmp-volume
        emptyDir: {}
      - name: cache-volume
        emptyDir: {}
---
apiVersion: v1
kind: Service
metadata:
  name: unifi-mcp-service
  namespace: production
spec:
  selector:
    app: unifi-mcp
  ports:
  - port: 80
    targetPort: 3000
  type: ClusterIP
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: unifi-mcp-ingress
  namespace: production
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    cert-manager.io/cluster-issuer: "letsencrypt-prod"
spec:
  tls:
  - hosts:
    - unifi-mcp.company.com
    secretName: unifi-mcp-tls
  rules:
  - host: unifi-mcp.company.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: unifi-mcp-service
            port:
              number: 80
```

## 🔄 High Availability Setup

### Load Balancer Configuration

**HAProxy Configuration:**

```bash
# haproxy.cfg
global
    daemon
    maxconn 4096
    log stdout local0
    
defaults
    mode http
    timeout connect 5000ms
    timeout client 50000ms
    timeout server 50000ms
    option httplog
    option dontlognull
    option httpchk GET /health
    
frontend unifi_mcp_frontend
    bind *:80
    bind *:443 ssl crt /etc/ssl/certs/unifi-mcp.pem
    redirect scheme https if !{ ssl_fc }
    
    # Rate limiting
    stick-table type ip size 100k expire 30s store http_req_rate(10s)
    http-request track-sc0 src
    http-request reject if { sc_http_req_rate(0) gt 20 }
    
    default_backend unifi_mcp_backend
    
backend unifi_mcp_backend
    balance roundrobin
    option httpchk GET /health
    
    server mcp1 10.0.1.10:3000 check inter 5s rise 2 fall 3
    server mcp2 10.0.1.11:3000 check inter 5s rise 2 fall 3
    server mcp3 10.0.1.12:3000 check inter 5s rise 2 fall 3
```

### Database High Availability

**Redis Cluster Setup:**

```bash
# redis-cluster-setup.sh
#!/bin/bash

# Create Redis cluster configuration
cat > redis.conf << EOF
port 7000
cluster-enabled yes
cluster-config-file nodes.conf
cluster-node-timeout 5000
appendonly yes
appendfilename "appendonly.aof"
cluster-require-full-coverage no

# Security
requirepass ${REDIS_PASSWORD}
masterauth ${REDIS_PASSWORD}
EOF

# Start Redis nodes
for port in 7000 7001 7002 7003 7004 7005; do
    mkdir -p cluster/${port}
    cp redis.conf cluster/${port}/
    sed -i "s/7000/${port}/g" cluster/${port}/redis.conf
    redis-server cluster/${port}/redis.conf &
done

# Create cluster
redis-cli --cluster create \
    127.0.0.1:7000 127.0.0.1:7001 127.0.0.1:7002 \
    127.0.0.1:7003 127.0.0.1:7004 127.0.0.1:7005 \
    --cluster-replicas 1 \
    --cluster-yes
```

## 📊 Monitoring & Observability

### Prometheus Configuration

```yaml
# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - "unifi_mcp_rules.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets:
          - alertmanager:9093

scrape_configs:
  - job_name: 'unifi-mcp'
    static_configs:
      - targets: ['unifi-mcp:3000']
    metrics_path: '/metrics'
    scrape_interval: 30s
    
  - job_name: 'node-exporter'
    static_configs:
      - targets: ['node-exporter:9100']

  - job_name: 'redis'
    static_configs:
      - targets: ['redis-exporter:9121']
```

### Grafana Dashboard

```json
{
  "dashboard": {
    "title": "UniFi PRO MCP Monitoring",
    "panels": [
      {
        "title": "Request Rate",
        "type": "graph",
        "targets": [
          {
            "expr": "rate(http_requests_total[5m])",
            "legendFormat": "{{method}} {{status}}"
          }
        ]
      },
      {
        "title": "Response Time",
        "type": "graph",
        "targets": [
          {
            "expr": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))",
            "legendFormat": "95th percentile"
          }
        ]
      },
      {
        "title": "UniFi Controller Health",
        "type": "singlestat",
        "targets": [
          {
            "expr": "unifi_controller_available",
            "legendFormat": "Controller Status"
          }
        ]
      }
    ]
  }
}
```

### Application Metrics

```typescript
// metrics.ts - Custom application metrics
import { createPrometheusMetrics } from 'prom-client';

export class MetricsCollector {
  private requestCounter = new Counter({
    name: 'unifi_mcp_requests_total',
    help: 'Total number of requests',
    labelNames: ['method', 'endpoint', 'status']
  });

  private responseTime = new Histogram({
    name: 'unifi_mcp_response_duration_seconds',
    help: 'Response time in seconds',
    labelNames: ['method', 'endpoint'],
    buckets: [0.1, 0.5, 1, 2, 5]
  });

  private unifiControllerStatus = new Gauge({
    name: 'unifi_controller_available',
    help: 'UniFi Controller availability',
    labelNames: ['controller']
  });

  recordRequest(method: string, endpoint: string, status: number, duration: number) {
    this.requestCounter.inc({ method, endpoint, status: status.toString() });
    this.responseTime.observe({ method, endpoint }, duration);
  }

  updateControllerStatus(controller: string, available: boolean) {
    this.unifiControllerStatus.set({ controller }, available ? 1 : 0);
  }
}
```

## 💾 Backup & Recovery

### Backup Strategy

```bash
#!/bin/bash
# backup-script.sh

set -euo pipefail

BACKUP_DIR="/backups/unifi-mcp"
DATE=$(date +%Y%m%d_%H%M%S)
RETENTION_DAYS=30

# Create backup directory
mkdir -p "${BACKUP_DIR}/${DATE}"

# Backup configuration
echo "Backing up configuration..."
cp -r /app/config "${BACKUP_DIR}/${DATE}/"

# Backup Redis data
echo "Backing up Redis data..."
redis-cli --rdb "${BACKUP_DIR}/${DATE}/dump.rdb"

# Backup logs
echo "Backing up logs..."
tar -czf "${BACKUP_DIR}/${DATE}/logs.tar.gz" /app/logs/

# Backup application state
echo "Backing up application state..."
kubectl get configmaps,secrets -o yaml > "${BACKUP_DIR}/${DATE}/k8s-resources.yaml"

# Encrypt backup
echo "Encrypting backup..."
tar -czf - "${BACKUP_DIR}/${DATE}" | \
    gpg --cipher-algo AES256 --compress-algo 1 --symmetric \
    --output "${BACKUP_DIR}/${DATE}.tar.gz.gpg"

# Upload to cloud storage
echo "Uploading to S3..."
aws s3 cp "${BACKUP_DIR}/${DATE}.tar.gz.gpg" \
    "s3://unifi-mcp-backups/${DATE}.tar.gz.gpg"

# Cleanup old backups
echo "Cleaning up old backups..."
find "${BACKUP_DIR}" -type d -mtime +${RETENTION_DAYS} -exec rm -rf {} +
aws s3 ls "s3://unifi-mcp-backups/" | \
    while read -r line; do
        createDate=$(echo "$line" | awk '{print $1" "$2}')
        createDate=$(date -d"$createDate" +%s)
        olderThan=$(date -d"-${RETENTION_DAYS} days" +%s)
        if [[ $createDate -lt $olderThan ]]; then
            fileName=$(echo "$line" | awk '{print $4}')
            if [[ $fileName != "" ]]; then
                aws s3 rm "s3://unifi-mcp-backups/$fileName"
            fi
        fi
    done

echo "Backup completed successfully"
```

### Disaster Recovery Procedures

```bash
#!/bin/bash
# disaster-recovery.sh

set -euo pipefail

BACKUP_DATE=${1:-latest}
BACKUP_DIR="/recovery/unifi-mcp"

echo "Starting disaster recovery for backup: ${BACKUP_DATE}"

# Download backup from S3
if [ "${BACKUP_DATE}" = "latest" ]; then
    BACKUP_FILE=$(aws s3 ls s3://unifi-mcp-backups/ | sort | tail -n 1 | awk '{print $4}')
else
    BACKUP_FILE="${BACKUP_DATE}.tar.gz.gpg"
fi

echo "Downloading backup: ${BACKUP_FILE}"
aws s3 cp "s3://unifi-mcp-backups/${BACKUP_FILE}" "${BACKUP_DIR}/"

# Decrypt and extract
echo "Decrypting backup..."
gpg --decrypt "${BACKUP_DIR}/${BACKUP_FILE}" | tar -xzf - -C "${BACKUP_DIR}/"

# Restore configuration
echo "Restoring configuration..."
cp -r "${BACKUP_DIR}"/*/config/* /app/config/

# Restore Redis data
echo "Restoring Redis data..."
redis-cli --rdb "${BACKUP_DIR}"/*/dump.rdb

# Restore Kubernetes resources
echo "Restoring Kubernetes resources..."
kubectl apply -f "${BACKUP_DIR}"/*/k8s-resources.yaml

# Restart services
echo "Restarting services..."
kubectl rollout restart deployment/unifi-mcp

# Verify recovery
echo "Verifying recovery..."
sleep 30
if curl -f http://localhost:3000/health; then
    echo "Disaster recovery completed successfully"
else
    echo "Disaster recovery failed - services not responding"
    exit 1
fi
```

## ⚡ Performance Optimization

### Node.js Optimization

```typescript
// performance-config.ts
export const performanceConfig = {
  // Node.js runtime options
  nodeOptions: [
    '--max-old-space-size=4096',     // 4GB heap
    '--gc-interval=100',              // Frequent GC
    '--optimize-for-size',            // Memory optimization
    '--max-semi-space-size=128'       // Young generation size
  ],
  
  // Application-level optimizations
  cache: {
    enabled: true,
    ttl: 300000,                      // 5 minutes
    maxSize: 10000,                   // 10k entries
    compression: true
  },
  
  // Connection pooling
  connections: {
    maxSockets: 50,
    keepAlive: true,
    keepAliveMsecs: 300000,
    timeout: 30000
  },
  
  // Request optimization
  requests: {
    compression: true,
    timeout: 30000,
    retries: 3,
    retryDelay: 1000
  }
};
```

### Caching Strategy

```typescript
// cache-manager.ts
import { Redis } from 'ioredis';
import { LRUCache } from 'lru-cache';

export class CacheManager {
  private redis: Redis;
  private memoryCache: LRUCache<string, any>;

  constructor() {
    this.redis = new Redis(process.env.REDIS_URL);
    this.memoryCache = new LRUCache({
      max: 1000,
      ttl: 5 * 60 * 1000, // 5 minutes
    });
  }

  async get<T>(key: string): Promise<T | null> {
    // Try memory cache first (fastest)
    const memoryResult = this.memoryCache.get(key);
    if (memoryResult) {
      return memoryResult;
    }

    // Try Redis (network cache)
    const redisResult = await this.redis.get(key);
    if (redisResult) {
      const parsed = JSON.parse(redisResult);
      this.memoryCache.set(key, parsed);
      return parsed;
    }

    return null;
  }

  async set<T>(key: string, value: T, ttl: number = 300): Promise<void> {
    const serialized = JSON.stringify(value);
    
    // Store in both caches
    this.memoryCache.set(key, value);
    await this.redis.setex(key, ttl, serialized);
  }

  async invalidate(pattern: string): Promise<void> {
    // Clear memory cache
    for (const key of this.memoryCache.keys()) {
      if (key.match(pattern)) {
        this.memoryCache.delete(key);
      }
    }

    // Clear Redis cache
    const keys = await this.redis.keys(pattern);
    if (keys.length > 0) {
      await this.redis.del(...keys);
    }
  }
}
```

## 🔄 CI/CD Pipeline

### GitHub Actions Workflow

```yaml
# .github/workflows/production-deploy.yml
name: Production Deploy

on:
  push:
    branches: [main]
  release:
    types: [published]

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      
      - name: Setup Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '18'
          cache: 'npm'
      
      - name: Install dependencies
        run: npm ci
      
      - name: Run tests
        run: npm run test:ci
      
      - name: Security audit
        run: npm audit --audit-level high
      
      - name: Lint code
        run: npm run lint

  build:
    needs: test
    runs-on: ubuntu-latest
    outputs:
      image: ${{ steps.image.outputs.image }}
    steps:
      - uses: actions/checkout@v3
      
      - name: Setup Docker Buildx
        uses: docker/setup-buildx-action@v2
      
      - name: Login to Container Registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      
      - name: Extract metadata
        id: meta
        uses: docker/metadata-action@v4
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          tags: |
            type=ref,event=branch
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
      
      - name: Build and push
        uses: docker/build-push-action@v4
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
      
      - name: Output image
        id: image
        run: echo "image=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}" >> $GITHUB_OUTPUT

  deploy-staging:
    needs: build
    runs-on: ubuntu-latest
    environment: staging
    steps:
      - name: Deploy to staging
        run: |
          echo "Deploying ${{ needs.build.outputs.image }} to staging"
          # Add deployment commands

  deploy-production:
    needs: [build, deploy-staging]
    runs-on: ubuntu-latest
    environment: production
    if: github.ref == 'refs/heads/main'
    steps:
      - uses: actions/checkout@v3
      
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v2
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: us-west-2
      
      - name: Update ECS service
        run: |
          # Update task definition with new image
          aws ecs describe-task-definition \
            --task-definition unifi-mcp \
            --query taskDefinition > task-def.json
          
          # Update image in task definition
          jq '.containerDefinitions[0].image = "${{ needs.build.outputs.image }}"' \
            task-def.json > new-task-def.json
          
          # Register new task definition
          aws ecs register-task-definition \
            --cli-input-json file://new-task-def.json
          
          # Update service
          aws ecs update-service \
            --cluster unifi-mcp-cluster \
            --service unifi-mcp-service \
            --task-definition unifi-mcp
      
      - name: Wait for deployment
        run: |
          aws ecs wait services-stable \
            --cluster unifi-mcp-cluster \
            --services unifi-mcp-service
```

### Quality Gates

```bash
#!/bin/bash
# quality-gates.sh

set -euo pipefail

echo "Running quality gates..."

# Code coverage threshold
COVERAGE_THRESHOLD=85
COVERAGE=$(npm run test:coverage --silent | grep -oP 'All files.*?(\d+\.\d+)' | grep -oP '\d+\.\d+' | tail -1)

if (( $(echo "$COVERAGE < $COVERAGE_THRESHOLD" | bc -l) )); then
    echo "Code coverage $COVERAGE% is below threshold $COVERAGE_THRESHOLD%"
    exit 1
fi

# Security vulnerabilities
if ! npm audit --audit-level high --dry-run; then
    echo "High severity vulnerabilities found"
    exit 1
fi

# Performance tests
if ! npm run test:performance; then
    echo "Performance tests failed"
    exit 1
fi

# Integration tests
if ! npm run test:integration; then
    echo "Integration tests failed"
    exit 1
fi

echo "All quality gates passed"
```

## 🔧 Maintenance Procedures

### Scheduled Maintenance

```bash
#!/bin/bash
# scheduled-maintenance.sh

set -euo pipefail

MAINTENANCE_WINDOW="02:00-04:00"
LOG_FILE="/var/log/unifi-mcp-maintenance.log"

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

check_maintenance_window() {
    local current_time=$(date '+%H:%M')
    local start_time="${MAINTENANCE_WINDOW%-*}"
    local end_time="${MAINTENANCE_WINDOW#*-}"
    
    if [[ "$current_time" > "$start_time" && "$current_time" < "$end_time" ]]; then
        return 0
    else
        log "Current time $current_time is outside maintenance window $MAINTENANCE_WINDOW"
        return 1
    fi
}

perform_maintenance() {
    log "Starting scheduled maintenance"
    
    # Update system packages
    log "Updating system packages"
    apt update && apt upgrade -y
    
    # Clean up Docker images
    log "Cleaning up Docker images"
    docker system prune -f
    
    # Rotate logs
    log "Rotating logs"
    logrotate -f /etc/logrotate.d/unifi-mcp
    
    # Update application
    log "Checking for application updates"
    if docker pull "$IMAGE_NAME:latest"; then
        log "New version available, updating..."
        docker-compose down
        docker-compose up -d
        
        # Health check
        sleep 30
        if curl -f http://localhost:3000/health; then
            log "Update successful"
        else
            log "Update failed, rolling back"
            docker-compose down
            docker-compose up -d
        fi
    else
        log "No updates available"
    fi
    
    # Database maintenance
    log "Performing database maintenance"
    redis-cli BGREWRITEAOF
    
    # Certificate renewal
    log "Checking certificate expiration"
    if openssl x509 -checkend 2592000 -noout -in /etc/ssl/certs/unifi-mcp.crt; then
        log "Certificate is valid"
    else
        log "Certificate expires soon, renewing..."
        certbot renew --nginx
    fi
    
    log "Scheduled maintenance completed"
}

# Main execution
if check_maintenance_window; then
    perform_maintenance
else
    log "Skipping maintenance - outside window"
fi
```

### Health Monitoring

```typescript
// health-monitor.ts
export class HealthMonitor {
  private checks: Map<string, () => Promise<boolean>> = new Map();

  constructor() {
    this.registerChecks();
    this.startMonitoring();
  }

  private registerChecks() {
    this.checks.set('database', this.checkDatabase);
    this.checks.set('unifi-controller', this.checkUniFiController);
    this.checks.set('memory', this.checkMemoryUsage);
    this.checks.set('disk', this.checkDiskSpace);
    this.checks.set('network', this.checkNetworkConnectivity);
  }

  private async checkDatabase(): Promise<boolean> {
    try {
      await this.redis.ping();
      return true;
    } catch (error) {
      logger.error('Database health check failed', error);
      return false;
    }
  }

  private async checkUniFiController(): Promise<boolean> {
    try {
      const response = await axios.get(`${this.controllerUrl}/inform`, {
        timeout: 5000,
        validateStatus: () => true
      });
      return response.status < 500;
    } catch (error) {
      logger.error('UniFi Controller health check failed', error);
      return false;
    }
  }

  private async checkMemoryUsage(): Promise<boolean> {
    const usage = process.memoryUsage();
    const heapUsed = usage.heapUsed / 1024 / 1024; // MB
    const heapLimit = 4096; // 4GB limit
    
    return (heapUsed / heapLimit) < 0.9; // Alert at 90%
  }

  private async checkDiskSpace(): Promise<boolean> {
    const stats = await fs.promises.statfs('/');
    const freePercent = (stats.free / stats.blocks) * 100;
    
    return freePercent > 10; // Alert when <10% free
  }

  private startMonitoring() {
    setInterval(async () => {
      const results = new Map<string, boolean>();
      
      for (const [name, check] of this.checks) {
        try {
          results.set(name, await check());
        } catch (error) {
          results.set(name, false);
        }
      }
      
      this.processHealthResults(results);
    }, 30000); // Check every 30 seconds
  }

  private processHealthResults(results: Map<string, boolean>) {
    const failedChecks = Array.from(results.entries())
      .filter(([_, healthy]) => !healthy)
      .map(([name]) => name);
    
    if (failedChecks.length > 0) {
      this.alerting.sendAlert({
        severity: 'warning',
        message: `Health checks failed: ${failedChecks.join(', ')}`,
        timestamp: new Date().toISOString()
      });
    }
    
    // Update metrics
    for (const [name, healthy] of results) {
      this.metrics.healthCheck.set({ check: name }, healthy ? 1 : 0);
    }
  }
}
```

---

This production deployment guide provides comprehensive coverage for enterprise-grade UniFi PRO MCP deployments. The configurations and procedures ensure reliability, security, and scalability in production environments.