DevOps Automation: Building Self-Healing Infrastructure

// Example: Auto-healing service
interface HealingPolicy {
  trigger: {
    metric: string;
    threshold: number;
    duration: string;
  };
  actions: [
    {
      type: "restart" | "scale" | "rollback";
      parameters: Record<string, any>;
    },
  ];
  cooldown: string;
}
 
const autoScalingPolicy: HealingPolicy = {
  trigger: {
    metric: "cpu_utilization",
    threshold: 80,
    duration: "5m",
  },
  actions: [
    {
      type: "scale",
      parameters: {
        increment: 2,
        max: 10,
      },
    },
  ],
  cooldown: "10m",
};

# Kubernetes liveness probe
livenessProbe:
  httpGet:
    path: /health
    port: 3000
  initialDelaySeconds: 30
  periodSeconds: 10
  failureThreshold: 3
# After 3 failed checks, Pod automatically restarts

class CircuitBreaker {
  constructor(threshold = 5, timeout = 60000) {
    this.failureCount = 0;
    this.threshold = threshold;
    this.timeout = timeout;
    this.state = "CLOSED"; // CLOSED, OPEN, HALF_OPEN
  }
 
  async call(func) {
    if (this.state === "OPEN") {
      throw new Error("Circuit breaker is OPEN");
    }
 
    try {
      const result = await func();
      this.onSuccess();
      return result;
    } catch (error) {
      this.onFailure();
      throw error;
    }
  }
 
  onSuccess() {
    this.failureCount = 0;
    this.state = "CLOSED";
  }
 
  onFailure() {
    this.failureCount++;
    if (this.failureCount >= this.threshold) {
      this.state = "OPEN";
      setTimeout(() => {
        this.state = "HALF_OPEN";
      }, this.timeout);
    }
  }
}

deployment:
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  progressDeadlineSeconds: 600
 
  # Automatic rollback on failure
  autoRollback:
    enabled: true
    triggers:
      - errorRate: 0.05 # 5% errors
      - latencyP95: 2000 # 2s p95 latency
      - availability: 0.99