Kubernetes Failover Testing: Simulating Node Failures and Pod Disruptions

#!/bin/bash
# test_pod_restart.sh — Validate pods restart and traffic routes correctly

NAMESPACE="production"
DEPLOYMENT="api"
SERVICE_URL="http://api.internal/health"
RTO_SECONDS=30

echo "=== Pod Restart Failover Test ==="
START_TIME=$(date +%s)

# Record initial pod
INITIAL_PODS=$(kubectl get pods -n "$NAMESPACE" -l app="$DEPLOYMENT" \
  -o jsonpath='{.items[*].metadata.name}')
echo "Initial pods: $INITIAL_PODS"

# Start continuous health monitoring
HEALTH_LOG=$(mktemp)
monitor_health() {
  while true; do
    STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 "$SERVICE_URL" || echo "000")
    echo "$(date +%s) $STATUS" >> "$HEALTH_LOG"
    sleep 0.5
  done
}
monitor_health &
MONITOR_PID=$!

sleep 5

# Kill all pods (force delete simulates crash)
echo "Deleting all pods..."
KILL_TIME=$(date +%s)
kubectl delete pods -n "$NAMESPACE" -l app="$DEPLOYMENT" --grace-period=0 --force

# Wait for recovery
RECOVERED=false
while [ "$(date +%s)" -lt "$((KILL_TIME + RTO_SECONDS + 10))" ]; do
  STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 "$SERVICE_URL" || echo "000")
  if [ "$STATUS" = "200" ]; then
    RECOVERY_TIME=$(date +%s)
    ACTUAL_RTO=$((RECOVERY_TIME - KILL_TIME))
    echo "Service recovered in ${ACTUAL_RTO}s"
    RECOVERED=true
    break
  fi
  sleep 1
done

kill "$MONITOR_PID" 2>/dev/null || true

# Analyze downtime
TOTAL_REQUESTS=$(wc -l < "$HEALTH_LOG")
FAILED=$(grep -c " 000\| 50[0-9]" "$HEALTH_LOG" || echo 0)
echo "Downtime requests: ${FAILED}/${TOTAL_REQUESTS}"

# Verify new pods are running
NEW_PODS=$(kubectl get pods -n "$NAMESPACE" -l app="$DEPLOYMENT" \
  -o jsonpath='{.items[*].metadata.name}')
echo "New pods: $NEW_PODS"

# Validate pod count matches replica set
DESIRED=$(kubectl get deployment "$DEPLOYMENT" -n "$NAMESPACE" \
  -o jsonpath='{.spec.replicas}')
RUNNING=$(kubectl get pods -n "$NAMESPACE" -l app="$DEPLOYMENT" \
  --field-selector=status.phase=Running --no-headers | wc -l)

echo "Desired: $DESIRED | Running: $RUNNING"
[ "$RUNNING" = "$DESIRED" ] && echo "PASS: All replicas running" || echo "FAIL: Replica count mismatch"

if [ "$RECOVERED" = "false" ]; then
  echo "FAIL: Service did not recover within $((RTO_SECONDS + 10))s"
  exit 1
fi

[ "$ACTUAL_RTO" -le "$RTO_SECONDS" ] && echo "PASS: RTO met" || echo "WARN: RTO exceeded"

rm -f "$HEALTH_LOG"

#!/bin/bash
# test_probes.sh — Verify liveness and readiness probe behavior

NAMESPACE="production"
POD_NAME="api-7d9f4b8c6-xk9p2"

echo "=== Liveness Probe Test ==="

# Make the app fail its liveness check
kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \
  sh -c "touch /tmp/unhealthy"  # If your app checks for this file

# Wait for Kubernetes to detect failure and restart
echo "Waiting for restart..."
RESTART_COUNT_BEFORE=$(kubectl get pod -n "$NAMESPACE" "$POD_NAME" \
  -o jsonpath='{.status.containerStatuses[0].restartCount}')

sleep 60  # Longer than liveness probe failure threshold

RESTART_COUNT_AFTER=$(kubectl get pod -n "$NAMESPACE" "$POD_NAME" \
  -o jsonpath='{.status.containerStatuses[0].restartCount}' 2>/dev/null || echo "pod replaced")

echo "Restart count before: $RESTART_COUNT_BEFORE"
echo "Restart count after: $RESTART_COUNT_AFTER"

echo ""
echo "=== Readiness Probe Test ==="

# Make app fail readiness (but stay alive)
kubectl exec -n "$NAMESPACE" "$POD_NAME" -- \
  sh -c "touch /tmp/not_ready"

sleep 15

# Check pod is not receiving traffic (NotReady)
POD_READY=$(kubectl get pod -n "$NAMESPACE" "$POD_NAME" \
  -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')

echo "Pod ready status: $POD_READY"
[ "$POD_READY" = "False" ] && echo "PASS: Pod removed from load balancing" || echo "FAIL: Pod still receiving traffic"

# Verify other pods are healthy
kubectl get endpoints -n "$NAMESPACE" api-service

#!/bin/bash
# test_node_failure.sh

NAMESPACE="production"
TARGET_NODE="k8s-worker-3"
SERVICE_URL="http://api.internal/health"

echo "=== Node Failure Test ==="

# Check current pod distribution
echo "Pod distribution before failure:"
kubectl get pods -n "$NAMESPACE" -o wide | grep -v "RESTARTS"

# Get pods on target node
PODS_ON_NODE=$(kubectl get pods -n "$NAMESPACE" --field-selector spec.nodeName="$TARGET_NODE" \
  -o jsonpath='{.items[*].metadata.name}')
echo "Pods on $TARGET_NODE: $PODS_ON_NODE"

# Start health monitoring
HEALTH_LOG=$(mktemp)
monitor() {
  while true; do
    CODE=$(curl -s -o /dev/null -w "%{http_code}" --max-time 2 "$SERVICE_URL" || echo "000")
    echo "$(date +%s) $CODE" >> "$HEALTH_LOG"
    sleep 0.5
  done
}
monitor &
MONITOR_PID=$!
sleep 5

# Simulate node failure by cordoning and draining
echo "Simulating node failure: $TARGET_NODE"
FAILURE_TIME=$(date +%s)

# Option 1: Graceful (eviction with drain)
kubectl drain "$TARGET_NODE" --ignore-daemonsets --delete-emptydir-data --force

# Option 2: Abrupt (for hardware crash simulation)
# kubectl delete node "$TARGET_NODE"
# ssh "$TARGET_NODE" "sudo poweroff"  # actual machine shutdown

# Wait for pods to reschedule
sleep 30

echo "Pod distribution after failure:"
kubectl get pods -n "$NAMESPACE" -o wide

# Stop monitoring
kill "$MONITOR_PID" 2>/dev/null || true

# Analyze
TOTAL=$(wc -l < "$HEALTH_LOG")
FAILED=$(grep -c " 000\| 50[0-9]" "$HEALTH_LOG" || echo 0)
echo "Error rate during node failure: ${FAILED}/${TOTAL} ($(( FAILED * 100 / TOTAL ))%)"

# Verify all pods are running on remaining nodes
DESIRED=$(kubectl get deployment api -n "$NAMESPACE" -o jsonpath='{.spec.replicas}')
RUNNING=$(kubectl get pods -n "$NAMESPACE" -l app=api --field-selector=status.phase=Running \
  --no-headers | wc -l)
echo "Desired replicas: $DESIRED | Running: $RUNNING"

# Re-add node to cluster
kubectl uncordon "$TARGET_NODE"

rm -f "$HEALTH_LOG"

# pdb.yaml — Example PDB
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: api-pdb
  namespace: production
spec:
  minAvailable: 2  # At least 2 pods must be available during disruptions
  selector:
    matchLabels:
      app: api

# Test PDB enforcement
kubectl get pdb -n production

# Try to drain a node when it would violate PDB
kubectl drain k8s-worker-2 --ignore-daemonsets --delete-emptydir-data

# Kubernetes should block or slow this down if PDB would be violated
# Expected output:
# error when evicting pods/"api-7d9f4b8c6-xk9p2" -n "production" 
# (will retry after 5s): Cannot evict pod as it would violate the pod's disruption budget.

#!/bin/bash
# Simulate packet loss between services using tc (traffic control)

TARGET_POD="api-7d9f4b8c6-xk9p2"
NAMESPACE="production"
DURATION_SECONDS=60
PACKET_LOSS_PERCENT=50

echo "=== Network Partition Test: ${PACKET_LOSS_PERCENT}% packet loss for ${DURATION_SECONDS}s ==="

# Inject packet loss into target pod's network
kubectl exec -n "$NAMESPACE" "$TARGET_POD" -- \
  tc qdisc add dev eth0 root netem loss "${PACKET_LOSS_PERCENT}%"

echo "Packet loss injected. Testing service behavior..."

# Test application behavior during network degradation
for i in {1..20}; do
  STATUS=$(curl -s -o /dev/null -w "%{http_code}" --max-time 5 "http://api.internal/endpoint" || echo "000")
  echo "Request $i: $STATUS"
  sleep 3
done

# Remove packet loss after test duration
sleep "$DURATION_SECONDS"
kubectl exec -n "$NAMESPACE" "$TARGET_POD" -- \
  tc qdisc del dev eth0 root

echo "Network partition removed"

# network-chaos.yaml
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: api-network-delay
  namespace: production
spec:
  action: delay
  mode: one
  selector:
    namespaces:
      - production
    labelSelectors:
      app: api
  delay:
    latency: "100ms"
    correlation: "25"
    jitter: "50ms"
  duration: "5m"
  direction: both

# Apply and monitor
kubectl apply -f network-chaos.yaml

# Check application response times during chaos
hey -z 5m -c 10 http://api.internal/endpoint

# Cleanup
kubectl delete networkchaos api-network-delay -n production

#!/bin/bash
# test_az_failure.sh — Simulate AZ outage

FAILED_AZ="us-east-1b"
NAMESPACE="production"

echo "=== AZ Failure Test: Simulating loss of $FAILED_AZ ==="

# Get all nodes in the target AZ
NODES_IN_AZ=$(kubectl get nodes \
  -l "topology.kubernetes.io/zone=$FAILED_AZ" \
  -o jsonpath='{.items[*].metadata.name}')

echo "Nodes in $FAILED_AZ: $NODES_IN_AZ"

# Count pods that will be affected
AFFECTED_PODS=$(kubectl get pods -n "$NAMESPACE" -o wide | grep -E "$NODES_IN_AZ" | wc -l)
echo "Pods to be evacuated: $AFFECTED_PODS"

# Start load testing
hey -z 5m -c 50 http://api.internal/endpoint &
HEY_PID=$!

# Simulate AZ failure: cordon and drain all nodes in AZ
FAILURE_START=$(date +%s)
for NODE in $NODES_IN_AZ; do
  kubectl cordon "$NODE"
done

for NODE in $NODES_IN_AZ; do
  kubectl drain "$NODE" --ignore-daemonsets --delete-emptydir-data --force --timeout=120s
done

DRAIN_END=$(date +%s)
echo "Drain completed in $((DRAIN_END - FAILURE_START))s"

# Wait for pods to settle
sleep 30

# Verify all pods are running on remaining AZs
echo "Pod distribution after AZ failure:"
kubectl get pods -n "$NAMESPACE" -o wide

# Check that remaining nodes are across multiple AZs
kubectl get nodes -l "topology.kubernetes.io/zone" \
  --label-columns topology.kubernetes.io/zone

# Wait for load test to finish
wait "$HEY_PID"

echo "Restoring nodes..."
for NODE in $NODES_IN_AZ; do
  kubectl uncordon "$NODE"
done

# CronJob for weekly failover testing
apiVersion: batch/v1
kind: CronJob
metadata:
  name: failover-test-weekly
  namespace: test-automation
spec:
  schedule: "0 4 * * 0"  # Sundays at 4am
  jobTemplate:
    spec:
      template:
        spec:
          serviceAccountName: failover-test-sa
          containers:
            - name: failover-test
              image: company/failover-test:latest
              command: ["python", "failover_tests.py"]
              env:
                - name: TARGET_NAMESPACE
                  value: "staging"
                - name: NOTIFY_SLACK
                  valueFrom:
                    secretKeyRef:
                      name: slack-webhook
                      key: url
          restartPolicy: Never

# failover_tests.py — Orchestrate all failover tests
import asyncio
import json
import os
from datetime import datetime

async def run_all_tests() -> dict:
    results = {
        'run_at': datetime.utcnow().isoformat(),
        'tests': {}
    }
    
    # Pod restart test
    from tests.pod_restart import test_pod_restart
    results['tests']['pod_restart'] = await test_pod_restart(
        namespace=os.environ['TARGET_NAMESPACE'],
        deployment='api',
        rto_seconds=30
    )
    
    # Node failure test (one node)
    from tests.node_failure import test_node_failure
    results['tests']['node_failure'] = await test_node_failure(
        namespace=os.environ['TARGET_NAMESPACE'],
        rto_seconds=120
    )
    
    # Network chaos test
    from tests.network_chaos import test_network_latency
    results['tests']['network_chaos'] = await test_network_latency(
        namespace=os.environ['TARGET_NAMESPACE'],
        latency_ms=100,
        duration_seconds=60
    )
    
    # Calculate overall pass/fail
    results['passed'] = all(
        t.get('rto_met', False) 
        for t in results['tests'].values()
    )
    
    # Notify
    await notify_slack(results, os.environ['NOTIFY_SLACK'])
    
    return results

if __name__ == '__main__':
    results = asyncio.run(run_all_tests())
    print(json.dumps(results, indent=2))
    exit(0 if results['passed'] else 1)