Platform Engineering QA: Testing Internal Developer Platforms End-to-End

        /\
       /E2E\          Golden path workflows
      /------\        (high priority — must pass)
     /  Integ  \      Component integration tests
    /------------\    (catalog↔ArgoCD, Crossplane↔provider)
   /   Contract   \   API contract tests between platform services
  /----------------\
 /    Smoke Tests   \ Platform API health checks
/--------------------\

1. Developer submits ServiceClaim CR to cluster
2. Crossplane creates: namespace, ServiceAccount, RoleBinding, NetworkPolicy
3. ArgoCD detects the new namespace and creates an Application for it
4. Backstage catalog ingests the new component entity
5. Developer can access the service URL within 10 minutes
6. Developer can deploy an image to the service within 15 minutes

1. Developer applies XDatabase claim
2. Crossplane provisions RDS instance + subnet group + security group
3. Connection string is written to a Kubernetes Secret in the developer's namespace
4. Developer can connect to the database within 20 minutes
5. Backstage shows the database as a dependency of the service

#!/usr/bin/env bash
# scripts/test-idempotency.sh

set -e

NAMESPACE="idempotency-test"

echo "Test: Creating namespace via platform API (first call)"
curl -sf -X POST "https://platform.internal/api/v1/namespaces" \
  -H "Authorization: Bearer $PLATFORM_TOKEN" \
  -H "Content-Type: application/json" \
  -d "{\"name\": \"$NAMESPACE\", \"team\": \"platform-test\", \"environment\": \"staging\"}"

echo "Test: Creating same namespace again (idempotency check)"
RESPONSE=$(curl -sf -X POST "https://platform.internal/api/v1/namespaces" \
  -H "Authorization: Bearer $PLATFORM_TOKEN" \
  -H "Content-Type: application/json" \
  -d "{\"name\": \"$NAMESPACE\", \"team\": \"platform-test\", \"environment\": \"staging\"}" \
  -w "\n%{http_code}")

HTTP_CODE=$(echo "$RESPONSE" | tail -1)
BODY=$(echo "$RESPONSE" | head -1)

# Should return 200 (already exists) or 201 (created), not 409 or 500
if [[ "$HTTP_CODE" == "409" || "$HTTP_CODE" == "500" ]]; then
  echo "FAIL: Idempotency check failed — got HTTP $HTTP_CODE"
  echo "Response: $BODY"
  exit 1
fi

echo "OK: Idempotency check passed (HTTP $HTTP_CODE)"

# Verify only one namespace was created
NS_COUNT=$(kubectl get namespace -l platform.internal/team=platform-test --no-headers | grep "$NAMESPACE" | wc -l)
if [ "$NS_COUNT" -ne 1 ]; then
  echo "FAIL: Expected 1 namespace, found $NS_COUNT"
  exit 1
fi

echo "OK: Exactly 1 namespace exists after 2 creation calls"

#!/usr/bin/env python3
# tests/golden-path-new-service.py

import subprocess
import time
import sys
import requests
import yaml

PLATFORM_API = "https://platform.internal/api/v1"
TOKEN = os.environ["PLATFORM_TOKEN"]
HEADERS = {"Authorization": f"Bearer {TOKEN}", "Content-Type": "application/json"}

def run(cmd, check=True):
    result = subprocess.run(cmd, shell=True, capture_output=True, text=True)
    if check and result.returncode != 0:
        print(f"FAIL: Command failed: {cmd}")
        print(result.stderr)
        sys.exit(1)
    return result.stdout.strip()

def wait_for(condition_fn, timeout_s, poll_interval_s=5, description="condition"):
    deadline = time.time() + timeout_s
    while time.time() < deadline:
        if condition_fn():
            return True
        print(f"  Waiting for {description}...")
        time.sleep(poll_interval_s)
    print(f"FAIL: Timed out waiting for {description} after {timeout_s}s")
    sys.exit(1)

SERVICE_NAME = f"golden-path-test-{int(time.time())}"

print(f"Golden Path Test: Create New Service '{SERVICE_NAME}'")

# Step 1: Submit service claim
print("\n[1/6] Submitting ServiceClaim...")
claim = {
    "apiVersion": "platform.internal/v1alpha1",
    "kind": "ServiceClaim",
    "metadata": {"name": SERVICE_NAME, "namespace": "platform-claims"},
    "spec": {
        "team": "platform-test",
        "environment": "staging",
        "language": "go",
        "template": "microservice",
    },
}
run(f"kubectl apply -f - <<< '{yaml.dump(claim)}'")

# Step 2: Wait for namespace to be created by Crossplane
print("[2/6] Waiting for namespace creation (max 3 minutes)...")
wait_for(
    lambda: run(f"kubectl get namespace {SERVICE_NAME} --no-headers 2>/dev/null", check=False) != "",
    timeout_s=180,
    description=f"namespace {SERVICE_NAME}",
)
print(f"  OK: Namespace {SERVICE_NAME} created")

# Step 3: Wait for ArgoCD application to appear
print("[3/6] Waiting for ArgoCD application (max 2 minutes)...")
wait_for(
    lambda: run(f"argocd app get {SERVICE_NAME} --grpc-web 2>/dev/null", check=False) != "",
    timeout_s=120,
    description=f"ArgoCD application {SERVICE_NAME}",
)
print(f"  OK: ArgoCD application {SERVICE_NAME} created")

# Step 4: Wait for Backstage catalog ingestion
print("[4/6] Waiting for Backstage catalog entry (max 5 minutes)...")
def backstage_entity_exists():
    try:
        resp = requests.get(
            f"https://backstage.internal/api/catalog/entities/by-name/component/default/{SERVICE_NAME}",
            headers={"Authorization": f"Bearer {TOKEN}"},
            timeout=10,
        )
        return resp.status_code == 200
    except Exception:
        return False

wait_for(backstage_entity_exists, timeout_s=300, description=f"Backstage entity {SERVICE_NAME}")
print(f"  OK: Backstage catalog entry created")

# Step 5: Verify RBAC (developer can deploy)
print("[5/6] Verifying developer RBAC...")
rbac_check = run(
    f"kubectl auth can-i create deployments --namespace {SERVICE_NAME} "
    f"--as system:serviceaccount:{SERVICE_NAME}:developer",
    check=False,
)
if "yes" not in rbac_check:
    print(f"FAIL: Developer ServiceAccount cannot create deployments in {SERVICE_NAME}")
    sys.exit(1)
print(f"  OK: Developer RBAC configured correctly")

# Step 6: Verify network policy
print("[6/6] Verifying network policies...")
netpol_count = run(
    f"kubectl get networkpolicies -n {SERVICE_NAME} --no-headers | wc -l"
).strip()
if int(netpol_count) < 1:
    print(f"FAIL: No NetworkPolicies created in {SERVICE_NAME}")
    sys.exit(1)
print(f"  OK: {netpol_count} NetworkPolicy/NetworkPolicies created")

# Cleanup
print(f"\nCleaning up...")
run(f"kubectl delete serviceclaim {SERVICE_NAME} -n platform-claims --ignore-not-found")

print(f"\nGolden Path Test PASSED — all 6 steps completed successfully")

#!/usr/bin/env bash
# scripts/smoke-tests.sh

PLATFORM_API="https://platform.internal/api/v1"
BACKSTAGE_API="https://backstage.internal/api"
ARGOCD_API="https://argocd.internal"

PASS=0
FAIL=0

smoke_test() {
  local name=$1
  local url=$2
  local expected_status=$3
  local auth_header=$4

  actual_status=$(curl -sf -o /dev/null -w "%{http_code}" \
    ${auth_header:+-H "Authorization: Bearer $auth_header"} \
    "$url" 2>/dev/null || echo "000")

  if [ "$actual_status" = "$expected_status" ]; then
    echo "OK: $name (HTTP $actual_status)"
    PASS=$((PASS + 1))
  else
    echo "FAIL: $name — expected HTTP $expected_status, got $actual_status"
    FAIL=$((FAIL + 1))
  fi
}

# Platform API
smoke_test "Platform API health" "$PLATFORM_API/health" "200"
smoke_test "Platform API auth required" "$PLATFORM_API/namespaces" "401"
smoke_test "Platform API authenticated" "$PLATFORM_API/namespaces" "200" "$PLATFORM_TOKEN"

# Backstage
smoke_test "Backstage health" "$BACKSTAGE_API/health" "200"
smoke_test "Backstage catalog entities" "$BACKSTAGE_API/catalog/entities?limit=1" "200" "$BACKSTAGE_TOKEN"
smoke_test "Backstage auth" "$BACKSTAGE_API/auth/providers" "200"

# ArgoCD
smoke_test "ArgoCD API server" "$ARGOCD_API/api/v1/applications?limit=1" "200"

# Crossplane
CROSSPLANE_HEALTHY=$(kubectl get provider -o jsonpath='{.items[*].status.conditions[?(@.type=="Healthy")].status}' 2>/dev/null)
if [[ "$CROSSPLANE_HEALTHY" == *"False"* ]]; then
  echo "FAIL: One or more Crossplane providers are unhealthy"
  kubectl get provider
  FAIL=$((FAIL + 1))
else
  echo "OK: All Crossplane providers are healthy"
  PASS=$((PASS + 1))
fi

echo ""
echo "Smoke tests: $PASS passed, $FAIL failed"
[ $FAIL -eq 0 ]

#!/usr/bin/env bash
# chaos/test-argocd-controller-failure.sh

echo "Chaos Test: ArgoCD application controller failure"

# Record current application count
INITIAL_APP_COUNT=$(argocd app list --grpc-web -o json | jq '. | length')
echo "Initial application count: $INITIAL_APP_COUNT"

# Kill the application controller
kubectl delete pod -n argocd -l app.kubernetes.io/name=argocd-application-controller

echo "Killed ArgoCD application controller — waiting for recovery..."

# Wait for the controller to restart
kubectl wait --for=condition=available \
  deployment/argocd-application-controller \
  -n argocd \
  --timeout=120s 2>/dev/null || \
kubectl wait --for=condition=ready \
  pod -l app.kubernetes.io/name=argocd-application-controller \
  -n argocd \
  --timeout=120s

echo "Controller restarted — verifying application state..."
sleep 30  # Give controller time to reconcile

# Verify no applications were lost
FINAL_APP_COUNT=$(argocd app list --grpc-web -o json | jq '. | length')
if [ "$FINAL_APP_COUNT" -ne "$INITIAL_APP_COUNT" ]; then
  echo "FAIL: Application count changed after controller restart ($INITIAL_APP_COUNT → $FINAL_APP_COUNT)"
  exit 1
fi

# Verify all applications are still healthy
UNHEALTHY=$(argocd app list --grpc-web -o json | \
  jq '[.[] | select(.status.health.status != "Healthy")] | length')
if [ "$UNHEALTHY" -gt 0 ]; then
  echo "FAIL: $UNHEALTHY applications became unhealthy after controller restart"
  argocd app list --grpc-web -o json | \
    jq '.[] | select(.status.health.status != "Healthy") | .metadata.name'
  exit 1
fi

echo "PASS: ArgoCD application controller recovered correctly"
echo "  - Application count preserved: $FINAL_APP_COUNT"
echo "  - All applications healthy"

#!/usr/bin/env bash
# chaos/test-crossplane-provider-failure.sh

echo "Chaos Test: Crossplane AWS provider pod failure"

# Record current managed resource count
INITIAL_MR_COUNT=$(kubectl get managed --all-namespaces --no-headers | wc -l)
echo "Initial managed resource count: $INITIAL_MR_COUNT"

# Kill the provider pod
kubectl delete pod -n crossplane-system \
  -l pkg.crossplane.io/revision=upbound-provider-aws

echo "Killed Crossplane AWS provider — waiting for recovery..."

# Wait for provider to recover
kubectl wait --for=condition=healthy provider/upbound-provider-aws \
  --timeout=180s

echo "Provider recovered — verifying managed resource state..."
sleep 60  # Give provider time to re-sync

# Verify no managed resources were deleted during outage
FINAL_MR_COUNT=$(kubectl get managed --all-namespaces --no-headers | wc -l)
if [ "$FINAL_MR_COUNT" -lt "$INITIAL_MR_COUNT" ]; then
  echo "FAIL: Managed resource count decreased during provider outage ($INITIAL_MR_COUNT → $FINAL_MR_COUNT)"
  exit 1
fi

# Verify no managed resources are in error state
ERROR_COUNT=$(kubectl get managed --all-namespaces -o json | \
  jq '[.items[] | select(.status.conditions[]? | select(.type == "Ready" and .status == "False"))] | length')
if [ "$ERROR_COUNT" -gt 0 ]; then
  echo "WARN: $ERROR_COUNT managed resources in error state (may be pre-existing)"
  kubectl get managed --all-namespaces -o json | \
    jq '.items[] | select(.status.conditions[]? | select(.type == "Ready" and .status == "False")) | .metadata.name'
fi

echo "PASS: Crossplane provider recovered without data loss"

# chaos/platform-chaos-experiments.yaml

# Experiment 1: API server latency injection
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: platform-api-latency
  namespace: chaos-testing
spec:
  action: delay
  mode: all
  selector:
    namespaces:
      - platform-system
    labelSelectors:
      app: platform-api
  delay:
    latency: "200ms"
    jitter: "50ms"
  duration: "5m"
---
# Experiment 2: Backstage database connection failure
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: backstage-db-partition
  namespace: chaos-testing
spec:
  action: partition
  mode: all
  selector:
    namespaces:
      - backstage
    labelSelectors:
      app: backstage
  direction: to
  target:
    mode: all
    selector:
      namespaces:
        - backstage
      labelSelectors:
        app: postgresql
  duration: "2m"

# Apply chaos experiment
kubectl apply -f chaos/platform-chaos-experiments.yaml

# Verify platform API degrades gracefully (circuit breaker, fallback cache)
sleep 30
smoke_test_result=$(./scripts/smoke-tests.sh 2>&1)
if echo "$smoke_test_result" | grep -q "FAIL: Platform API"; then
  echo "FAIL: Platform API did not degrade gracefully under 200ms latency"
  echo "$smoke_test_result"
  exit 1
fi
echo "OK: Platform API remained functional under network latency"

# Verify Backstage served cached responses during DB partition
BACKSTAGE_RESPONSE=$(curl -sf \
  -H "Authorization: Bearer $BACKSTAGE_TOKEN" \
  "https://backstage.internal/api/catalog/entities?limit=1")
if [ -z "$BACKSTAGE_RESPONSE" ]; then
  echo "FAIL: Backstage returned empty response during DB partition (no cache)"
  exit 1
fi
echo "OK: Backstage served cached catalog data during DB partition"

# Remove chaos
kubectl delete -f chaos/platform-chaos-experiments.yaml

# slos/platform-slos.yaml
slos:
  - name: service-provisioning-latency
    description: "New service provisioned within 10 minutes of claim submission"
    target: 0.95  # 95% of requests meet this SLO
    threshold_seconds: 600

  - name: deployment-pipeline-availability
    description: "ArgoCD API available 99.5% of the time"
    target: 0.995
    measurement_window: 30d

  - name: platform-api-latency-p99
    description: "Platform API p99 latency under 2 seconds"
    target: 0.99
    threshold_ms: 2000

  - name: catalog-freshness
    description: "Backstage catalog reflects Git state within 15 minutes"
    target: 0.99
    threshold_seconds: 900

#!/usr/bin/env python3
# tests/slo-test.py

import time
import statistics
import sys
import subprocess
import yaml

def test_service_provisioning_latency(n_samples=10, timeout_s=600):
    """
    Creates N service claims and measures provisioning time.
    Fails if fewer than 95% complete within 600 seconds.
    """
    print(f"SLO Test: Service provisioning latency (n={n_samples})")
    latencies = []
    failures = 0

    for i in range(n_samples):
        service_name = f"slo-test-{int(time.time())}-{i}"
        start = time.time()

        # Submit claim
        claim_yaml = f"""
apiVersion: platform.internal/v1alpha1
kind: ServiceClaim
metadata:
  name: {service_name}
  namespace: platform-claims
spec:
  team: slo-test
  environment: staging
  language: go
"""
        subprocess.run(
            f"kubectl apply -f - <<< '{claim_yaml}'",
            shell=True, check=True, capture_output=True
        )

        # Wait for namespace (proxy for provisioning complete)
        deadline = time.time() + timeout_s
        provisioned = False
        while time.time() < deadline:
            result = subprocess.run(
                f"kubectl get namespace {service_name} --no-headers 2>/dev/null",
                shell=True, capture_output=True, text=True
            )
            if result.returncode == 0 and result.stdout.strip():
                provisioned = True
                break
            time.sleep(10)

        elapsed = time.time() - start
        latencies.append(elapsed)

        if provisioned:
            print(f"  [{i+1}/{n_samples}] {service_name}: {elapsed:.1f}s")
        else:
            print(f"  [{i+1}/{n_samples}] {service_name}: TIMEOUT after {elapsed:.1f}s")
            failures += 1

        # Cleanup
        subprocess.run(
            f"kubectl delete serviceclaim {service_name} -n platform-claims --ignore-not-found",
            shell=True, capture_output=True
        )

    success_rate = (n_samples - failures) / n_samples
    p50 = statistics.median(latencies)
    p95 = sorted(latencies)[int(0.95 * len(latencies))]

    print(f"\nResults:")
    print(f"  Success rate: {success_rate:.1%} (target: 95%)")
    print(f"  p50 latency: {p50:.1f}s")
    print(f"  p95 latency: {p95:.1f}s (threshold: {timeout_s}s)")

    if success_rate < 0.95:
        print(f"FAIL: SLO not met — success rate {success_rate:.1%} < 95%")
        sys.exit(1)

    print("PASS: Service provisioning SLO met")

if __name__ == "__main__":
    test_service_provisioning_latency()

# .github/workflows/platform-qa.yaml
name: Platform QA

on:
  schedule:
    - cron: '0 */6 * * *'  # every 6 hours
  workflow_dispatch:        # manual trigger

jobs:
  smoke-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Run smoke tests
        env:
          PLATFORM_TOKEN: ${{ secrets.PLATFORM_TOKEN }}
          BACKSTAGE_TOKEN: ${{ secrets.BACKSTAGE_TOKEN }}
        run: ./scripts/smoke-tests.sh

  golden-path:
    runs-on: ubuntu-latest
    needs: smoke-tests
    steps:
      - uses: actions/checkout@v4
      - name: Run golden path test
        env:
          PLATFORM_TOKEN: ${{ secrets.PLATFORM_TOKEN }}
          KUBECONFIG: ${{ secrets.KUBECONFIG }}
        run: python3 tests/golden-path-new-service.py

  slo-tests:
    runs-on: ubuntu-latest
    needs: smoke-tests
    steps:
      - uses: actions/checkout@v4
      - name: Run SLO measurement
        env:
          PLATFORM_TOKEN: ${{ secrets.PLATFORM_TOKEN }}
        run: python3 tests/slo-test.py

  chaos:
    runs-on: ubuntu-latest
    needs: [smoke-tests, golden-path]
    if: github.event_name == 'schedule'  # only in scheduled runs
    steps:
      - uses: actions/checkout@v4
      - name: Run chaos tests
        run: |
          ./chaos/test-argocd-controller-failure.sh
          ./chaos/test-crossplane-provider-failure.sh

#!/usr/bin/env bash
# scripts/record-qa-results.sh

TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ)
RUN_ID=$GITHUB_RUN_ID

# Write results as JSON for ingestion into your observability platform
cat > /tmp/qa-results.json << EOF
{
  "timestamp": "$TIMESTAMP",
  "run_id": "$RUN_ID",
  "smoke_tests": {
    "passed": $SMOKE_PASS,
    "failed": $SMOKE_FAIL
  },
  "golden_path": {
    "passed": $GOLDEN_PASS,
    "duration_seconds": $GOLDEN_DURATION
  },
  "slo": {
    "service_provisioning_success_rate": $SLO_SUCCESS_RATE,
    "service_provisioning_p95_seconds": $SLO_P95
  }
}
EOF

# Post to your metrics endpoint
curl -sf -X POST "https://metrics.internal/api/platform-qa" \
  -H "Content-Type: application/json" \
  -d @/tmp/qa-results.json