Golden Signals Testing: How to Automate Latency, Traffic, Errors, and Saturation Checks

Golden Signals Testing: How to Automate Latency, Traffic, Errors, and Saturation Checks

Google's Site Reliability Engineering book defines four golden signals that, if measured, give you sufficient visibility into any service: latency, traffic, errors, and saturation. Most teams instrument these signals for dashboards. Far fewer teams test them — verifying that signals are within bounds, that alerting works correctly, and that degradation in any signal gets caught before users notice.

This guide shows how to make the four golden signals testable, not just observable.

The Four Golden Signals

Latency — How long it takes to service a request. Distinguish successful requests from failed ones — failing fast doesn't count as low latency.

Traffic — How much demand is being placed on the system. Requests per second, messages per second, transactions per minute depending on your domain.

Errors — The rate of requests that fail. Explicit failures (HTTP 5xx) and implicit failures (wrong content, timeouts, correct HTTP code but broken response).

Saturation — How "full" the service is. CPU utilization, memory usage, queue depth, connection pool exhaustion.

Instrumenting the Signals

Before you can test signals, you need to instrument them:

# instrumentation.py
from prometheus_client import (
    Histogram, Counter, Gauge, 
    CollectorRegistry, start_http_server
)
import time
import functools

# Latency
REQUEST_LATENCY = Histogram(
    'http_request_duration_seconds',
    'HTTP request latency',
    ['method', 'endpoint', 'status_code'],
    buckets=[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
)

# Traffic
REQUEST_RATE = Counter(
    'http_requests_total',
    'Total HTTP requests',
    ['method', 'endpoint', 'status_code'],
)

# Errors (subset of traffic)
REQUEST_ERRORS = Counter(
    'http_request_errors_total',
    'HTTP request errors',
    ['method', 'endpoint', 'error_type'],  # error_type: timeout, 5xx, invalid_response
)

# Saturation
ACTIVE_CONNECTIONS = Gauge('http_active_connections', 'Active HTTP connections')
DB_POOL_USED = Gauge('db_pool_connections_used', 'Database pool connections in use')
DB_POOL_SIZE = Gauge('db_pool_size', 'Database pool total size')
QUEUE_DEPTH = Gauge('message_queue_depth', 'Message queue depth', ['queue_name'])
CPU_UTILIZATION = Gauge('process_cpu_usage_ratio', 'Process CPU utilization')

def track_request(endpoint: str):
    """Decorator to track all four golden signals."""
    def decorator(func):
        @functools.wraps(func)
        async def wrapper(*args, **kwargs):
            ACTIVE_CONNECTIONS.inc()
            start = time.perf_counter()
            status_code = "unknown"
            error_type = None
            
            try:
                result = await func(*args, **kwargs)
                status_code = str(result.status_code)
                
                # Implicit error: successful HTTP code but wrong response
                if hasattr(result, 'json'):
                    body = result.json()
                    if body.get('error'):
                        error_type = 'implicit_error'
                        REQUEST_ERRORS.labels(
                            method='GET', endpoint=endpoint, error_type=error_type
                        ).inc()
                
                return result
            
            except Exception as e:
                status_code = "500"
                error_type = type(e).__name__
                REQUEST_ERRORS.labels(
                    method='GET', endpoint=endpoint, error_type=error_type
                ).inc()
                raise
            
            finally:
                duration = time.perf_counter() - start
                REQUEST_LATENCY.labels(
                    method='GET', endpoint=endpoint, status_code=status_code
                ).observe(duration)
                REQUEST_RATE.labels(
                    method='GET', endpoint=endpoint, status_code=status_code
                ).inc()
                ACTIVE_CONNECTIONS.dec()
        
        return wrapper
    return decorator

Testing Golden Signals in CI

For every service, write tests that validate signal behavior:

# tests/golden_signals/test_latency.py
import pytest
import asyncio
import time
import statistics
import aiohttp

BASE_URL = "http://localhost:8080"

@pytest.fixture(scope="session")
async def warm_up():
    """Send a few warmup requests before measuring latency."""
    async with aiohttp.ClientSession() as session:
        for _ in range(10):
            await session.get(f"{BASE_URL}/health")
    await asyncio.sleep(0.1)

@pytest.mark.asyncio
async def test_successful_request_latency(warm_up):
    """Successful requests must complete within latency budget."""
    latencies = []
    
    async with aiohttp.ClientSession() as session:
        for _ in range(100):
            start = time.perf_counter()
            async with session.get(f"{BASE_URL}/api/products") as resp:
                await resp.read()
                assert resp.status == 200
            latencies.append((time.perf_counter() - start) * 1000)
    
    latencies.sort()
    p50 = latencies[49]
    p95 = latencies[94]
    p99 = latencies[98]
    
    assert p50 <= 50, f"p50 latency {p50:.1f}ms exceeds 50ms"
    assert p95 <= 150, f"p95 latency {p95:.1f}ms exceeds 150ms"
    assert p99 <= 300, f"p99 latency {p99:.1f}ms exceeds 300ms"

@pytest.mark.asyncio
async def test_error_request_latency_bounded():
    """Failed requests should fail fast, not time out."""
    start = time.perf_counter()
    async with aiohttp.ClientSession() as session:
        async with session.get(f"{BASE_URL}/api/nonexistent") as resp:
            await resp.read()
            assert resp.status == 404
    latency_ms = (time.perf_counter() - start) * 1000
    
    assert latency_ms <= 100, \
        f"404 response took {latency_ms:.1f}ms — errors should fail fast"

@pytest.mark.asyncio
async def test_latency_under_concurrent_load():
    """Latency should not degrade significantly under concurrent requests."""
    single_request_latency = []
    concurrent_latencies = []
    
    async with aiohttp.ClientSession() as session:
        # Baseline: sequential requests
        for _ in range(10):
            start = time.perf_counter()
            async with session.get(f"{BASE_URL}/api/products") as resp:
                await resp.read()
            single_request_latency.append((time.perf_counter() - start) * 1000)
        
        # Under load: 20 concurrent requests
        async def single_req():
            start = time.perf_counter()
            async with session.get(f"{BASE_URL}/api/products") as resp:
                await resp.read()
            return (time.perf_counter() - start) * 1000
        
        tasks = [single_req() for _ in range(20)]
        concurrent_latencies = await asyncio.gather(*tasks)
    
    baseline_p50 = statistics.median(single_request_latency)
    concurrent_p99 = sorted(concurrent_latencies)[18]
    
    # p99 under load should be < 5× baseline p50
    assert concurrent_p99 <= baseline_p50 * 5, \
        f"Severe latency degradation under load: " \
        f"concurrent p99={concurrent_p99:.1f}ms vs baseline p50={baseline_p50:.1f}ms"
# tests/golden_signals/test_errors.py

@pytest.mark.asyncio
async def test_error_rate_nominal_traffic():
    """Error rate under normal traffic must be under 0.1%."""
    total = 0
    errors = 0
    
    async with aiohttp.ClientSession() as session:
        for _ in range(1000):
            async with session.get(f"{BASE_URL}/api/products") as resp:
                total += 1
                if resp.status >= 500:
                    errors += 1
    
    error_rate = errors / total
    assert error_rate <= 0.001, \
        f"Error rate {error_rate:.3%} exceeds 0.1% ({errors}/{total} errors)"

@pytest.mark.asyncio
async def test_error_responses_have_valid_format():
    """4xx/5xx responses must return structured error bodies."""
    async with aiohttp.ClientSession() as session:
        # Test 404
        async with session.get(f"{BASE_URL}/api/nonexistent") as resp:
            assert resp.status == 404
            body = await resp.json()
            assert "error" in body or "message" in body, \
                f"404 response missing error field: {body}"
        
        # Test 400 (bad input)
        async with session.post(
            f"{BASE_URL}/api/products",
            json={"invalid": "data"}
        ) as resp:
            assert resp.status == 400
            body = await resp.json()
            assert "error" in body or "message" in body

@pytest.mark.asyncio
async def test_no_implicit_errors():
    """Successful responses must contain valid data."""
    async with aiohttp.ClientSession() as session:
        async with session.get(f"{BASE_URL}/api/products") as resp:
            assert resp.status == 200
            body = await resp.json()
            
            # Response structure must be valid
            assert "items" in body, "Missing 'items' field"
            assert isinstance(body["items"], list), "'items' must be a list"
            assert "total" in body, "Missing 'total' field"
# tests/golden_signals/test_saturation.py
import psutil
import asyncio
import aiohttp

@pytest.mark.asyncio
async def test_cpu_saturation_under_load():
    """Service must not reach CPU saturation under normal load."""
    service_pid = get_service_pid()  # Your implementation
    process = psutil.Process(service_pid)
    
    cpu_samples = []
    
    async def sample_cpu():
        for _ in range(30):  # Sample every second for 30 seconds
            cpu_samples.append(process.cpu_percent(interval=1))
    
    async def generate_load():
        async with aiohttp.ClientSession() as session:
            tasks = [
                session.get(f"{BASE_URL}/api/products")
                for _ in range(500)
            ]
            await asyncio.gather(*tasks)
    
    await asyncio.gather(sample_cpu(), generate_load())
    
    avg_cpu = sum(cpu_samples) / len(cpu_samples)
    max_cpu = max(cpu_samples)
    
    assert avg_cpu <= 70, f"Average CPU usage {avg_cpu:.1f}% is too high under normal load"
    assert max_cpu <= 90, f"Peak CPU usage {max_cpu:.1f}% indicates saturation risk"

@pytest.mark.asyncio
async def test_connection_pool_not_exhausted():
    """Database connection pool should not reach saturation."""
    async with aiohttp.ClientSession() as session:
        # Query internal metrics endpoint
        async with session.get(f"{BASE_URL}/internal/metrics") as resp:
            metrics_text = await resp.text()
    
    # Parse pool metrics
    import re
    pool_used_match = re.search(r'db_pool_connections_used (\d+)', metrics_text)
    pool_size_match = re.search(r'db_pool_size (\d+)', metrics_text)
    
    if pool_used_match and pool_size_match:
        pool_used = int(pool_used_match.group(1))
        pool_size = int(pool_size_match.group(1))
        utilization = pool_used / pool_size
        
        assert utilization <= 0.80, \
            f"Connection pool at {utilization:.0%} utilization — saturation risk"

@pytest.mark.asyncio
async def test_queue_depth_not_backing_up():
    """Message queue depth should not grow under normal throughput."""
    import asyncio
    
    samples = []
    
    async with aiohttp.ClientSession() as session:
        for _ in range(5):
            async with session.get(f"{BASE_URL}/internal/metrics") as resp:
                text = await resp.text()
            
            import re
            match = re.search(r'message_queue_depth{queue_name="events"} (\d+)', text)
            if match:
                samples.append(int(match.group(1)))
            
            await asyncio.sleep(5)
    
    if len(samples) >= 2:
        # Queue should not grow continuously (indicates processing can't keep up)
        queue_growing = all(
            samples[i] < samples[i+1] for i in range(len(samples) - 1)
        )
        assert not queue_growing, \
            f"Queue depth growing continuously: {samples} — consumer not keeping up"

Traffic Signal Tests

# tests/golden_signals/test_traffic.py

def test_traffic_counter_increments():
    """Request counter must increment on every request."""
    import requests
    
    # Read counter before
    metrics_before = requests.get(f"{BASE_URL}/internal/metrics").text
    count_before = extract_metric(metrics_before, "http_requests_total")
    
    # Make 10 requests
    for _ in range(10):
        requests.get(f"{BASE_URL}/api/products")
    
    # Read counter after
    metrics_after = requests.get(f"{BASE_URL}/internal/metrics").text
    count_after = extract_metric(metrics_after, "http_requests_total")
    
    assert count_after >= count_before + 10, \
        f"Counter did not increment correctly: before={count_before}, after={count_after}"

def test_traffic_labels_correct():
    """Traffic counters must have correct labels for route differentiation."""
    import requests
    
    # Hit different endpoints
    requests.get(f"{BASE_URL}/api/products")
    requests.get(f"{BASE_URL}/api/users")
    requests.post(f"{BASE_URL}/api/orders", json={"product_id": "test"})
    
    metrics = requests.get(f"{BASE_URL}/internal/metrics").text
    
    # Each endpoint should have its own counter
    assert 'endpoint="/api/products"' in metrics
    assert 'endpoint="/api/users"' in metrics
    assert 'endpoint="/api/orders"' in metrics
    assert 'method="POST"' in metrics

Running Golden Signal Tests in CI

# .github/workflows/golden-signals.yml
name: Golden Signals Tests

on:
  push:
    branches: [main]
  pull_request:

jobs:
  golden-signals:
    runs-on: ubuntu-latest
    
    steps:
      - uses: actions/checkout@v4
      
      - name: Start service
        run: docker-compose up -d api
      
      - name: Wait for service health
        run: |
          timeout 60 bash -c \
            'until curl -sf http://localhost:8080/health; do sleep 2; done'
      
      - name: Run golden signal tests
        run: |
          pip install pytest pytest-asyncio aiohttp psutil
          pytest tests/golden_signals/ -v --tb=short
      
      - name: Stop service
        if: always()
        run: docker-compose down

Continuous Golden Signal Monitoring

In production, run golden signal checks on a schedule:

# monitoring/golden_signal_health_check.py
"""
Scheduled health check that tests all four golden signals.
Run every 5 minutes via cron or HelpMeTest health checks.
"""

import requests
import sys

def check_golden_signals(service_url: str, metrics_url: str) -> bool:
    """Returns True if all signals are healthy."""
    
    all_ok = True
    
    # 1. Latency check
    import time
    latencies = []
    for _ in range(10):
        start = time.perf_counter()
        resp = requests.get(f"{service_url}/api/products", timeout=5)
        latency_ms = (time.perf_counter() - start) * 1000
        if resp.status_code == 200:
            latencies.append(latency_ms)
    
    if latencies:
        p95 = sorted(latencies)[int(len(latencies) * 0.95)]
        if p95 > 500:
            print(f"⚠️  LATENCY: p95={p95:.0f}ms exceeds 500ms")
            all_ok = False
        else:
            print(f"✓ LATENCY: p95={p95:.0f}ms")
    
    # 2. Error rate check (from metrics)
    metrics = requests.get(metrics_url).text
    # (parse error rate from Prometheus metrics)
    
    # 3. Traffic check (is service receiving traffic?)
    # (parse request rate from metrics)
    
    # 4. Saturation check
    # (parse CPU, memory, connection pool from metrics)
    
    return all_ok

if __name__ == "__main__":
    ok = check_golden_signals(
        service_url="https://api.example.com",
        metrics_url="https://api.example.com/internal/metrics",
    )
    sys.exit(0 if ok else 1)

Summary

The four golden signals are testable, not just observable:

  1. Latency — test p50, p95, p99 under normal and concurrent load; test that errors fail fast
  2. Traffic — test that counters increment correctly with right labels
  3. Errors — test error rate under load; test implicit errors (valid HTTP code, broken response)
  4. Saturation — test CPU, memory, connection pool, queue depth under expected load

Run these tests in CI on every deploy, and schedule them as continuous health checks in production. A service that passes golden signal tests on every deploy rarely surprises you in production.

Read more

Start now free