Golden Signals Testing: How to Automate Latency, Traffic, Errors, and Saturation Checks
Google's Site Reliability Engineering book defines four golden signals that, if measured, give you sufficient visibility into any service: latency, traffic, errors, and saturation. Most teams instrument these signals for dashboards. Far fewer teams test them — verifying that signals are within bounds, that alerting works correctly, and that degradation in any signal gets caught before users notice.
This guide shows how to make the four golden signals testable, not just observable.
The Four Golden Signals
Latency — How long it takes to service a request. Distinguish successful requests from failed ones — failing fast doesn't count as low latency.
Traffic — How much demand is being placed on the system. Requests per second, messages per second, transactions per minute depending on your domain.
Errors — The rate of requests that fail. Explicit failures (HTTP 5xx) and implicit failures (wrong content, timeouts, correct HTTP code but broken response).
Saturation — How "full" the service is. CPU utilization, memory usage, queue depth, connection pool exhaustion.
Instrumenting the Signals
Before you can test signals, you need to instrument them:
# instrumentation.py
from prometheus_client import (
Histogram, Counter, Gauge,
CollectorRegistry, start_http_server
)
import time
import functools
# Latency
REQUEST_LATENCY = Histogram(
'http_request_duration_seconds',
'HTTP request latency',
['method', 'endpoint', 'status_code'],
buckets=[0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10],
)
# Traffic
REQUEST_RATE = Counter(
'http_requests_total',
'Total HTTP requests',
['method', 'endpoint', 'status_code'],
)
# Errors (subset of traffic)
REQUEST_ERRORS = Counter(
'http_request_errors_total',
'HTTP request errors',
['method', 'endpoint', 'error_type'], # error_type: timeout, 5xx, invalid_response
)
# Saturation
ACTIVE_CONNECTIONS = Gauge('http_active_connections', 'Active HTTP connections')
DB_POOL_USED = Gauge('db_pool_connections_used', 'Database pool connections in use')
DB_POOL_SIZE = Gauge('db_pool_size', 'Database pool total size')
QUEUE_DEPTH = Gauge('message_queue_depth', 'Message queue depth', ['queue_name'])
CPU_UTILIZATION = Gauge('process_cpu_usage_ratio', 'Process CPU utilization')
def track_request(endpoint: str):
"""Decorator to track all four golden signals."""
def decorator(func):
@functools.wraps(func)
async def wrapper(*args, **kwargs):
ACTIVE_CONNECTIONS.inc()
start = time.perf_counter()
status_code = "unknown"
error_type = None
try:
result = await func(*args, **kwargs)
status_code = str(result.status_code)
# Implicit error: successful HTTP code but wrong response
if hasattr(result, 'json'):
body = result.json()
if body.get('error'):
error_type = 'implicit_error'
REQUEST_ERRORS.labels(
method='GET', endpoint=endpoint, error_type=error_type
).inc()
return result
except Exception as e:
status_code = "500"
error_type = type(e).__name__
REQUEST_ERRORS.labels(
method='GET', endpoint=endpoint, error_type=error_type
).inc()
raise
finally:
duration = time.perf_counter() - start
REQUEST_LATENCY.labels(
method='GET', endpoint=endpoint, status_code=status_code
).observe(duration)
REQUEST_RATE.labels(
method='GET', endpoint=endpoint, status_code=status_code
).inc()
ACTIVE_CONNECTIONS.dec()
return wrapper
return decoratorTesting Golden Signals in CI
For every service, write tests that validate signal behavior:
# tests/golden_signals/test_latency.py
import pytest
import asyncio
import time
import statistics
import aiohttp
BASE_URL = "http://localhost:8080"
@pytest.fixture(scope="session")
async def warm_up():
"""Send a few warmup requests before measuring latency."""
async with aiohttp.ClientSession() as session:
for _ in range(10):
await session.get(f"{BASE_URL}/health")
await asyncio.sleep(0.1)
@pytest.mark.asyncio
async def test_successful_request_latency(warm_up):
"""Successful requests must complete within latency budget."""
latencies = []
async with aiohttp.ClientSession() as session:
for _ in range(100):
start = time.perf_counter()
async with session.get(f"{BASE_URL}/api/products") as resp:
await resp.read()
assert resp.status == 200
latencies.append((time.perf_counter() - start) * 1000)
latencies.sort()
p50 = latencies[49]
p95 = latencies[94]
p99 = latencies[98]
assert p50 <= 50, f"p50 latency {p50:.1f}ms exceeds 50ms"
assert p95 <= 150, f"p95 latency {p95:.1f}ms exceeds 150ms"
assert p99 <= 300, f"p99 latency {p99:.1f}ms exceeds 300ms"
@pytest.mark.asyncio
async def test_error_request_latency_bounded():
"""Failed requests should fail fast, not time out."""
start = time.perf_counter()
async with aiohttp.ClientSession() as session:
async with session.get(f"{BASE_URL}/api/nonexistent") as resp:
await resp.read()
assert resp.status == 404
latency_ms = (time.perf_counter() - start) * 1000
assert latency_ms <= 100, \
f"404 response took {latency_ms:.1f}ms — errors should fail fast"
@pytest.mark.asyncio
async def test_latency_under_concurrent_load():
"""Latency should not degrade significantly under concurrent requests."""
single_request_latency = []
concurrent_latencies = []
async with aiohttp.ClientSession() as session:
# Baseline: sequential requests
for _ in range(10):
start = time.perf_counter()
async with session.get(f"{BASE_URL}/api/products") as resp:
await resp.read()
single_request_latency.append((time.perf_counter() - start) * 1000)
# Under load: 20 concurrent requests
async def single_req():
start = time.perf_counter()
async with session.get(f"{BASE_URL}/api/products") as resp:
await resp.read()
return (time.perf_counter() - start) * 1000
tasks = [single_req() for _ in range(20)]
concurrent_latencies = await asyncio.gather(*tasks)
baseline_p50 = statistics.median(single_request_latency)
concurrent_p99 = sorted(concurrent_latencies)[18]
# p99 under load should be < 5× baseline p50
assert concurrent_p99 <= baseline_p50 * 5, \
f"Severe latency degradation under load: " \
f"concurrent p99={concurrent_p99:.1f}ms vs baseline p50={baseline_p50:.1f}ms"# tests/golden_signals/test_errors.py
@pytest.mark.asyncio
async def test_error_rate_nominal_traffic():
"""Error rate under normal traffic must be under 0.1%."""
total = 0
errors = 0
async with aiohttp.ClientSession() as session:
for _ in range(1000):
async with session.get(f"{BASE_URL}/api/products") as resp:
total += 1
if resp.status >= 500:
errors += 1
error_rate = errors / total
assert error_rate <= 0.001, \
f"Error rate {error_rate:.3%} exceeds 0.1% ({errors}/{total} errors)"
@pytest.mark.asyncio
async def test_error_responses_have_valid_format():
"""4xx/5xx responses must return structured error bodies."""
async with aiohttp.ClientSession() as session:
# Test 404
async with session.get(f"{BASE_URL}/api/nonexistent") as resp:
assert resp.status == 404
body = await resp.json()
assert "error" in body or "message" in body, \
f"404 response missing error field: {body}"
# Test 400 (bad input)
async with session.post(
f"{BASE_URL}/api/products",
json={"invalid": "data"}
) as resp:
assert resp.status == 400
body = await resp.json()
assert "error" in body or "message" in body
@pytest.mark.asyncio
async def test_no_implicit_errors():
"""Successful responses must contain valid data."""
async with aiohttp.ClientSession() as session:
async with session.get(f"{BASE_URL}/api/products") as resp:
assert resp.status == 200
body = await resp.json()
# Response structure must be valid
assert "items" in body, "Missing 'items' field"
assert isinstance(body["items"], list), "'items' must be a list"
assert "total" in body, "Missing 'total' field"# tests/golden_signals/test_saturation.py
import psutil
import asyncio
import aiohttp
@pytest.mark.asyncio
async def test_cpu_saturation_under_load():
"""Service must not reach CPU saturation under normal load."""
service_pid = get_service_pid() # Your implementation
process = psutil.Process(service_pid)
cpu_samples = []
async def sample_cpu():
for _ in range(30): # Sample every second for 30 seconds
cpu_samples.append(process.cpu_percent(interval=1))
async def generate_load():
async with aiohttp.ClientSession() as session:
tasks = [
session.get(f"{BASE_URL}/api/products")
for _ in range(500)
]
await asyncio.gather(*tasks)
await asyncio.gather(sample_cpu(), generate_load())
avg_cpu = sum(cpu_samples) / len(cpu_samples)
max_cpu = max(cpu_samples)
assert avg_cpu <= 70, f"Average CPU usage {avg_cpu:.1f}% is too high under normal load"
assert max_cpu <= 90, f"Peak CPU usage {max_cpu:.1f}% indicates saturation risk"
@pytest.mark.asyncio
async def test_connection_pool_not_exhausted():
"""Database connection pool should not reach saturation."""
async with aiohttp.ClientSession() as session:
# Query internal metrics endpoint
async with session.get(f"{BASE_URL}/internal/metrics") as resp:
metrics_text = await resp.text()
# Parse pool metrics
import re
pool_used_match = re.search(r'db_pool_connections_used (\d+)', metrics_text)
pool_size_match = re.search(r'db_pool_size (\d+)', metrics_text)
if pool_used_match and pool_size_match:
pool_used = int(pool_used_match.group(1))
pool_size = int(pool_size_match.group(1))
utilization = pool_used / pool_size
assert utilization <= 0.80, \
f"Connection pool at {utilization:.0%} utilization — saturation risk"
@pytest.mark.asyncio
async def test_queue_depth_not_backing_up():
"""Message queue depth should not grow under normal throughput."""
import asyncio
samples = []
async with aiohttp.ClientSession() as session:
for _ in range(5):
async with session.get(f"{BASE_URL}/internal/metrics") as resp:
text = await resp.text()
import re
match = re.search(r'message_queue_depth{queue_name="events"} (\d+)', text)
if match:
samples.append(int(match.group(1)))
await asyncio.sleep(5)
if len(samples) >= 2:
# Queue should not grow continuously (indicates processing can't keep up)
queue_growing = all(
samples[i] < samples[i+1] for i in range(len(samples) - 1)
)
assert not queue_growing, \
f"Queue depth growing continuously: {samples} — consumer not keeping up"Traffic Signal Tests
# tests/golden_signals/test_traffic.py
def test_traffic_counter_increments():
"""Request counter must increment on every request."""
import requests
# Read counter before
metrics_before = requests.get(f"{BASE_URL}/internal/metrics").text
count_before = extract_metric(metrics_before, "http_requests_total")
# Make 10 requests
for _ in range(10):
requests.get(f"{BASE_URL}/api/products")
# Read counter after
metrics_after = requests.get(f"{BASE_URL}/internal/metrics").text
count_after = extract_metric(metrics_after, "http_requests_total")
assert count_after >= count_before + 10, \
f"Counter did not increment correctly: before={count_before}, after={count_after}"
def test_traffic_labels_correct():
"""Traffic counters must have correct labels for route differentiation."""
import requests
# Hit different endpoints
requests.get(f"{BASE_URL}/api/products")
requests.get(f"{BASE_URL}/api/users")
requests.post(f"{BASE_URL}/api/orders", json={"product_id": "test"})
metrics = requests.get(f"{BASE_URL}/internal/metrics").text
# Each endpoint should have its own counter
assert 'endpoint="/api/products"' in metrics
assert 'endpoint="/api/users"' in metrics
assert 'endpoint="/api/orders"' in metrics
assert 'method="POST"' in metricsRunning Golden Signal Tests in CI
# .github/workflows/golden-signals.yml
name: Golden Signals Tests
on:
push:
branches: [main]
pull_request:
jobs:
golden-signals:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Start service
run: docker-compose up -d api
- name: Wait for service health
run: |
timeout 60 bash -c \
'until curl -sf http://localhost:8080/health; do sleep 2; done'
- name: Run golden signal tests
run: |
pip install pytest pytest-asyncio aiohttp psutil
pytest tests/golden_signals/ -v --tb=short
- name: Stop service
if: always()
run: docker-compose downContinuous Golden Signal Monitoring
In production, run golden signal checks on a schedule:
# monitoring/golden_signal_health_check.py
"""
Scheduled health check that tests all four golden signals.
Run every 5 minutes via cron or HelpMeTest health checks.
"""
import requests
import sys
def check_golden_signals(service_url: str, metrics_url: str) -> bool:
"""Returns True if all signals are healthy."""
all_ok = True
# 1. Latency check
import time
latencies = []
for _ in range(10):
start = time.perf_counter()
resp = requests.get(f"{service_url}/api/products", timeout=5)
latency_ms = (time.perf_counter() - start) * 1000
if resp.status_code == 200:
latencies.append(latency_ms)
if latencies:
p95 = sorted(latencies)[int(len(latencies) * 0.95)]
if p95 > 500:
print(f"⚠️ LATENCY: p95={p95:.0f}ms exceeds 500ms")
all_ok = False
else:
print(f"✓ LATENCY: p95={p95:.0f}ms")
# 2. Error rate check (from metrics)
metrics = requests.get(metrics_url).text
# (parse error rate from Prometheus metrics)
# 3. Traffic check (is service receiving traffic?)
# (parse request rate from metrics)
# 4. Saturation check
# (parse CPU, memory, connection pool from metrics)
return all_ok
if __name__ == "__main__":
ok = check_golden_signals(
service_url="https://api.example.com",
metrics_url="https://api.example.com/internal/metrics",
)
sys.exit(0 if ok else 1)Summary
The four golden signals are testable, not just observable:
- Latency — test p50, p95, p99 under normal and concurrent load; test that errors fail fast
- Traffic — test that counters increment correctly with right labels
- Errors — test error rate under load; test implicit errors (valid HTTP code, broken response)
- Saturation — test CPU, memory, connection pool, queue depth under expected load
Run these tests in CI on every deploy, and schedule them as continuous health checks in production. A service that passes golden signal tests on every deploy rarely surprises you in production.