Testing gRPC Observability: Tracing, Metrics, and Logs with OpenTelemetry

Testing gRPC Observability: Tracing, Metrics, and Logs with OpenTelemetry

gRPC services need observability — traces that show request flow across services, metrics that show latency and error rates, and structured logs that correlate with trace IDs. But observability code is rarely tested. Teams instrument their services and assume it works, discovering gaps only when an incident requires a trace that doesn't exist.

This guide covers how to test OpenTelemetry instrumentation in gRPC services: verifying that spans are created with correct attributes, metrics are emitted with the right labels, and context propagation works across service boundaries.

Why Test Observability?

Untested observability leads to:

  • Missing spans when a new code path was added
  • Wrong status codes on spans (UNSET instead of ERROR for 5xx responses)
  • gRPC method names missing from metrics labels
  • Trace context not propagated through background workers
  • Metric cardinality explosion from uncontrolled label values

The OpenTelemetry Testing Pattern

OpenTelemetry provides in-memory exporters for testing — they collect telemetry data locally without requiring Jaeger, Prometheus, or any external system.

Setting Up the Test SDK in Go

// testutil/otel.go
package testutil

import (
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/sdk/metric"
    "go.opentelemetry.io/otel/sdk/metric/metricdata"
    "go.opentelemetry.io/otel/sdk/trace"
    "go.opentelemetry.io/otel/sdk/trace/tracetest"
)

type TestOTel struct {
    SpanExporter   *tracetest.InMemoryExporter
    MetricReader    *metric.ManualReader
    TracerProvider  *trace.TracerProvider
    MeterProvider   *metric.MeterProvider
}

func NewTestOTel() *TestOTel {
    spanExporter := tracetest.NewInMemoryExporter()
    tracerProvider := trace.NewTracerProvider(
        trace.WithSyncer(spanExporter),
    )
    otel.SetTracerProvider(tracerProvider)
    
    metricReader := metric.NewManualReader()
    meterProvider := metric.NewMeterProvider(
        metric.WithReader(metricReader),
    )
    otel.SetMeterProvider(meterProvider)
    
    return &TestOTel{
        SpanExporter:  spanExporter,
        MetricReader:   metricReader,
        TracerProvider: tracerProvider,
        MeterProvider:  meterProvider,
    }
}

func (t *TestOTel) Spans() []tracetest.SpanStub {
    return t.SpanExporter.GetSpans()
}

func (t *TestOTel) Reset() {
    t.SpanExporter.Reset()
}

func (t *TestOTel) CollectMetrics() metricdata.ResourceMetrics {
    var rm metricdata.ResourceMetrics
    t.MetricReader.Collect(context.Background(), &rm)
    return rm
}

Testing Traces in gRPC Services

Your gRPC service should create spans for each operation:

// user-service.go
func (s *UserServiceImpl) GetUser(ctx context.Context, req *pb.GetUserRequest) (*pb.GetUserResponse, error) {
    ctx, span := otel.Tracer("user-service").Start(ctx, "UserService.GetUser")
    defer span.End()
    
    span.SetAttributes(
        attribute.String("user.id", req.UserId),
    )
    
    user, err := s.db.FindUser(ctx, req.UserId)
    if err != nil {
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
        return nil, status.Errorf(codes.NotFound, "user not found: %v", req.UserId)
    }
    
    span.SetStatus(codes.Ok, "")
    return &pb.GetUserResponse{User: user}, nil
}

Test that spans are emitted correctly:

// user-service_test.go
package service_test

import (
    "context"
    "testing"
    "time"
    
    "go.opentelemetry.io/otel/codes"
    "go.opentelemetry.io/otel/sdk/trace/tracetest"
    "github.com/stretchr/testify/assert"
    "github.com/stretchr/testify/require"
)

func TestGetUser_SpanCreated(t *testing.T) {
    tel := testutil.NewTestOTel()
    defer tel.Reset()
    
    svc := NewUserService(tel.TracerProvider, mockDB())
    
    _, err := svc.GetUser(context.Background(), &pb.GetUserRequest{UserId: "user-1"})
    require.NoError(t, err)
    
    spans := tel.Spans()
    require.Len(t, spans, 1, "expected exactly 1 span")
    
    span := spans[0]
    assert.Equal(t, "UserService.GetUser", span.Name)
    assert.Equal(t, codes.Ok, span.Status.Code)
    
    // Check attributes
    attrs := spanAttrsMap(span.Attributes)
    assert.Equal(t, "user-1", attrs["user.id"])
}

func TestGetUser_SpanHasErrorOnNotFound(t *testing.T) {
    tel := testutil.NewTestOTel()
    defer tel.Reset()
    
    svc := NewUserService(tel.TracerProvider, emptyDB()) // No users
    
    _, err := svc.GetUser(context.Background(), &pb.GetUserRequest{UserId: "missing"})
    require.Error(t, err)
    
    spans := tel.Spans()
    require.Len(t, spans, 1)
    
    span := spans[0]
    assert.Equal(t, codes.Error, span.Status.Code)
    assert.Len(t, span.Events, 1)  // RecordError creates an event
    assert.Equal(t, "exception", span.Events[0].Name)
}

func spanAttrsMap(attrs []attribute.KeyValue) map[string]string {
    m := make(map[string]string)
    for _, kv := range attrs {
        m[string(kv.Key)] = kv.Value.AsString()
    }
    return m
}

Testing Context Propagation

Distributed tracing only works if trace context is propagated across service calls. Test it:

func TestTraceContextPropagation(t *testing.T) {
    tel := testutil.NewTestOTel()
    defer tel.Reset()
    
    // Simulate: service A creates a trace and calls service B
    
    // Service A: create a root span
    ctx, rootSpan := otel.Tracer("service-a").Start(context.Background(), "ServiceA.CallB")
    defer rootSpan.End()
    
    // Propagate context via gRPC metadata
    md := metadata.New(nil)
    otel.GetTextMapPropagator().Inject(ctx, &metadataTextMapCarrier{md})
    
    // Service B: extract context from incoming metadata
    incomingCtx := metadata.NewIncomingContext(context.Background(), md)
    extractedCtx := otel.GetTextMapPropagator().Extract(
        incomingCtx,
        &metadataTextMapCarrier{md},
    )
    
    // Service B creates a child span
    _, childSpan := otel.Tracer("service-b").Start(extractedCtx, "ServiceB.HandleRequest")
    childSpan.End()
    
    spans := tel.Spans()
    require.Len(t, spans, 2)
    
    // Verify parent-child relationship
    root := findSpan(spans, "ServiceA.CallB")
    child := findSpan(spans, "ServiceB.HandleRequest")
    
    require.NotNil(t, root)
    require.NotNil(t, child)
    assert.Equal(t, root.SpanContext.TraceID(), child.SpanContext.TraceID(), 
        "child span should share trace ID with parent")
    assert.Equal(t, root.SpanContext.SpanID(), child.Parent.SpanID(),
        "child span's parent should be the root span")
}

Testing gRPC Metrics

The go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc package automatically adds gRPC metrics. Test that they're emitted:

func TestGRPCMetricsEmitted(t *testing.T) {
    tel := testutil.NewTestOTel()
    
    // Create a server with OpenTelemetry instrumentation
    server := grpc.NewServer(
        grpc.StatsHandler(otelgrpc.NewServerHandler()),
    )
    pb.RegisterUserServiceServer(server, NewUserService(tel.TracerProvider, mockDB()))
    
    lis, _ := net.Listen("tcp", "localhost:0")
    go server.Serve(lis)
    defer server.Stop()
    
    // Make a request
    conn, _ := grpc.Dial(lis.Addr().String(),
        grpc.WithInsecure(),
        grpc.WithStatsHandler(otelgrpc.NewClientHandler()),
    )
    client := pb.NewUserServiceClient(conn)
    client.GetUser(context.Background(), &pb.GetUserRequest{UserId: "user-1"})
    conn.Close()
    
    // Collect metrics
    rm := tel.CollectMetrics()
    
    // Find the RPC server duration metric
    rpcDurationMetric := findMetric(rm, "rpc.server.duration")
    require.NotNil(t, rpcDurationMetric, "rpc.server.duration metric should exist")
    
    // Verify labels include the gRPC method
    hist := rpcDurationMetric.Data.(metricdata.Histogram[float64])
    require.NotEmpty(t, hist.DataPoints)
    
    dp := hist.DataPoints[0]
    attrs := metricAttrsMap(dp.Attributes)
    assert.Equal(t, "/user.v1.UserService/GetUser", attrs["rpc.method"])
    assert.Equal(t, "ok", attrs["rpc.grpc.status_code"])
}

Testing Error Rate Metrics

func TestGRPCErrorMetrics(t *testing.T) {
    tel := testutil.NewTestOTel()
    
    // ... setup server and client as above
    
    // Make a request that will fail (NOT_FOUND)
    client.GetUser(context.Background(), &pb.GetUserRequest{UserId: "nonexistent"})
    
    rm := tel.CollectMetrics()
    rpcMetric := findMetric(rm, "rpc.server.duration")
    
    dp := rpcMetric.Data.(metricdata.Histogram[float64]).DataPoints[0]
    attrs := metricAttrsMap(dp.Attributes)
    
    // Status code should reflect the error
    assert.Equal(t, "not_found", attrs["rpc.grpc.status_code"])
}

Python: Testing with OpenTelemetry SDK

# test_observability.py
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from opentelemetry.sdk.trace.export import SimpleSpanProcessor

def make_test_tracer():
    exporter = InMemorySpanExporter()
    provider = TracerProvider()
    provider.add_span_processor(SimpleSpanProcessor(exporter))
    return provider, exporter

def test_grpc_handler_creates_span():
    provider, exporter = make_test_tracer()
    
    service = UserService(tracer_provider=provider, db=mock_db())
    
    import grpc
    request = user_pb2.GetUserRequest(user_id="user-1")
    context = MockGrpcContext()
    
    service.GetUser(request, context)
    
    spans = exporter.get_finished_spans()
    assert len(spans) == 1
    
    span = spans[0]
    assert span.name == "UserService.GetUser"
    assert span.status.status_code.value == "OK"
    
    attrs = dict(span.attributes)
    assert attrs["user.id"] == "user-1"

def test_grpc_error_sets_span_error():
    provider, exporter = make_test_tracer()
    service = UserService(tracer_provider=provider, db=empty_db())
    
    request = user_pb2.GetUserRequest(user_id="nonexistent")
    context = MockGrpcContext()
    service.GetUser(request, context)
    
    spans = exporter.get_finished_spans()
    span = spans[0]
    
    from opentelemetry.trace import StatusCode
    assert span.status.status_code == StatusCode.ERROR
    assert len(span.events) == 1  # RecordException creates an event

CI Integration

- name: Test gRPC observability
  run: go test ./... -run "TestObservability|TestSpan|TestMetric|TestTrace" -v
  env:
    OTEL_SDK_DISABLED: "false"  # Ensure SDK is not disabled in tests

Conclusion

Testing observability means testing that your service emits the right signals — not just that it handles requests. In-memory exporters give you a fast, dependency-free way to assert on span creation, status codes, attribute values, and context propagation. Add these tests to your service alongside functional tests, and you'll ship observability that actually works when you need it during an incident.

Read more

Start now free