Testing gRPC Observability: Tracing, Metrics, and Logs with OpenTelemetry
gRPC services need observability — traces that show request flow across services, metrics that show latency and error rates, and structured logs that correlate with trace IDs. But observability code is rarely tested. Teams instrument their services and assume it works, discovering gaps only when an incident requires a trace that doesn't exist.
This guide covers how to test OpenTelemetry instrumentation in gRPC services: verifying that spans are created with correct attributes, metrics are emitted with the right labels, and context propagation works across service boundaries.
Why Test Observability?
Untested observability leads to:
- Missing spans when a new code path was added
- Wrong status codes on spans (UNSET instead of ERROR for 5xx responses)
- gRPC method names missing from metrics labels
- Trace context not propagated through background workers
- Metric cardinality explosion from uncontrolled label values
The OpenTelemetry Testing Pattern
OpenTelemetry provides in-memory exporters for testing — they collect telemetry data locally without requiring Jaeger, Prometheus, or any external system.
Setting Up the Test SDK in Go
// testutil/otel.go
package testutil
import (
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/metric/metricdata"
"go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
)
type TestOTel struct {
SpanExporter *tracetest.InMemoryExporter
MetricReader *metric.ManualReader
TracerProvider *trace.TracerProvider
MeterProvider *metric.MeterProvider
}
func NewTestOTel() *TestOTel {
spanExporter := tracetest.NewInMemoryExporter()
tracerProvider := trace.NewTracerProvider(
trace.WithSyncer(spanExporter),
)
otel.SetTracerProvider(tracerProvider)
metricReader := metric.NewManualReader()
meterProvider := metric.NewMeterProvider(
metric.WithReader(metricReader),
)
otel.SetMeterProvider(meterProvider)
return &TestOTel{
SpanExporter: spanExporter,
MetricReader: metricReader,
TracerProvider: tracerProvider,
MeterProvider: meterProvider,
}
}
func (t *TestOTel) Spans() []tracetest.SpanStub {
return t.SpanExporter.GetSpans()
}
func (t *TestOTel) Reset() {
t.SpanExporter.Reset()
}
func (t *TestOTel) CollectMetrics() metricdata.ResourceMetrics {
var rm metricdata.ResourceMetrics
t.MetricReader.Collect(context.Background(), &rm)
return rm
}Testing Traces in gRPC Services
Your gRPC service should create spans for each operation:
// user-service.go
func (s *UserServiceImpl) GetUser(ctx context.Context, req *pb.GetUserRequest) (*pb.GetUserResponse, error) {
ctx, span := otel.Tracer("user-service").Start(ctx, "UserService.GetUser")
defer span.End()
span.SetAttributes(
attribute.String("user.id", req.UserId),
)
user, err := s.db.FindUser(ctx, req.UserId)
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
return nil, status.Errorf(codes.NotFound, "user not found: %v", req.UserId)
}
span.SetStatus(codes.Ok, "")
return &pb.GetUserResponse{User: user}, nil
}Test that spans are emitted correctly:
// user-service_test.go
package service_test
import (
"context"
"testing"
"time"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestGetUser_SpanCreated(t *testing.T) {
tel := testutil.NewTestOTel()
defer tel.Reset()
svc := NewUserService(tel.TracerProvider, mockDB())
_, err := svc.GetUser(context.Background(), &pb.GetUserRequest{UserId: "user-1"})
require.NoError(t, err)
spans := tel.Spans()
require.Len(t, spans, 1, "expected exactly 1 span")
span := spans[0]
assert.Equal(t, "UserService.GetUser", span.Name)
assert.Equal(t, codes.Ok, span.Status.Code)
// Check attributes
attrs := spanAttrsMap(span.Attributes)
assert.Equal(t, "user-1", attrs["user.id"])
}
func TestGetUser_SpanHasErrorOnNotFound(t *testing.T) {
tel := testutil.NewTestOTel()
defer tel.Reset()
svc := NewUserService(tel.TracerProvider, emptyDB()) // No users
_, err := svc.GetUser(context.Background(), &pb.GetUserRequest{UserId: "missing"})
require.Error(t, err)
spans := tel.Spans()
require.Len(t, spans, 1)
span := spans[0]
assert.Equal(t, codes.Error, span.Status.Code)
assert.Len(t, span.Events, 1) // RecordError creates an event
assert.Equal(t, "exception", span.Events[0].Name)
}
func spanAttrsMap(attrs []attribute.KeyValue) map[string]string {
m := make(map[string]string)
for _, kv := range attrs {
m[string(kv.Key)] = kv.Value.AsString()
}
return m
}Testing Context Propagation
Distributed tracing only works if trace context is propagated across service calls. Test it:
func TestTraceContextPropagation(t *testing.T) {
tel := testutil.NewTestOTel()
defer tel.Reset()
// Simulate: service A creates a trace and calls service B
// Service A: create a root span
ctx, rootSpan := otel.Tracer("service-a").Start(context.Background(), "ServiceA.CallB")
defer rootSpan.End()
// Propagate context via gRPC metadata
md := metadata.New(nil)
otel.GetTextMapPropagator().Inject(ctx, &metadataTextMapCarrier{md})
// Service B: extract context from incoming metadata
incomingCtx := metadata.NewIncomingContext(context.Background(), md)
extractedCtx := otel.GetTextMapPropagator().Extract(
incomingCtx,
&metadataTextMapCarrier{md},
)
// Service B creates a child span
_, childSpan := otel.Tracer("service-b").Start(extractedCtx, "ServiceB.HandleRequest")
childSpan.End()
spans := tel.Spans()
require.Len(t, spans, 2)
// Verify parent-child relationship
root := findSpan(spans, "ServiceA.CallB")
child := findSpan(spans, "ServiceB.HandleRequest")
require.NotNil(t, root)
require.NotNil(t, child)
assert.Equal(t, root.SpanContext.TraceID(), child.SpanContext.TraceID(),
"child span should share trace ID with parent")
assert.Equal(t, root.SpanContext.SpanID(), child.Parent.SpanID(),
"child span's parent should be the root span")
}Testing gRPC Metrics
The go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc package automatically adds gRPC metrics. Test that they're emitted:
func TestGRPCMetricsEmitted(t *testing.T) {
tel := testutil.NewTestOTel()
// Create a server with OpenTelemetry instrumentation
server := grpc.NewServer(
grpc.StatsHandler(otelgrpc.NewServerHandler()),
)
pb.RegisterUserServiceServer(server, NewUserService(tel.TracerProvider, mockDB()))
lis, _ := net.Listen("tcp", "localhost:0")
go server.Serve(lis)
defer server.Stop()
// Make a request
conn, _ := grpc.Dial(lis.Addr().String(),
grpc.WithInsecure(),
grpc.WithStatsHandler(otelgrpc.NewClientHandler()),
)
client := pb.NewUserServiceClient(conn)
client.GetUser(context.Background(), &pb.GetUserRequest{UserId: "user-1"})
conn.Close()
// Collect metrics
rm := tel.CollectMetrics()
// Find the RPC server duration metric
rpcDurationMetric := findMetric(rm, "rpc.server.duration")
require.NotNil(t, rpcDurationMetric, "rpc.server.duration metric should exist")
// Verify labels include the gRPC method
hist := rpcDurationMetric.Data.(metricdata.Histogram[float64])
require.NotEmpty(t, hist.DataPoints)
dp := hist.DataPoints[0]
attrs := metricAttrsMap(dp.Attributes)
assert.Equal(t, "/user.v1.UserService/GetUser", attrs["rpc.method"])
assert.Equal(t, "ok", attrs["rpc.grpc.status_code"])
}Testing Error Rate Metrics
func TestGRPCErrorMetrics(t *testing.T) {
tel := testutil.NewTestOTel()
// ... setup server and client as above
// Make a request that will fail (NOT_FOUND)
client.GetUser(context.Background(), &pb.GetUserRequest{UserId: "nonexistent"})
rm := tel.CollectMetrics()
rpcMetric := findMetric(rm, "rpc.server.duration")
dp := rpcMetric.Data.(metricdata.Histogram[float64]).DataPoints[0]
attrs := metricAttrsMap(dp.Attributes)
// Status code should reflect the error
assert.Equal(t, "not_found", attrs["rpc.grpc.status_code"])
}Python: Testing with OpenTelemetry SDK
# test_observability.py
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
def make_test_tracer():
exporter = InMemorySpanExporter()
provider = TracerProvider()
provider.add_span_processor(SimpleSpanProcessor(exporter))
return provider, exporter
def test_grpc_handler_creates_span():
provider, exporter = make_test_tracer()
service = UserService(tracer_provider=provider, db=mock_db())
import grpc
request = user_pb2.GetUserRequest(user_id="user-1")
context = MockGrpcContext()
service.GetUser(request, context)
spans = exporter.get_finished_spans()
assert len(spans) == 1
span = spans[0]
assert span.name == "UserService.GetUser"
assert span.status.status_code.value == "OK"
attrs = dict(span.attributes)
assert attrs["user.id"] == "user-1"
def test_grpc_error_sets_span_error():
provider, exporter = make_test_tracer()
service = UserService(tracer_provider=provider, db=empty_db())
request = user_pb2.GetUserRequest(user_id="nonexistent")
context = MockGrpcContext()
service.GetUser(request, context)
spans = exporter.get_finished_spans()
span = spans[0]
from opentelemetry.trace import StatusCode
assert span.status.status_code == StatusCode.ERROR
assert len(span.events) == 1 # RecordException creates an eventCI Integration
- name: Test gRPC observability
run: go test ./... -run "TestObservability|TestSpan|TestMetric|TestTrace" -v
env:
OTEL_SDK_DISABLED: "false" # Ensure SDK is not disabled in testsConclusion
Testing observability means testing that your service emits the right signals — not just that it handles requests. In-memory exporters give you a fast, dependency-free way to assert on span creation, status codes, attribute values, and context propagation. Add these tests to your service alongside functional tests, and you'll ship observability that actually works when you need it during an incident.