Jaeger Distributed Tracing: Setup and Testing Tutorial

docker run -d --name jaeger \
  -e COLLECTOR_OTLP_ENABLED=true \
  -p 16686:16686 \
  -p 4317:4317 \
  -p 4318:4318 \
  jaegertracing/all-in-one:latest

npm install @opentelemetry/sdk-node \
  @opentelemetry/auto-instrumentations-node \
  @opentelemetry/exporter-trace-otlp-http

// tracing.js — load before anything else
const { NodeSDK } = require('@opentelemetry/sdk-node');
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-http');
const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node');

const sdk = new NodeSDK({
  traceExporter: new OTLPTraceExporter({
    url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT || 'http://localhost:4318/v1/traces',
  }),
  instrumentations: [getNodeAutoInstrumentations()],
});

sdk.start();

pip install opentelemetry-sdk opentelemetry-exporter-otlp

# tracing.py
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter

provider = TracerProvider()
exporter = OTLPSpanExporter(endpoint="http://localhost:4318/v1/traces")
provider.add_span_processor(BatchSpanProcessor(exporter))
trace.set_tracer_provider(provider)

GET /api/services                          — list all services
GET /api/traces?service=order-service      — search traces by service
GET /api/traces/{traceId}                  — get specific trace
GET /api/services/{service}/operations     — list operations for a service

async function findRecentTraces(service, operation, sinceMs) {
  const params = new URLSearchParams({
    service,
    operation,
    start: (sinceMs * 1000).toString(), // Jaeger uses microseconds
    limit: '10',
  });
  const res = await fetch(`http://localhost:16686/api/traces?${params}`);
  const json = await res.json();
  return json.data || [];
}

function extractSpans(trace) {
  const processMap = {};
  trace.processes.forEach((proc, key) => {
    processMap[key] = proc.serviceName;
  });
  
  return trace.spans.map(span => ({
    traceId: span.traceID,
    spanId: span.spanID,
    operation: span.operationName,
    service: processMap[span.processID],
    startMs: span.startTime / 1000,
    durationMs: span.duration / 1000,
    tags: Object.fromEntries(span.tags.map(t => [t.key, t.value])),
    logs: span.logs,
    hasError: span.tags.some(t => t.key === 'error' && t.value === true),
  }));
}

// jest.config.js — run Jaeger in setup
module.exports = {
  globalSetup: './test/jaeger-setup.js',
  globalTeardown: './test/jaeger-teardown.js',
};

// test/jaeger-setup.js
const { execSync } = require('child_process');

module.exports = async () => {
  execSync('docker run -d --name jaeger-test \
    -e COLLECTOR_OTLP_ENABLED=true \
    -p 16686:16686 -p 4317:4317 -p 4318:4318 \
    jaegertracing/all-in-one:latest');
  
  // wait for Jaeger to be ready
  await waitForJaeger('http://localhost:16686');
};

// order.test.js
const { findRecentTraces, extractSpans } = require('./jaeger-client');

describe('Order service traces', () => {
  let startedAt;

  beforeEach(() => {
    startedAt = Date.now();
  });

  test('successful order creates complete multi-service trace', async () => {
    // Act
    const res = await fetch('http://localhost:3000/orders', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ userId: 'u1', sku: 'WIDGET-1', qty: 2 }),
    });
    expect(res.status).toBe(201);

    await new Promise(r => setTimeout(r, 800)); // wait for trace export

    // Assert on trace
    const traces = await findRecentTraces('order-service', 'POST /orders', startedAt);
    expect(traces).toHaveLength(1);

    const spans = extractSpans(traces[0]);
    const services = [...new Set(spans.map(s => s.service))];

    expect(services).toContain('order-service');
    expect(services).toContain('inventory-service');
    expect(services).toContain('payment-service');
    
    const errorSpans = spans.filter(s => s.hasError);
    expect(errorSpans).toHaveLength(0);
  });

  test('database spans have correct attributes', async () => {
    await fetch('http://localhost:3000/orders', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ userId: 'u1', sku: 'WIDGET-1', qty: 1 }),
    });

    await new Promise(r => setTimeout(r, 800));

    const traces = await findRecentTraces('order-service', 'POST /orders', startedAt);
    const spans = extractSpans(traces[0]);
    const dbSpans = spans.filter(s => s.tags['db.system']);

    expect(dbSpans.length).toBeGreaterThan(0);
    dbSpans.forEach(span => {
      expect(span.tags['db.system']).toBeDefined();
      expect(span.durationMs).toBeLessThan(200);
    });
  });

  test('payment failure appears as error span', async () => {
    // Arrange: inject failure via header
    await fetch('http://localhost:3000/orders', {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'X-Test-Payment-Fail': 'card_declined',
      },
      body: JSON.stringify({ userId: 'u1', sku: 'WIDGET-1', qty: 1 }),
    });

    await new Promise(r => setTimeout(r, 800));

    const traces = await findRecentTraces('order-service', 'POST /orders', startedAt);
    const spans = extractSpans(traces[0]);
    const paymentSpan = spans.find(s => s.service === 'payment-service');

    expect(paymentSpan).toBeDefined();
    expect(paymentSpan.hasError).toBe(true);
    expect(paymentSpan.tags['error.message']).toContain('card_declined');
  });
});

// Clear by restarting Jaeger between test files
beforeAll(async () => {
  execSync('docker restart jaeger-test');
  await waitForJaeger('http://localhost:16686');
});

// All services that ever sent a trace
const services = await fetch('http://localhost:16686/api/services')
  .then(r => r.json())
  .then(d => d.data);

// Traces with errors only
const errorTraces = await findRecentTraces('order-service', '', startedAt)
  .then(traces => traces.filter(t => 
    t.spans.some(s => s.tags.some(tag => tag.key === 'error' && tag.value))
  ));

// Slowest traces (for performance regression detection)
const sortedByDuration = traces.sort((a, b) => {
  const durationA = Math.max(...a.spans.map(s => s.startTime + s.duration)) - Math.min(...a.spans.map(s => s.startTime));
  const durationB = Math.max(...b.spans.map(s => s.startTime + s.duration)) - Math.min(...b.spans.map(s => s.startTime));
  return durationB - durationA;
});

# docker-compose.test.yml
version: '3.8'
services:
  jaeger:
    image: jaegertracing/all-in-one:latest
    environment:
      - COLLECTOR_OTLP_ENABLED=true
    ports:
      - "16686:16686"
      - "4317:4317"
      - "4318:4318"

  order-service:
    build: ./services/order
    environment:
      - OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318/v1/traces
      - OTEL_SERVICE_NAME=order-service
    depends_on:
      - jaeger

  inventory-service:
    build: ./services/inventory
    environment:
      - OTEL_EXPORTER_OTLP_ENDPOINT=http://jaeger:4318/v1/traces
      - OTEL_SERVICE_NAME=inventory-service
    depends_on:
      - jaeger