Testing Kafka Consumer Groups: Partition Assignment, Rebalancing, and Offsets
Kafka consumer groups distribute partitions across consumer instances. Most teams test individual message processing but skip the behavior that causes production incidents: what happens during rebalancing, what happens when offset commits fail, and how consumers recover from restarts. This guide covers that gap.
What Needs Testing in Consumer Groups
- Processing logic: does the consumer handle individual messages correctly?
- Offset management: are offsets committed only after successful processing?
- Rebalance handling: does the consumer flush in-progress work before partitions are revoked?
- Duplicate handling: is processing idempotent for at-least-once delivery?
- Lag and backpressure: does the consumer handle slow downstream dependencies?
Unit Testing Processing Logic
Keep processing logic separate from Kafka mechanics. This makes it testable without a Kafka broker.
# order_consumer.py
import json
import logging
from dataclasses import dataclass
from typing import Optional
logger = logging.getLogger(__name__)
@dataclass
class KafkaMessage:
key: str
value: dict
topic: str
partition: int
offset: int
class OrderProcessor:
"""Stateless processing logic — no Kafka dependency"""
def __init__(self, order_repo, notification_service):
self.repo = order_repo
self.notifications = notification_service
def process(self, message: KafkaMessage) -> bool:
"""Returns True on success, False on recoverable error"""
try:
order_data = message.value
if not self._validate(order_data):
logger.warning(f"Invalid order at offset {message.offset}, skipping")
return True # Skip invalid messages (they'd just loop forever)
order_id = order_data['order_id']
# Idempotency check
if self.repo.exists(order_id):
logger.info(f"Order {order_id} already processed, skipping duplicate")
return True
self.repo.save(order_data)
self.notifications.notify_order_received(order_id)
return True
except self.repo.TransientError:
return False # Will be retried (don't commit offset)
def _validate(self, order_data):
required = ['order_id', 'customer_id', 'items']
return all(k in order_data for k in required)# test_order_processor.py
import pytest
from unittest.mock import MagicMock, patch
from order_consumer import OrderProcessor, KafkaMessage
def make_message(order_data, offset=0):
return KafkaMessage(
key=order_data.get('order_id', 'key'),
value=order_data,
topic='orders',
partition=0,
offset=offset
)
class TestOrderProcessor:
def test_valid_new_order_is_saved_and_notified(self):
mock_repo = MagicMock()
mock_repo.exists.return_value = False
mock_notifications = MagicMock()
processor = OrderProcessor(mock_repo, mock_notifications)
result = processor.process(make_message({
'order_id': 'ord-001',
'customer_id': 'cust-1',
'items': [{'sku': 'A', 'qty': 1}]
}))
assert result is True
mock_repo.save.assert_called_once()
mock_notifications.notify_order_received.assert_called_once_with('ord-001')
def test_duplicate_order_is_skipped(self):
mock_repo = MagicMock()
mock_repo.exists.return_value = True # already processed
mock_notifications = MagicMock()
processor = OrderProcessor(mock_repo, mock_notifications)
result = processor.process(make_message({
'order_id': 'ord-001',
'customer_id': 'cust-1',
'items': [{'sku': 'A'}]
}))
assert result is True
mock_repo.save.assert_not_called()
mock_notifications.notify_order_received.assert_not_called()
def test_invalid_message_is_skipped_not_retried(self):
mock_repo = MagicMock()
mock_notifications = MagicMock()
processor = OrderProcessor(mock_repo, mock_notifications)
result = processor.process(make_message({'broken': 'message'}))
assert result is True # returns True to commit offset (skip it)
mock_repo.save.assert_not_called()
def test_transient_repo_error_triggers_retry(self):
mock_repo = MagicMock()
mock_repo.exists.return_value = False
mock_repo.save.side_effect = mock_repo.TransientError("DB timeout")
mock_notifications = MagicMock()
processor = OrderProcessor(mock_repo, mock_notifications)
result = processor.process(make_message({
'order_id': 'ord-001',
'customer_id': 'cust-1',
'items': [{'sku': 'A'}]
}))
assert result is False # signals: don't commit offsetIntegration Testing with Embedded Kafka
Use an embedded Kafka (or Testcontainers) to test the offset commit behavior.
// Java with Spring Kafka and EmbeddedKafka
@SpringBootTest
@EmbeddedKafka(
partitions = 3,
topics = {"orders", "orders.DLT"},
brokerProperties = {
"auto.create.topics.enable=true",
"offsets.topic.replication.factor=1"
}
)
class ConsumerGroupOffsetTest {
@Autowired
private KafkaTemplate<String, String> kafkaTemplate;
@Autowired
private EmbeddedKafkaBroker embeddedKafka;
@Test
void offsetIsNotCommittedOnProcessingFailure() throws Exception {
String groupId = "test-consumer-group-" + UUID.randomUUID();
AtomicInteger processCount = new AtomicInteger(0);
// First consumer: fails on first attempt
KafkaMessageListenerContainer<String, String> container = createContainer(
groupId,
(record) -> {
int count = processCount.incrementAndGet();
if (count == 1) {
throw new RuntimeException("Simulated failure");
}
// Second attempt succeeds
}
);
container.start();
kafkaTemplate.send("orders", "ord-001", "{\"order_id\":\"ord-001\"}");
// Wait for successful processing (2 attempts)
Thread.sleep(5000);
container.stop();
// Verify the message was processed exactly once to completion
assertEquals(2, processCount.get()); // 1 failure + 1 success
// Restart consumer — should not see the message again (offset was committed after success)
AtomicInteger reprocessCount = new AtomicInteger(0);
KafkaMessageListenerContainer<String, String> restartedContainer = createContainer(
groupId,
(record) -> reprocessCount.incrementAndGet()
);
restartedContainer.start();
Thread.sleep(2000);
restartedContainer.stop();
assertEquals(0, reprocessCount.get(), "Message should not be reprocessed after successful commit");
}
}Testing Rebalance Listeners
When partitions are revoked during rebalancing, in-flight processing should complete before the partition is handed off.
# consumer_with_rebalance.py
from confluent_kafka import Consumer, TopicPartition
import threading
class RebalancingConsumer:
def __init__(self, config, processor):
self.processor = processor
self._lock = threading.Lock()
self._processing = set() # partition IDs currently processing
config['on_assign'] = self._on_assign
config['on_revoke'] = self._on_revoke
self.consumer = Consumer(config)
def _on_assign(self, consumer, partitions):
print(f"Assigned partitions: {[p.partition for p in partitions]}")
def _on_revoke(self, consumer, partitions):
# Wait for in-flight messages on revoked partitions to complete
revoked = {p.partition for p in partitions}
timeout = 30
elapsed = 0
while revoked & self._processing and elapsed < timeout:
import time
time.sleep(0.1)
elapsed += 0.1
# Commit offsets before handing off partitions
consumer.commit(asynchronous=False)
print(f"Revoked partitions after drain: {[p.partition for p in partitions]}")def test_rebalance_listener_commits_before_revoke():
committed_before_revoke = []
mock_consumer = MagicMock()
mock_consumer.commit = MagicMock(
side_effect=lambda **kwargs: committed_before_revoke.append(True)
)
consumer = RebalancingConsumer.__new__(RebalancingConsumer)
consumer._processing = set()
consumer.consumer = mock_consumer
# Simulate partition revocation
from confluent_kafka import TopicPartition
partitions = [TopicPartition('orders', 0)]
consumer._on_revoke(mock_consumer, partitions)
assert committed_before_revoke, "Offset should be committed before partition revocation"
mock_consumer.commit.assert_called_with(asynchronous=False)Testing Consumer Lag
Consumer lag — the gap between the latest offset and the committed offset — should be monitored and tested.
def test_consumer_lag_calculation():
mock_admin = MagicMock()
mock_consumer_client = MagicMock()
# Simulated Kafka state
mock_admin.list_topics.return_value = MagicMock(topics={'orders': MagicMock(
partitions={0: None, 1: None, 2: None}
)})
# Latest offsets from broker
mock_consumer_client.get_watermark_offsets.side_effect = [
(0, 1000), # partition 0: latest offset 1000
(0, 800), # partition 1: latest offset 800
(0, 600), # partition 2: latest offset 600
]
# Committed offsets for consumer group
mock_admin.list_consumer_group_offsets.return_value = {
'orders': {
0: MagicMock(offset=990),
1: MagicMock(offset=790),
2: MagicMock(offset=500)
}
}
lag_calculator = ConsumerLagCalculator(mock_admin, mock_consumer_client)
lag = lag_calculator.get_lag('order-processors', 'orders')
assert lag['total'] == 120 # (10 + 10 + 100)
assert lag['max_partition_lag'] == 100 # partition 2 is furthest behind
assert lag['lag_by_partition'] == {0: 10, 1: 10, 2: 100}Consumer lag testing is typically done in integration or load tests — but the lag calculation logic is worth unit testing so that your monitoring tool is correct before you trust it in production.
Key Consumer Group Testing Patterns
| Test type | What it covers | Tool |
|---|---|---|
| Unit: processing logic | Message transformation, validation, idempotency | Mocks only |
| Unit: offset decision | When to commit, when to retry | Mock consumer |
| Unit: rebalance logic | Pre-revoke cleanup, post-assign setup | Mock consumer |
| Integration: offset commits | Committed after success, not after failure | Embedded Kafka |
| Integration: rebalancing | Partition handoff, duplicate handling | Embedded Kafka / Testcontainers |
| Load: consumer lag | Throughput under load, lag under backpressure | Testcontainers + real throughput |
The most overlooked: integration tests for offset commits. If offsets commit on exception, messages are lost. If offsets never commit, messages are reprocessed forever. Both are production bugs that pure unit tests won't catch.