MQTT Protocol Testing for IoT Systems: QoS, Retained Messages, and Broker Failures
MQTT is the backbone of most IoT messaging architectures. It's lightweight, well-supported, and designed for unreliable networks — but "designed for unreliable networks" doesn't mean "tested against unreliable networks."
Most MQTT implementations are tested against a healthy broker on a reliable network. The failure modes that matter — broker unavailability, message ordering under QoS 1/2, retained message behavior, session resumption — are rarely tested systematically.
MQTT Protocol Fundamentals for Testing
MQTT has three Quality of Service (QoS) levels, and each has different delivery semantics that require different tests:
QoS 0 — At most once: fire and forget. Messages may be lost. No acknowledgment. Appropriate for high-frequency sensor data where occasional loss is acceptable.
QoS 1 — At least once: message is delivered at least once, possibly more. Requires PUBACK acknowledgment. Your subscriber must handle duplicate messages.
QoS 2 — Exactly once: message is delivered exactly once. Four-way handshake (PUBLISH, PUBREC, PUBREL, PUBCOMP). Highest overhead, guaranteed delivery.
Most IoT applications need QoS 1 minimum for important messages. Most teams test with QoS 0 because it's simplest. This gap creates bugs.
Test Environment Setup
For MQTT testing, you need a broker you can control. Options:
Mosquitto (open source) with testing configuration:
# mosquitto-test.conf
listener 1883
allow_anonymous true
persistence true
persistence_location /tmp/mosquitto/
log_dest stdout
log_type allEMQ X / EMQX in Docker for more advanced features (retained messages at scale, shared subscriptions):
docker run -d --name emqx \
-p 1883:1883 \
-p 18083:18083 \ # Management API
emqx/emqx:latestHiveMQ has a public test broker (broker.hivemq.com:1883) for development, but don't depend on it for automated tests — use a local broker.
For automated tests, control the broker lifecycle in your test setup/teardown:
import subprocess
import pytest
import paho.mqtt.client as mqtt
@pytest.fixture(scope="session")
def mqtt_broker():
proc = subprocess.Popen(["mosquitto", "-c", "mosquitto-test.conf"])
yield "localhost"
proc.terminate()
proc.wait()
@pytest.fixture
def mqtt_client(mqtt_broker):
client = mqtt.Client()
client.connect(mqtt_broker, 1883)
client.loop_start()
yield client
client.loop_stop()
client.disconnect()QoS Level Tests
QoS 1 Duplicate Handling
Verify your subscriber handles duplicate messages correctly when PUBACK is delayed.
def test_qos1_duplicate_handling(mqtt_client):
received_messages = []
message_ids = []
def on_message(client, userdata, message):
received_messages.append(message.payload.decode())
mqtt_client.subscribe("test/sensor/temperature", qos=1)
mqtt_client.on_message = on_message
# Publish with QoS 1
for i in range(10):
mqtt_client.publish("test/sensor/temperature", f"{i}", qos=1)
time.sleep(1)
# With a healthy broker, should receive exactly 10 messages
assert len(received_messages) == 10
# Verify message content (no unexpected duplicates under normal conditions)
assert received_messages == [str(i) for i in range(10)]QoS 2 Exactly-Once Delivery
Verify QoS 2 delivers exactly once even under simulated network disruption:
def test_qos2_exactly_once(mqtt_broker_with_toxiproxy):
"""Test QoS 2 exactly-once delivery with network disruption"""
received = []
subscriber = mqtt.Client()
subscriber.on_message = lambda c, u, m: received.append(m.payload.decode())
subscriber.connect(mqtt_broker_with_toxiproxy, 1883)
subscriber.subscribe("test/critical", qos=2)
subscriber.loop_start()
publisher = mqtt.Client()
publisher.connect(mqtt_broker_with_toxiproxy, 1883)
# Inject 50% packet loss during publish
toxiproxy.add_toxic("latency", stream="downstream", latency=200)
publisher.publish("test/critical", "important_payload", qos=2)
time.sleep(5) # Allow retries to complete
toxiproxy.remove_toxic("latency")
# With QoS 2, exactly one delivery regardless of network issues
assert len(received) == 1
assert received[0] == "important_payload"Retained Message Tests
Retained messages cause subtle bugs that only appear when clients reconnect or new clients subscribe.
New Subscriber Receives Retained Message
def test_retained_message_on_subscribe(mqtt_client):
# Publish a retained message
mqtt_client.publish(
"device/sensor1/status",
"online",
qos=1,
retain=True
)
time.sleep(0.5)
# New client subscribes — should immediately receive the retained message
new_client = mqtt.Client()
received = []
new_client.on_message = lambda c, u, m: received.append(m.payload.decode())
new_client.connect("localhost", 1883)
new_client.subscribe("device/sensor1/status", qos=1)
new_client.loop_start()
time.sleep(1)
assert len(received) == 1, "New subscriber should receive retained message"
assert received[0] == "online"
new_client.disconnect()Clearing Retained Messages
def test_clear_retained_message(mqtt_client):
# Set retained message
mqtt_client.publish("device/sensor1/status", "online", retain=True)
time.sleep(0.5)
# Clear retained message (publish empty payload with retain=True)
mqtt_client.publish("device/sensor1/status", "", retain=True)
time.sleep(0.5)
# New subscriber should NOT receive a retained message
new_client = mqtt.Client()
received = []
new_client.on_message = lambda c, u, m: received.append(m)
new_client.connect("localhost", 1883)
new_client.subscribe("device/sensor1/status", qos=1)
new_client.loop_start()
time.sleep(1)
assert len(received) == 0, "Cleared retained message should not be delivered"Client Reconnection Tests
IoT devices disconnect and reconnect constantly — poor WiFi, power cycling, firmware updates. Your MQTT client must handle reconnection correctly.
Clean vs Persistent Session
def test_persistent_session_receives_queued_messages(mqtt_broker):
"""Verify that messages published while client is offline are delivered on reconnect"""
# Connect with persistent session (clean_session=False)
subscriber = mqtt.Client(client_id="test-device-001", clean_session=False)
subscriber.connect(mqtt_broker, 1883)
subscriber.subscribe("device/001/commands", qos=1)
subscriber.disconnect()
# Publish messages while subscriber is offline
publisher = mqtt.Client()
publisher.connect(mqtt_broker, 1883)
for i in range(5):
publisher.publish("device/001/commands", f"command_{i}", qos=1)
publisher.disconnect()
time.sleep(1)
# Reconnect with same client ID — should receive queued messages
received = []
subscriber2 = mqtt.Client(client_id="test-device-001", clean_session=False)
subscriber2.on_message = lambda c, u, m: received.append(m.payload.decode())
subscriber2.connect(mqtt_broker, 1883)
subscriber2.loop_start()
time.sleep(2)
assert len(received) == 5, f"Expected 5 queued messages, got {len(received)}"
subscriber2.disconnect()Last Will and Testament
def test_last_will_published_on_disconnect(mqtt_broker):
"""Verify Last Will message is published when client disconnects unexpectedly"""
will_received = []
# Monitor for will message
monitor = mqtt.Client()
monitor.on_message = lambda c, u, m: will_received.append(m.payload.decode())
monitor.connect(mqtt_broker, 1883)
monitor.subscribe("device/001/status", qos=1)
monitor.loop_start()
# Connect device with Last Will
device = mqtt.Client(client_id="test-device-001")
device.will_set(
"device/001/status",
payload="offline",
qos=1,
retain=True
)
device.connect(mqtt_broker, 1883)
# Simulate ungraceful disconnect (socket close without DISCONNECT packet)
device._sock.close()
time.sleep(2) # Wait for keepalive timeout + will delivery
assert len(will_received) >= 1, "Last Will should be published on ungraceful disconnect"
assert will_received[-1] == "offline"Broker Failure and Failover
Broker Restart Recovery
def test_client_reconnects_after_broker_restart(mosquitto_process):
reconnected = threading.Event()
client = mqtt.Client()
client.on_connect = lambda c, u, f, rc: reconnected.set() if rc == 0 else None
client.reconnect_delay_set(min_delay=1, max_delay=30)
client.connect("localhost", 1883)
client.loop_start()
reconnected.wait(timeout=5)
reconnected.clear()
# Restart the broker
mosquitto_process.terminate()
mosquitto_process.wait()
time.sleep(2) # Let client attempt reconnects
mosquitto_process = subprocess.Popen(["mosquitto", "-c", "mosquitto-test.conf"])
# Client should reconnect automatically
reconnected.wait(timeout=30)
assert reconnected.is_set(), "Client should reconnect after broker restart"MQTT Load Testing
For IoT deployments with thousands of devices, broker capacity is a real constraint. Test it before you hit it.
Concurrent Connection Load Test
import threading
import concurrent.futures
def connect_client(client_id, broker_host):
client = mqtt.Client(client_id=client_id)
result = {"id": client_id, "success": False, "latency_ms": None}
start = time.time()
try:
client.connect(broker_host, 1883, keepalive=60)
client.loop_start()
result["success"] = True
result["latency_ms"] = (time.time() - start) * 1000
except Exception as e:
result["error"] = str(e)
return result
def test_concurrent_connections(mqtt_broker):
"""Test broker handles 1000 concurrent connections"""
NUM_CLIENTS = 1000
with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
futures = [
executor.submit(connect_client, f"device-{i:04d}", mqtt_broker)
for i in range(NUM_CLIENTS)
]
results = [f.result() for f in concurrent.futures.as_completed(futures)]
success_count = sum(1 for r in results if r["success"])
avg_latency = sum(r["latency_ms"] for r in results if r.get("latency_ms")) / success_count
assert success_count >= 990, f"Only {success_count}/{NUM_CLIENTS} clients connected"
assert avg_latency < 1000, f"Average connection latency {avg_latency:.0f}ms exceeds 1s"Message Throughput Test
def test_message_throughput(mqtt_broker):
"""Test broker sustains 10,000 messages/second"""
MESSAGES_PER_SECOND = 10000
DURATION_SECONDS = 10
TOTAL_MESSAGES = MESSAGES_PER_SECOND * DURATION_SECONDS
received_count = threading.Event()
received = []
subscriber = mqtt.Client()
subscriber.on_message = lambda c, u, m: received.append(1)
subscriber.connect(mqtt_broker, 1883)
subscriber.subscribe("load/test/#", qos=0)
subscriber.loop_start()
publisher = mqtt.Client()
publisher.connect(mqtt_broker, 1883)
start = time.time()
for i in range(TOTAL_MESSAGES):
publisher.publish(f"load/test/{i % 100}", f"msg_{i}", qos=0)
# Rate limiting
if i % MESSAGES_PER_SECOND == 0:
elapsed = time.time() - start
expected_time = (i // MESSAGES_PER_SECOND)
if elapsed < expected_time:
time.sleep(expected_time - elapsed)
time.sleep(2) # Flush remaining messages
throughput = len(received) / DURATION_SECONDS
print(f"Throughput: {throughput:.0f} msg/s, received {len(received)}/{TOTAL_MESSAGES}")
# At QoS 0, some loss is acceptable; verify we're close
assert len(received) >= TOTAL_MESSAGES * 0.95, f"Message loss exceeded 5%: received {len(received)}"Topic Design and Testing
MQTT topic structure is architectural. Test that your topic hierarchy works as intended.
Wildcard Subscription Tests
def test_single_level_wildcard(mqtt_client):
"""+ matches exactly one topic level"""
received = []
mqtt_client.on_message = lambda c, u, m: received.append(m.topic)
mqtt_client.subscribe("factory/+/temperature", qos=0)
mqtt_client.publish("factory/line1/temperature", "25")
mqtt_client.publish("factory/line2/temperature", "26")
mqtt_client.publish("factory/line1/humidity", "60") # Should NOT match
mqtt_client.publish("factory/line1/zone2/temperature", "24") # Should NOT match
time.sleep(1)
assert "factory/line1/temperature" in received
assert "factory/line2/temperature" in received
assert "factory/line1/humidity" not in received
assert "factory/line1/zone2/temperature" not in received
def test_multi_level_wildcard(mqtt_client):
"""# matches multiple topic levels"""
received = []
mqtt_client.on_message = lambda c, u, m: received.append(m.topic)
mqtt_client.subscribe("factory/line1/#", qos=0)
mqtt_client.publish("factory/line1/temperature", "25")
mqtt_client.publish("factory/line1/zone2/temperature", "24")
mqtt_client.publish("factory/line2/temperature", "26") # Should NOT match
time.sleep(1)
assert "factory/line1/temperature" in received
assert "factory/line1/zone2/temperature" in received
assert "factory/line2/temperature" not in receivedSecurity Testing for MQTT
Authentication and Authorization
def test_unauthenticated_connection_rejected(mqtt_broker_with_auth):
"""Verify broker rejects connections without credentials"""
client = mqtt.Client()
connection_result = []
client.on_connect = lambda c, u, f, rc: connection_result.append(rc)
client.connect(mqtt_broker_with_auth, 1883)
client.loop_start()
time.sleep(2)
# Return code 4 = bad username/password
# Return code 5 = not authorized
assert connection_result[0] in [4, 5], "Unauthenticated connection should be rejected"
def test_topic_authorization(mqtt_broker_with_acl):
"""Verify ACL prevents unauthorized topic access"""
client = mqtt.Client()
client.username_pw_set("device-001", "secret")
client.connect(mqtt_broker_with_acl, 1883)
# device-001 should be allowed to publish to its own topic
result = client.publish("device/001/data", "test", qos=1)
assert result.rc == 0
# device-001 should NOT be allowed to publish to another device's topic
result = client.publish("device/002/data", "test", qos=1)
# In most brokers, unauthorized publish silently drops; verify no delivery
# or check broker logs for denialHelpMeTest monitors your MQTT-connected devices and IoT infrastructure continuously — detecting when devices go offline, message queues back up, or broker health degrades. Start free.