Simulating IoT Device Fleets for Integration Testing
You can't test an IoT platform with five devices. The failure modes that matter — broker overload during reconnect storms, message ordering under high concurrency, race conditions in device provisioning, thundering-herd behavior after an outage — only appear at scale. But you can't buy and rack 10,000 physical devices for a test lab.
The answer is fleet simulation: running hundreds or thousands of software-emulated devices that behave like real hardware, without the hardware. This post covers how to build and use IoT fleet simulators for integration testing.
What a Good Fleet Simulator Provides
A device simulator needs to do more than open an MQTT connection and publish messages. A useful simulator:
- Implements realistic device state machines — devices go through provisioning, idle, active, sleep, and reconnecting states
- Produces realistic data patterns — sensor readings follow physical distributions, not
Math.random() - Models failure behavior — disconnects, dropped messages, and slow responses
- Is controllable — the test can command specific devices to fail, reconnect, or send anomalous data
- Is observable — you can query the simulator's internal state to write assertions
Building a Device Simulator in Python
Here's a foundational VirtualDevice class you can extend for your specific device type:
# simulator/virtual_device.py
import asyncio
import json
import random
import time
import uuid
from enum import Enum
from typing import Optional
import asyncio_mqtt
class DeviceState(Enum):
PROVISIONING = "provisioning"
IDLE = "idle"
ACTIVE = "active"
SLEEPING = "sleeping"
RECONNECTING = "reconnecting"
OFFLINE = "offline"
class VirtualDevice:
def __init__(
self,
device_id: str,
broker_host: str,
broker_port: int = 1883,
telemetry_interval_sec: float = 5.0,
):
self.device_id = device_id
self.broker_host = broker_host
self.broker_port = broker_port
self.telemetry_interval = telemetry_interval_sec
self.state = DeviceState.PROVISIONING
self.firmware_version = "1.0.0"
self.uptime_sec = 0
self._client: Optional[asyncio_mqtt.Client] = None
self._running = False
self._published_count = 0
self._error_count = 0
# Physical state — override in subclasses
self._temperature = 20.0 + random.gauss(0, 2)
self._humidity = 50.0 + random.gauss(0, 5)
def _generate_reading(self) -> dict:
"""Simulate sensor drift and noise."""
self._temperature += random.gauss(0, 0.1)
self._temperature = max(-40, min(85, self._temperature))
self._humidity += random.gauss(0, 0.5)
self._humidity = max(0, min(100, self._humidity))
return {
"device_id": self.device_id,
"firmware_version": self.firmware_version,
"timestamp": int(time.time() * 1000),
"uptime_sec": self.uptime_sec,
"readings": {
"temperature": round(self._temperature, 2),
"humidity": round(self._humidity, 1),
"battery_pct": max(0, 100 - self.uptime_sec // 360),
},
"state": self.state.value,
}
async def connect(self):
self.state = DeviceState.PROVISIONING
self._client = asyncio_mqtt.Client(
self.broker_host,
port=self.broker_port,
client_id=self.device_id,
will=asyncio_mqtt.Will(
topic=f"devices/{self.device_id}/status",
payload=json.dumps({"status": "offline", "device_id": self.device_id}),
qos=1,
retain=True,
)
)
await self._client.__aenter__()
# Publish online presence
await self._client.publish(
f"devices/{self.device_id}/status",
json.dumps({"status": "online", "device_id": self.device_id}),
qos=1,
retain=True,
)
self.state = DeviceState.ACTIVE
async def run(self):
self._running = True
await self.connect()
while self._running:
try:
reading = self._generate_reading()
await self._client.publish(
f"devices/{self.device_id}/telemetry",
json.dumps(reading),
qos=1,
)
self._published_count += 1
self.uptime_sec += self.telemetry_interval
await asyncio.sleep(self.telemetry_interval)
except Exception as e:
self._error_count += 1
self.state = DeviceState.RECONNECTING
await asyncio.sleep(5)
await self.connect()
async def disconnect(self, graceful=True):
self._running = False
if self._client and not graceful:
# Simulate ungraceful disconnect — broker will publish LWT
self._client._client.socket().close()
elif self._client:
await self._client.publish(
f"devices/{self.device_id}/status",
json.dumps({"status": "offline"}),
qos=1,
retain=True,
)
await self._client.__aexit__(None, None, None)
self.state = DeviceState.OFFLINE
def get_stats(self) -> dict:
return {
"device_id": self.device_id,
"state": self.state.value,
"published_count": self._published_count,
"error_count": self._error_count,
"uptime_sec": self.uptime_sec,
}Running a Fleet
# simulator/fleet.py
import asyncio
from typing import List, Dict
from .virtual_device import VirtualDevice, DeviceState
class DeviceFleet:
def __init__(self, broker_host: str, broker_port: int = 1883):
self.broker_host = broker_host
self.broker_port = broker_port
self.devices: Dict[str, VirtualDevice] = {}
self._tasks: List[asyncio.Task] = []
def add_devices(self, count: int, prefix: str = "sim-device", **kwargs) -> List[str]:
ids = []
for i in range(count):
device_id = f"{prefix}-{i:05d}"
self.devices[device_id] = VirtualDevice(
device_id,
self.broker_host,
self.broker_port,
**kwargs
)
ids.append(device_id)
return ids
async def start(self):
self._tasks = [
asyncio.create_task(device.run())
for device in self.devices.values()
]
async def stop_all(self):
for device in self.devices.values():
await device.disconnect()
for task in self._tasks:
task.cancel()
await asyncio.gather(*self._tasks, return_exceptions=True)
async def disconnect_device(self, device_id: str, graceful=True):
if device_id in self.devices:
await self.devices[device_id].disconnect(graceful=graceful)
async def reconnect_all_simultaneously(self):
"""Simulate a reconnect storm — all devices reconnect at once."""
reconnect_tasks = [
device.connect()
for device in self.devices.values()
if device.state == DeviceState.OFFLINE
]
await asyncio.gather(*reconnect_tasks)
def get_fleet_stats(self) -> dict:
stats = [d.get_stats() for d in self.devices.values()]
active = sum(1 for s in stats if s["state"] == "active")
total_published = sum(s["published_count"] for s in stats)
total_errors = sum(s["error_count"] for s in stats)
return {
"total_devices": len(stats),
"active_devices": active,
"total_messages_published": total_published,
"total_errors": total_errors,
}Writing Fleet-Scale Integration Tests
# tests/test_fleet_integration.py
import asyncio
import pytest
import time
@pytest.fixture
async def fleet(mqtt_broker):
f = DeviceFleet(mqtt_broker.host, mqtt_broker.port)
f.add_devices(100, prefix="test-device", telemetry_interval_sec=1.0)
await f.start()
await asyncio.sleep(3) # Let devices connect and send initial readings
yield f
await f.stop_all()
async def test_platform_ingests_all_devices(fleet, backend_api):
"""Backend should register all 100 simulated devices."""
await asyncio.sleep(5) # Give time for device registration
registered = await backend_api.list_devices(prefix="test-device")
assert len(registered) == 100, \
f"Expected 100 registered devices, got {len(registered)}"
async def test_telemetry_throughput(fleet, backend_api):
"""Platform should ingest all telemetry messages without drops."""
# Record baseline
start_count = await backend_api.count_messages(prefix="test-device")
await asyncio.sleep(10) # 10 seconds at 1msg/s/device = ~1000 messages expected
end_count = await backend_api.count_messages(prefix="test-device")
received = end_count - start_count
# Allow 5% message loss (network simulation, async timing)
expected = 100 * 10 # 100 devices × 10 seconds
assert received >= expected * 0.95, \
f"Expected ~{expected} messages, only received {received} ({100*received/expected:.1f}%)"
async def test_reconnect_storm_handling(fleet, backend_api, broker_metrics):
"""Platform must survive all 100 devices reconnecting simultaneously."""
# Disconnect all devices
for device_id in list(fleet.devices.keys()):
await fleet.disconnect_device(device_id, graceful=False)
await asyncio.sleep(2)
# Reconnect all at once
reconnect_start = time.monotonic()
await fleet.reconnect_all_simultaneously()
# Wait for all to reconnect
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
stats = fleet.get_fleet_stats()
if stats["active_devices"] == 100:
break
await asyncio.sleep(0.5)
reconnect_duration = time.monotonic() - reconnect_start
stats = fleet.get_fleet_stats()
assert stats["active_devices"] == 100, \
f"Only {stats['active_devices']}/100 devices reconnected after storm"
assert reconnect_duration < 15, \
f"Reconnect storm took {reconnect_duration:.1f}s — broker may be overwhelmed"
# Broker should not have crashed
broker_health = await broker_metrics.get_health()
assert broker_health["status"] == "healthy"Injecting Chaos Into the Simulator
The real value of a fleet simulator is the ability to inject failures programmatically. Add a ChaosController that can command devices to misbehave:
# simulator/chaos.py
import asyncio
import random
from typing import List
from .fleet import DeviceFleet
class ChaosController:
def __init__(self, fleet: DeviceFleet):
self.fleet = fleet
async def random_disconnects(self, rate_pct: float = 10, duration_sec: float = 30):
"""Randomly disconnect `rate_pct` percent of devices every second."""
deadline = asyncio.get_event_loop().time() + duration_sec
while asyncio.get_event_loop().time() < deadline:
candidates = list(self.fleet.devices.keys())
count = max(1, int(len(candidates) * rate_pct / 100))
targets = random.sample(candidates, count)
for device_id in targets:
asyncio.create_task(
self.fleet.disconnect_device(device_id, graceful=False)
)
await asyncio.sleep(1)
async def send_malformed_payloads(self, device_ids: List[str], count: int = 10):
"""Have specific devices send payloads that should be rejected."""
malformed = [
b"not-json",
b'{"device_id": null, "timestamp": "bad"}',
b"\x00\x01\x02\x03", # binary garbage
b"{" * 10000, # oversized
]
for device_id in device_ids:
device = self.fleet.devices.get(device_id)
if device and device._client:
for i in range(count):
payload = malformed[i % len(malformed)]
await device._client.publish(
f"devices/{device_id}/telemetry",
payload,
qos=0,
)
async def simulate_firmware_update_wave(self, new_version: str, batch_size: int = 10):
"""Simulate a staged OTA rollout across the fleet."""
device_ids = list(self.fleet.devices.keys())
for i in range(0, len(device_ids), batch_size):
batch = device_ids[i:i + batch_size]
for device_id in batch:
self.fleet.devices[device_id].firmware_version = new_version
await asyncio.sleep(2) # stagger batches
async def test_platform_rejects_malformed_payloads_from_fleet(fleet, backend_api):
"""Platform must reject malformed messages without taking down processing pipeline."""
chaos = ChaosController(fleet)
# Identify 10 devices to send bad data
bad_actors = list(fleet.devices.keys())[:10]
await chaos.send_malformed_payloads(bad_actors, count=20)
await asyncio.sleep(5)
# Good devices should still be ingesting normally
good_devices = list(fleet.devices.keys())[10:]
good_device_id = good_devices[0]
recent_messages = await backend_api.get_recent_messages(good_device_id, limit=5)
assert len(recent_messages) > 0, \
"Good devices stopped sending after bad actor injection — pipeline may have crashed"
# Bad actor messages should be in rejection log, not in normal message store
rejected = await backend_api.get_rejected_messages(limit=100)
bad_actor_rejections = [m for m in rejected if m["device_id"] in bad_actors]
assert len(bad_actor_rejections) > 0, \
"Malformed messages were silently dropped without appearing in rejection log"Modeling Realistic Device Populations
Real fleets aren't homogeneous. Some devices are on slow cellular connections, some are on fast WiFi, some have old firmware, and some are failing. Model this distribution:
def build_realistic_fleet(broker_host: str, total_devices: int) -> DeviceFleet:
fleet = DeviceFleet(broker_host)
# 70% normal devices on WiFi
fleet.add_devices(
int(total_devices * 0.70),
prefix="wifi-device",
telemetry_interval_sec=5.0,
)
# 20% on cellular — slower, higher latency
fleet.add_devices(
int(total_devices * 0.20),
prefix="cellular-device",
telemetry_interval_sec=30.0, # less frequent to save data
)
# 5% on old firmware
old_firmware_devices = fleet.add_devices(
int(total_devices * 0.05),
prefix="legacy-device",
telemetry_interval_sec=10.0,
)
for device_id in old_firmware_devices:
fleet.devices[device_id].firmware_version = "0.8.2"
# 5% flaky devices that disconnect frequently
flaky_devices = fleet.add_devices(
int(total_devices * 0.05),
prefix="flaky-device",
telemetry_interval_sec=5.0,
)
return fleetCI Integration
Fleet simulation tests are too slow for every PR, but essential before a release. Run them on merge to main:
# .github/workflows/fleet-integration.yml
name: Fleet Integration Tests
on:
push:
branches: [main]
schedule:
- cron: '0 2 * * *' # Nightly
jobs:
fleet-simulation:
runs-on: ubuntu-latest
services:
mosquitto:
image: eclipse-mosquitto:2.0
ports:
- 1883:1883
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install dependencies
run: pip install -r requirements-test.txt
- name: Run fleet simulation tests (100 devices)
run: |
pytest tests/fleet/ -v \
--fleet-size=100 \
--timeout=300 \
-k "not nightly"
- name: Run nightly fleet tests (1000 devices)
if: github.event_name == 'schedule'
run: |
pytest tests/fleet/ -v \
--fleet-size=1000 \
--timeout=900 \
-k "nightly"Key Takeaways
- Fleet simulators must model state machines, not just publish random data — devices have provisioning, sleep, reconnecting, and error states
- Simulate realistic data distributions — Gaussian noise around a baseline is more realistic than uniform random values
- Build a
ChaosController— the ability to inject specific failures programmatically is what makes fleet tests useful - Model device heterogeneity — different firmware versions, connection types, and failure rates
- Reconnect storm tests are mandatory — this is the most common real-world failure mode that only appears at scale
- Separate fleet test tiers — small fleet (100 devices) on every merge, large fleet (1000+) nightly
The goal of fleet simulation is not to replace hardware testing but to give you rapid feedback on platform behavior at a scale you can't achieve with physical devices in a lab.