Embedded Linux Testing: How to Test Yocto and Buildroot Images Automatically

# example_job.yaml — LAVA job definition
job_name: "Yocto Image Boot Test"
timeouts:
  job:
    minutes: 30
priority: medium
visibility: public

actions:
  - deploy:
      namespace: target
      to: tftp
      kernel:
        url: https://ci.example.com/artifacts/zImage
        type: zimage
      dtb:
        url: https://ci.example.com/artifacts/board.dtb
      ramdisk:
        url: https://ci.example.com/artifacts/rootfs.cpio.gz
        compression: gz
  
  - boot:
      namespace: target
      method: u-boot
      commands: ramdisk
      prompts:
        - "login:"
      auto_login:
        login_prompt: "login:"
        username: root
  
  - test:
      namespace: target
      definitions:
        - repository: https://github.com/my-org/lava-tests.git
          from: git
          path: tests/smoke/
          name: smoke-tests

# tests/test_embedded_image.py
import subprocess
import pexpect
import pytest
import time

QEMU_IMAGE = "build/tmp/deploy/images/qemux86-64/core-image-minimal-qemux86-64.wic"

@pytest.fixture(scope="session")
def qemu_session():
    """Boot image in QEMU and return a pexpect session."""
    
    # Start QEMU with the embedded image
    child = pexpect.spawn(
        "qemu-system-x86_64",
        args=[
            "-drive", f"file={QEMU_IMAGE},format=raw",
            "-m", "512",
            "-nographic",
            "-net", "nic",
            "-net", "user,hostfwd=tcp::2222-:22",
            "-enable-kvm",  # Remove if KVM not available
        ],
        timeout=120,
        encoding="utf-8",
    )
    
    # Wait for login prompt
    idx = child.expect(["login:", pexpect.EOF, pexpect.TIMEOUT], timeout=120)
    
    if idx != 0:
        pytest.fail(f"System did not boot to login prompt within 120s")
    
    # Log in
    child.sendline("root")
    child.expect(r"[#$]")  # Shell prompt
    
    yield child
    
    # Cleanup
    child.sendline("poweroff")
    child.close()

def test_system_boots(qemu_session):
    """System must boot to a login prompt."""
    # If we got here, the fixture succeeded = system booted
    pass

def test_kernel_version(qemu_session):
    """Kernel version must match expected."""
    qemu_session.sendline("uname -r")
    qemu_session.expect(r"[#$]")
    output = qemu_session.before
    
    assert "6.6" in output, f"Expected kernel 6.6.x, got: {output.strip()}"

def test_essential_commands_present(qemu_session):
    """Essential busybox commands must be available."""
    commands = ["ls", "cat", "cp", "mv", "rm", "ps", "grep", "find", "ip"]
    
    for cmd in commands:
        qemu_session.sendline(f"which {cmd}")
        qemu_session.expect(r"[#$]")
        output = qemu_session.before
        
        assert "/bin/" in output or "/usr/bin/" in output, \
            f"Command '{cmd}' not found: {output.strip()}"

def test_network_interface_up(qemu_session):
    """Ethernet interface must be configured."""
    qemu_session.sendline("ip addr show")
    qemu_session.expect(r"[#$]")
    output = qemu_session.before
    
    assert "eth0" in output or "ens3" in output, \
        f"No ethernet interface found: {output}"
    assert "inet " in output, "No IP address assigned"

def test_no_kernel_oops_in_dmesg(qemu_session):
    """dmesg should not contain kernel errors."""
    qemu_session.sendline("dmesg | grep -i 'oops\\|BUG:\\|panic'")
    qemu_session.expect(r"[#$]")
    output = qemu_session.before
    
    assert "Oops" not in output and "BUG:" not in output and "panic" not in output, \
        f"Kernel errors in dmesg: {output}"

# tests/embedded_test_utils.py
import paramiko
import pytest
import subprocess

class EmbeddedTarget:
    """SSH connection to embedded Linux target."""
    
    def __init__(self, host: str, port: int = 22, username: str = "root"):
        self.client = paramiko.SSHClient()
        self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        self.client.connect(host, port=port, username=username)
    
    def run(self, command: str, timeout: int = 30) -> tuple[str, str, int]:
        """Run command, return (stdout, stderr, exit_code)."""
        stdin, stdout, stderr = self.client.exec_command(command, timeout=timeout)
        exit_code = stdout.channel.recv_exit_status()
        return stdout.read().decode(), stderr.read().decode(), exit_code
    
    def run_assert_ok(self, command: str) -> str:
        """Run command and assert exit code 0."""
        out, err, code = self.run(command)
        assert code == 0, f"Command '{command}' failed (exit {code}):\n{err}"
        return out
    
    def close(self):
        self.client.close()

@pytest.fixture(scope="session")
def target():
    t = EmbeddedTarget("127.0.0.1", port=2222)
    yield t
    t.close()

# tests/test_via_ssh.py
def test_application_installed(target):
    """Application binary must be present and executable."""
    out, _, code = target.run("ls -la /usr/bin/myapp")
    assert code == 0, "myapp not found"
    assert "-rwx" in out or "-r-x" in out, "myapp not executable"

def test_service_running(target):
    """System service must be running."""
    out = target.run_assert_ok("systemctl is-active myservice")
    assert "active" in out.strip(), f"Service not active: {out}"

def test_application_starts_cleanly(target):
    """Application should start and respond to status query."""
    target.run("killall myapp 2>/dev/null || true")
    
    out, err, code = target.run("/usr/bin/myapp --version")
    assert code == 0, f"App failed to start: {err}"
    assert "v" in out, "Version string not in output"

def test_syslog_no_errors(target):
    """System log should not contain critical errors."""
    out = target.run_assert_ok("journalctl -p err --since '5 minutes ago' --no-pager")
    
    critical_errors = [
        "segfault",
        "kernel BUG",
        "Memory leak",
    ]
    
    for error in critical_errors:
        assert error not in out, f"Critical error in syslog: {error}"

# In local.conf
INHERIT += "testimage"
TEST_SUITES = "ping auto"

# Build and run tests
bitbake core-image-minimal -c testimage

# meta-myboard/lib/oeqa/runtime/cases/myapp_test.py
from oeqa.runtime.case import OERuntimeTestCase

class MyAppTest(OERuntimeTestCase):
    """Yocto testimage test for myapp."""
    
    def test_myapp_installed(self):
        """myapp should be installed."""
        status, output = self.target.run("which myapp")
        self.assertEqual(status, 0, "myapp not installed")
    
    def test_myapp_version(self):
        """myapp should report expected version."""
        status, output = self.target.run("myapp --version")
        self.assertEqual(status, 0)
        self.assertIn("1.2.3", output, f"Wrong version: {output}")
    
    def test_myapp_configuration_valid(self):
        """Default config file should be syntactically valid."""
        status, output = self.target.run("myapp --check-config")
        self.assertEqual(status, 0, f"Config invalid: {output}")

def test_boot_time_within_budget(qemu_session):
    """System should boot within timing budget."""
    # systemd-analyze provides boot timing breakdown
    out, _, code = ssh_target.run("systemd-analyze")
    
    if code != 0:
        pytest.skip("systemd not available")
    
    # Parse total boot time
    import re
    match = re.search(r"graphical.target reached after ([\d.]+)s", out)
    if match:
        boot_time_s = float(match.group(1))
        assert boot_time_s <= 15.0, f"Boot took {boot_time_s}s, exceeds 15s budget"
    
    # Find slow services
    out, _, _ = ssh_target.run("systemd-analyze blame | head -5")
    print(f"Slowest services:\n{out}")

def test_kernel_boot_time(qemu_session):
    """Kernel boot phase should complete within 3 seconds."""
    out = qemu_session.run_assert_ok("dmesg | tail -1")
    
    import re
    match = re.search(r"\[\s*([\d.]+)\]", out)
    if match:
        last_dmesg_s = float(match.group(1))
        # dmesg timestamps are seconds since kernel start

def test_available_memory(target):
    """Should have enough free memory for operation."""
    out = target.run_assert_ok("free -m")
    
    import re
    match = re.search(r"Mem:\s+\d+\s+\d+\s+(\d+)", out)
    assert match, f"Could not parse free output: {out}"
    
    free_mb = int(match.group(1))
    assert free_mb >= 50, f"Only {free_mb}MB free — system may be too constrained"

def test_disk_usage(target):
    """Root filesystem should have headroom."""
    out = target.run_assert_ok("df -h /")
    
    import re
    match = re.search(r"(\d+)%", out)
    assert match, f"Could not parse df output: {out}"
    
    usage_pct = int(match.group(1))
    assert usage_pct <= 80, f"Root filesystem {usage_pct}% full — too little headroom"

def test_no_zombie_processes(target):
    """Should not have zombie processes after steady state."""
    out = target.run_assert_ok("ps aux | grep ' Z '")
    # ps grep includes itself, so filter it out
    zombies = [l for l in out.splitlines() if " Z " in l and "grep" not in l]
    
    assert len(zombies) == 0, f"Zombie processes found:\n{chr(10).join(zombies)}"

# .github/workflows/embedded-image-test.yml
name: Embedded Image Tests

on:
  push:
    paths: ['recipes/**', 'conf/**', 'tests/**']

jobs:
  build-and-test:
    runs-on: ubuntu-22.04
    
    steps:
      - uses: actions/checkout@v4
      
      - name: Download pre-built image
        run: |
          # Pull from artifact storage (don't rebuild in CI)
          aws s3 cp s3://my-builds/${{ github.sha }}/core-image-minimal-qemux86-64.wic .
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      
      - name: Install QEMU
        run: |
          sudo apt-get install -y qemu-system-x86 python3-pexpect openssh-client
      
      - name: Boot image in QEMU (background)
        run: |
          qemu-system-x86_64 \
            -drive file=core-image-minimal-qemux86-64.wic,format=raw \
            -m 512 \
            -nographic \
            -net nic \
            -net user,hostfwd=tcp::2222-:22 \
            -daemonize \
            -pidfile /tmp/qemu.pid
      
      - name: Wait for SSH
        run: |
          timeout 120 bash -c \
            'until ssh -p 2222 -o StrictHostKeyChecking=no root@localhost true 2>/dev/null; do sleep 2; done'
      
      - name: Run tests
        run: |
          pip install paramiko pytest
          pytest tests/test_via_ssh.py tests/test_embedded_image.py -v
      
      - name: Stop QEMU
        if: always()
        run: kill $(cat /tmp/qemu.pid) || true