[PATCH 0/3] test: test boot with Debian netinstaller CDROM image

[PATCH 0/3] test: test boot with Debian netinstaller CDROM image

[Ahmad Fatoum]

The nocloud image takes quite long to boot and now includes
systemd-firstboot, which doesn't lend itself that well to automated
testing.

Switch to the CDROM image instead. That one doesn't autoboot
unfortunately and opens a screen, so we need a few workarounds for that,
but the tests work again now.

The end goal remains building a dedicated image for testing instead.

Ahmad Fatoum (3):
  test: py: linux: use utilities available in Debian netinstall images
  test: test boot with Debian netinstaller CDROM image
  test: py: linux: remove test_ prefix for intermediate functions

 .github/workflows/test-labgrid-pytest.yml  |  2 +-
 scripts/fetch-os.sh                        |  2 +-
 test/arm/multi_v8_efiloader_defconfig.yaml | 12 ++--
 test/py/helper.py                          | 39 ++++++++++++
 test/py/test_linux_efiloader.py            | 73 +++++++++++-----------
 test/py/test_linux_smbios.py               | 27 +++++---
 test/strategy.py                           | 64 +++++++++++++++----
 7 files changed, 156 insertions(+), 63 deletions(-)

-- 
2.47.3

[PATCH 1/3] test: py: linux: use utilities available in Debian netinstall images

[Ahmad Fatoum]

The Debian CDROMs for netinstall on ARM64 don't have journalctl (no
systemd) and no stat or strings.

Adapt the tests we have to use the commands available within the busybox
included on the CDROM images.

Signed-off-by: Ahmad Fatoum <a.fatoum@pengutronix.de>
---
 test/py/test_linux_efiloader.py | 19 +++++++++----------
 test/py/test_linux_smbios.py    | 11 ++++++-----
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/test/py/test_linux_efiloader.py b/test/py/test_linux_efiloader.py
index b9ea82d27c0d..385f3fde4c30 100644
--- a/test/py/test_linux_efiloader.py
+++ b/test/py/test_linux_efiloader.py
@@ -4,13 +4,11 @@ import re
 import pytest
 
 
-def get_journalctl(shell, kernel=True, grep=None):
-    opts = ''
+def get_dmesg(shell, grep=None):
+    cmd = 'dmesg'
     if grep is not None:
-        opts += f" --grep={grep}"
-    if kernel:
-        opts += " -k"
-    stdout, _, ret = shell.run(f"journalctl --no-pager {opts} -o cat")
+        cmd += f" | grep '{grep}'"
+    stdout, _, ret = shell.run(cmd)
     assert ret == 0
     return stdout
 
@@ -56,11 +54,11 @@ def test_boot_manual_with_initrd(strategy, barebox, env, efiloader):
         shell.run_check("grep -q apparmor=0 /proc/cmdline")
 
         initrd_freed = any("Freeing initrd memory"
-                           in line for line in get_journalctl(shell, 'initrd'))
+                           in line for line in get_dmesg(shell, 'initrd'))
         assert initrd_freed, "initrd was not loaded or freed"
 
         # Verify we booted to shell
-        dmesg = get_journalctl(shell, 'efi')
+        dmesg = get_dmesg(shell, 'efi')
 
         uefi_not_found = re.search("efi: UEFI not found.",
                                    "\n".join(dmesg)) is not None
@@ -81,14 +79,14 @@ def test_boot_manual_with_initrd(strategy, barebox, env, efiloader):
 
 @pytest.mark.lg_feature(['bootable', 'efi'])
 def test_efi_kernel_no_warn(shell):
-    stdout, stderr, ret = shell.run("journalctl -k --no-pager --grep efi -o cat -p warning")
+    stdout, stderr, ret = shell.run("dmesg -r | grep '<[0-4]>.*\\<efi\\>'")
     assert stdout == []
     assert stderr == []
 
 
 @pytest.mark.lg_feature(['bootable', 'efi'])
 def test_expected_efi_messages(shell, env):
-    dmesg = get_journalctl(shell, 'efi')
+    dmesg = get_dmesg(shell, 'efi')
 
     expected_patterns = [
         r"efi:\s+EFI v2\.8 by barebox",
@@ -122,6 +120,7 @@ def test_efi_systab(shell, env):
 @pytest.mark.lg_feature(['bootable', 'efi'])
 def test_efivars_filesystem_not_empty(shell):
     # Directory must not be empty
+    shell.run("mount -t efivarfs efivarfs /sys/firmware/efi/efivars")
     stdout, _, ret = shell.run("ls -1 /sys/firmware/efi/efivars")
     assert ret == 0
 
diff --git a/test/py/test_linux_smbios.py b/test/py/test_linux_smbios.py
index d0c2b7ca5665..97a6ae3c80a4 100644
--- a/test/py/test_linux_smbios.py
+++ b/test/py/test_linux_smbios.py
@@ -8,15 +8,16 @@ def test_smbios3_tables_present(shell):
     _, _, ret = shell.run("test -e /sys/firmware/dmi/tables/smbios_entry_point")
     assert ret == 0, "SMBIOS entry point not found"
 
-    [stdout], _, ret = shell.run("stat -c '%s' /sys/firmware/dmi/tables/DMI")
+    [stdout], _, ret = shell.run("wc -c </sys/firmware/dmi/tables/DMI")
     assert ret == 0
 
     size = int(stdout)
     assert size > 0, "SMBIOS DMI table is empty"
 
-    [stdout], _, ret = shell.run("dd if=/sys/firmware/dmi/tables/smbios_entry_point bs=1 count=5 2>/dev/null")
-    assert ret == 0
-    assert stdout == "_SM3_", "SMBIOS entry point is not SMBIOS 3.x"
+    shell.run_check("echo _SM3_ >/tmp/sm3")
+    stdout, _, ret = shell.run("cmp --bytes 5 /tmp/sm3 /sys/firmware/dmi/tables/smbios_entry_point")
+    assert stdout == []
+    assert ret == 0, "SMBIOS entry point is not SMBIOS 3.x"
 
 
 @pytest.mark.lg_feature(['bootable', 'smbios'])
@@ -26,5 +27,5 @@ def test_smbios_contains_barebox(shell):
     This avoids dmidecode and relies on simple string matching.
     """
     # The DMI table is binary; strings are still ASCII embedded
-    stdout, _, ret = shell.run("strings /sys/firmware/dmi/tables/DMI | grep -i barebox")
+    stdout, _, ret = shell.run("grep -a barebox /sys/firmware/dmi/tables/DMI")
     assert len(stdout) > 0, "barebox not found in SMBIOS/DMI tables"
-- 
2.47.3

[PATCH 2/3] test: test boot with Debian netinstaller CDROM image

[Ahmad Fatoum]

The nocloud image takes quite long to boot and now includes
systemd-firstboot, which doesn't lend itself that well to automated
testing.

Switch to the CDROM image instead. That one doesn't autoboot
unfortunately and opens a screen, so we need a few workarounds for that,
but the tests work again now.

The end goal remains building a dedicated image for testing instead.

Signed-off-by: Ahmad Fatoum <a.fatoum@pengutronix.de>
---
 .github/workflows/test-labgrid-pytest.yml  |  2 +-
 scripts/fetch-os.sh                        |  2 +-
 test/arm/multi_v8_efiloader_defconfig.yaml | 12 ++--
 test/py/helper.py                          | 39 +++++++++++++
 test/py/test_linux_efiloader.py            | 33 ++++++-----
 test/py/test_linux_smbios.py               | 16 +++++-
 test/strategy.py                           | 64 +++++++++++++++++-----
 7 files changed, 133 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/test-labgrid-pytest.yml b/.github/workflows/test-labgrid-pytest.yml
index 7e385af65245..9f459fe2c30e 100644
--- a/.github/workflows/test-labgrid-pytest.yml
+++ b/.github/workflows/test-labgrid-pytest.yml
@@ -35,7 +35,7 @@ jobs:
           - ARCH: arm
             lgenv: test/arm/multi_v8_efiloader_defconfig.yaml
             defconfig: multi_v8_efiloader_defconfig
-            osimg: debian-13-nocloud-arm64.qcow2
+            osimg: debian-13.3.0-arm64-netinst.iso
 
           - ARCH: mips
             lgenv: test/mips/qemu-malta_defconfig.yaml
diff --git a/scripts/fetch-os.sh b/scripts/fetch-os.sh
index 649e4efe9a05..e22185af2a00 100755
--- a/scripts/fetch-os.sh
+++ b/scripts/fetch-os.sh
@@ -8,7 +8,7 @@ fi
 set -euo pipefail
 
 declare -A images=(
-	["debian-13-nocloud-arm64.qcow2"]="https://cloud.debian.org/images/cloud/trixie/latest/debian-13-nocloud-arm64.qcow2"
+	["debian-13.3.0-arm64-netinst.iso"]="https://cloud.debian.org/images/release/current/arm64/iso-cd/debian-13.3.0-arm64-netinst.iso"
 )
 
 found=0
diff --git a/test/arm/multi_v8_efiloader_defconfig.yaml b/test/arm/multi_v8_efiloader_defconfig.yaml
index b24058da8e99..579ac9f50d25 100644
--- a/test/arm/multi_v8_efiloader_defconfig.yaml
+++ b/test/arm/multi_v8_efiloader_defconfig.yaml
@@ -9,17 +9,18 @@ targets:
         kernel: barebox-dt-2nd.img
         display: qemu-default
         extra_args: >
-          -drive if=virtio,snapshot=on,file=debian-13-nocloud-arm64.qcow2
+          -drive if=virtio,snapshot=on,file=debian-13.3.0-arm64-netinst.iso
           -device virtio-rng
       BareboxDriver:
         prompt: 'barebox@[^:]+:[^ ]+ '
         autoboot: 'stop autoboot'
         bootstring: 'commandline:|Starting EFI payload at'
       ShellDriver:
-        prompt: '\x1b\[\?2004hroot@[^:]+:[^ ]+'
+        prompt: '~ # '
         login_prompt: ' login: '
         login_timeout: 300
         await_login_timeout: 20
+        post_login_settle_time: 3
         username: 'root'
       BareboxTestStrategy: {}
     features:
@@ -27,12 +28,13 @@ targets:
       - smbios
       - bootable
       - efi
+      - testfs
     options:
-      root_dev: virtioblk0.0
-      bootm.image: 'boot/vmlinuz-*'
-      bootm.initrd: 'boot/initrd.img-*'
+      bootm.image: 'install.a64/vmlinuz'
+      bootm.initrd: 'install.a64/initrd.gz'
 
 images:
   barebox-dt-2nd.img: !template "$LG_BUILDDIR/images/barebox-dt-2nd.img"
+  debian_iso: !template "../../debian-13.3.0-arm64-netinst.iso"
 imports:
   -  ../strategy.py
diff --git a/test/py/helper.py b/test/py/helper.py
index 17d40c584679..ab615280048f 100644
--- a/test/py/helper.py
+++ b/test/py/helper.py
@@ -5,6 +5,7 @@ import pytest
 import os
 import re
 import shlex
+import subprocess
 
 
 def parse_config(lines):
@@ -197,3 +198,41 @@ def skip_disabled(config, *options):
 
         if bool(undefined):
             pytest.skip("skipping test due to disabled " + (",".join(undefined)) + " dependency")
+
+
+def ensure_debian_iso(env, destdir):
+    """
+    Extract Debian kernel and initrd from ISO into destdir.
+
+    The debian_iso specified under images in the YAML will be used
+    Files are extracted into destdir/install.a64/{vmlinuz,initrd.gz}
+    and skipped if they already exist.
+
+    Returns destdir, or None if the ISO doesn't exist.
+    """
+    iso_path = env.config.get_image_path("debian_iso")
+    if iso_path is None:
+        return None
+
+    outdir = os.path.join(destdir, "install.a64")
+    vmlinuz_path = os.path.join(outdir, "vmlinuz")
+    initrd_path = os.path.join(outdir, "initrd.gz")
+
+    if os.path.exists(vmlinuz_path) and os.path.exists(initrd_path):
+        return destdir
+
+    os.makedirs(outdir, exist_ok=True)
+
+    with open(vmlinuz_path, "wb") as f:
+        subprocess.run(
+            ["isoinfo", "-i", iso_path, "-x", "/INSTALL.A64/VMLINUZ.;1"],
+            stdout=f, check=True,
+        )
+
+    with open(initrd_path, "wb") as f:
+        subprocess.run(
+            ["isoinfo", "-i", iso_path, "-x", "/INSTALL.A64/INITRD.GZ;1"],
+            stdout=f, check=True,
+        )
+
+    return destdir
diff --git a/test/py/test_linux_efiloader.py b/test/py/test_linux_efiloader.py
index 385f3fde4c30..df50f21bffc0 100644
--- a/test/py/test_linux_efiloader.py
+++ b/test/py/test_linux_efiloader.py
@@ -1,8 +1,21 @@
 # SPDX-License-Identifier: GPL-2.0-only
+# Boot to Linux via EFI CDROM installer image
 
 import re
 import pytest
 
+from .helper import ensure_debian_iso
+
+fetchdir = "/mnt/9p/testfs"
+
+
+@pytest.fixture(scope="module")
+def debian_iso(env, testfs):
+    result = ensure_debian_iso(env, testfs)
+    if result is None:
+        pytest.skip("Debian ISO not found")
+    return result
+
 
 def get_dmesg(shell, grep=None):
     cmd = 'dmesg'
@@ -13,9 +26,9 @@ def get_dmesg(shell, grep=None):
     return stdout
 
 
-@pytest.mark.lg_feature(['bootable', 'efi'])
+@pytest.mark.lg_feature(['bootable', 'efi', 'testfs'])
 @pytest.mark.parametrize('efiloader', [False, True])
-def test_boot_manual_with_initrd(strategy, barebox, env, efiloader):
+def test_boot_manual_with_initrd(strategy, barebox, env, efiloader, debian_iso):
     """Test booting Debian kernel directly without GRUB"""
 
     barebox.run_check(f"global.bootm.efi={'required' if efiloader else 'disabled'}")
@@ -25,27 +38,22 @@ def test_boot_manual_with_initrd(strategy, barebox, env, efiloader):
         return config.get_target_option(strategy.target.name, opt)
 
     try:
-        root_dev = get_option(strategy, "root_dev")
         kernel_path = get_option(strategy, "bootm.image")
     except KeyError:
-        pytest.fail("Feature bootable enabled, but root_dev/bootm.image option missing.")  # noqa
+        pytest.fail("Feature bootable enabled, but bootm.image option missing.")
 
-    # Detect block devices
-    barebox.run_check("detect -a")
-    barebox.run_check(f"ls /mnt/{root_dev}/")
-
-    [kernel_path] = barebox.run_check(f"ls /mnt/{root_dev}/{kernel_path}")
+    kernel_path = f"{fetchdir}/{kernel_path}"
+    [kernel_path] = barebox.run_check(f"ls {kernel_path}")
 
     try:
         initrd_path = get_option(strategy, "bootm.initrd")
-        [initrd_path] = barebox.run_check(f"ls /mnt/{root_dev}/{initrd_path}")
+        initrd_path = f"{fetchdir}/{initrd_path}"
+        [initrd_path] = barebox.run_check(f"ls {initrd_path}")
         barebox.run_check(f"global.bootm.initrd={initrd_path}")
     except KeyError:
         pass
 
     barebox.run_check(f"global.bootm.image={kernel_path}")
-    barebox.run_check(f"global.bootm.root_dev=/dev/{root_dev}")
-    barebox.run_check("global.bootm.appendroot=1")
     # Speed up subsequent runs a bit
     barebox.run_check("global linux.bootargs.noapparmor=apparmor=0")
 
@@ -90,7 +98,6 @@ def test_expected_efi_messages(shell, env):
 
     expected_patterns = [
         r"efi:\s+EFI v2\.8 by barebox",
-        r"Remapping and enabling EFI services\.",
         r"efivars:\s+Registered efivars operations",
     ]
 
diff --git a/test/py/test_linux_smbios.py b/test/py/test_linux_smbios.py
index 97a6ae3c80a4..48051e29d0d0 100644
--- a/test/py/test_linux_smbios.py
+++ b/test/py/test_linux_smbios.py
@@ -2,8 +2,20 @@
 
 import pytest
 
+from .helper import ensure_debian_iso
 
-@pytest.mark.lg_feature(['bootable', 'smbios'])
+fetchdir = "/mnt/9p/testfs"
+
+
+@pytest.fixture(scope="module")
+def debian_iso(env, testfs):
+    result = ensure_debian_iso(env, testfs)
+    if result is None:
+        pytest.skip("Debian ISO not found")
+    return result
+
+
+@pytest.mark.lg_feature(['bootable', 'smbios', 'testfs'])
 def test_smbios3_tables_present(shell):
     _, _, ret = shell.run("test -e /sys/firmware/dmi/tables/smbios_entry_point")
     assert ret == 0, "SMBIOS entry point not found"
@@ -20,7 +32,7 @@ def test_smbios3_tables_present(shell):
     assert ret == 0, "SMBIOS entry point is not SMBIOS 3.x"
 
 
-@pytest.mark.lg_feature(['bootable', 'smbios'])
+@pytest.mark.lg_feature(['bootable', 'smbios', 'testfs'])
 def test_smbios_contains_barebox(shell):
     """
     Search raw SMBIOS/DMI tables for a barebox vendor string.
diff --git a/test/strategy.py b/test/strategy.py
index 35cda3aca498..3ad3cb2a45f2 100644
--- a/test/strategy.py
+++ b/test/strategy.py
@@ -1,16 +1,19 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
 import enum
+import time
 
 import attr
 import pytest
 import subprocess
 import re
 from contextlib import contextmanager
+import pexpect
 
 from labgrid import target_factory, step, driver
 from labgrid.strategy import Strategy, StrategyError
 from labgrid.util import labgrid_version
+from labgrid.util.timeout import Timeout
 
 match = re.match(r'^(\d+?)\.', labgrid_version())
 if match is None or int(match.group(1)) < 25:
@@ -74,11 +77,7 @@ class BareboxTestStrategy(Strategy):
             # interrupt barebox
             self.target.activate(self.barebox)
         elif status == Status.shell:
-            # transition to barebox
-            self.transition(Status.barebox)  # pylint: disable=missing-kwoa
-            self.barebox.boot("")
-            self.barebox.await_boot()
-            self.target.activate(self.shell)
+            self._boot_kernel("")
         else:
             raise StrategyError(
                 "no transition found from {} to {}".
@@ -113,18 +112,57 @@ class BareboxTestStrategy(Strategy):
             self.power.cycle()
             self.target.activate(self.barebox)
 
+    def skip_cdrom_installer(self):
+        # Wait for installer to boot. Send Enter every cycle to dismiss
+        # GRUB menu and any installer prompts. The screen TUI has a clock
+        # in the status bar that makes pattern matching unreliable, so
+        # avoid matching on it.
+        timeout = Timeout(float(self.shell.login_timeout))
+
+        while not timeout.expired:
+            self.console.sendline("")
+            index, _, _, _ = self.console.expect(
+                ["Starting system log daemon", pexpect.TIMEOUT],
+                timeout=self.shell.await_login_timeout)
+            if index == 0:
+                break
+
+        # Switch to screen window 3 (shell) and disable the status bar.
+        # Retry until the shell prompt appears.
+        while not timeout.expired:
+            self.console.sendline("\x013")
+            time.sleep(0.5)
+            self.console.sendline("\x01:hardstatus ignore")
+            time.sleep(0.5)
+            self.console.sendline("")
+            index, _, _, _ = self.console.expect(
+                [self.shell.prompt, pexpect.TIMEOUT],
+                timeout=self.shell.await_login_timeout)
+            if index == 0:
+                break
+
+    def _boot_kernel(self, boottarget=None, bootm=False, skip_cdrom_installer=True):
+        self.transition(Status.barebox)
+
+        if bootm:
+            self.barebox_bootm(boottarget)
+        else:
+            self.barebox.boot(boottarget)
+
+        self.barebox.await_boot()
+
+        if skip_cdrom_installer:
+            self.skip_cdrom_installer()
+
+        self.target.activate(self.shell)
+
     @contextmanager
-    def boot_kernel(self, boottarget=None, bootm=False):
+    def boot_kernel(self, boottarget=None, bootm=False, skip_cdrom_installer=True):
         self.transition(Status.barebox)
 
         try:
-            if bootm:
-                self.barebox_bootm(boottarget)
-            else:
-                self.barebox.boot(boottarget)
-
-            self.barebox.await_boot()
-            self.target.activate(self.shell)
+            self._boot_kernel(boottarget=boottarget, bootm=bootm,
+                              skip_cdrom_installer=skip_cdrom_installer)
             yield self.shell
         finally:
             self.target.deactivate(self.shell)
-- 
2.47.3

[PATCH 3/3] test: py: linux: remove test_ prefix for intermediate functions

[Ahmad Fatoum]

From: Ahmad Fatoum <a.fatoum@barebox.org>

These functions are not standalone tests, but they are called from
actual tests, so rename them to prevent pytest from collecting them.

Signed-off-by: Ahmad Fatoum <a.fatoum@pengutronix.de>
---
 test/py/test_linux_efiloader.py | 23 +++++++++--------------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/test/py/test_linux_efiloader.py b/test/py/test_linux_efiloader.py
index df50f21bffc0..a76e55a91e52 100644
--- a/test/py/test_linux_efiloader.py
+++ b/test/py/test_linux_efiloader.py
@@ -72,10 +72,10 @@ def test_boot_manual_with_initrd(strategy, barebox, env, efiloader, debian_iso):
                                    "\n".join(dmesg)) is not None
 
         if efiloader:
-            test_efi_kernel_no_warn(shell)
-            test_expected_efi_messages(shell, env)
-            test_efi_systab(shell, env)
-            test_efivars_filesystem_not_empty(shell)
+            check_efi_kernel_no_warn(shell)
+            check_expected_efi_messages(shell, env)
+            check_efi_systab(shell, env)
+            check_efivars_filesystem_not_empty(shell)
 
             assert not uefi_not_found, \
                    "EFI stub was not used despite global.bootm.efi=required"
@@ -85,15 +85,13 @@ def test_boot_manual_with_initrd(strategy, barebox, env, efiloader, debian_iso):
                    "EFI stub was used despite global.bootm.efi=disabled"
 
 
-@pytest.mark.lg_feature(['bootable', 'efi'])
-def test_efi_kernel_no_warn(shell):
-    stdout, stderr, ret = shell.run("dmesg -r | grep '<[0-4]>.*\\<efi\\>'")
+def check_efi_kernel_no_warn(shell):
+    stdout, stderr, _ = shell.run("dmesg -r | grep '<[0-4]>.*\\<efi\\>'")
     assert stdout == []
     assert stderr == []
 
 
-@pytest.mark.lg_feature(['bootable', 'efi'])
-def test_expected_efi_messages(shell, env):
+def check_expected_efi_messages(shell, env):
     dmesg = get_dmesg(shell, 'efi')
 
     expected_patterns = [
@@ -106,8 +104,7 @@ def test_expected_efi_messages(shell, env):
                f"Missing expected EFI message: {pattern}"
 
 
-@pytest.mark.lg_feature(['bootable', 'efi'])
-def test_efi_systab(shell, env):
+def check_efi_systab(shell, env):
     stdout, stderr, ret = shell.run("cat /sys/firmware/efi/systab")
     assert ret == 0
     assert stderr == []
@@ -124,9 +121,7 @@ def test_efi_systab(shell, env):
                f"Missing expected entry in systab : {pattern}"
 
 
-@pytest.mark.lg_feature(['bootable', 'efi'])
-def test_efivars_filesystem_not_empty(shell):
-    # Directory must not be empty
+def check_efivars_filesystem_not_empty(shell):
     shell.run("mount -t efivarfs efivarfs /sys/firmware/efi/efivars")
     stdout, _, ret = shell.run("ls -1 /sys/firmware/efi/efivars")
     assert ret == 0
-- 
2.47.3