.ci/ruff.sh

@@ -8,15 +8,19 @@ if [ "$(id -u)" = 0 ]; then
 	exec su "${TESTUSER:-build}" -c "sh -e $0"
 fi
 
+DID_FAIL=0
+
 set -x
 
 # __init__.py with additional ignore:
 # F401: imported, but not used
 # shellcheck disable=SC2046
-ruff check --ignore "F401" $(find . -not -path '*/venv/*' -name '__init__.py')
+ruff check --ignore "F401" $(find . -not -path '*/venv/*' -name '__init__.py') || DID_FAIL=1
 
 # Check all other files
-ruff check --exclude=__init__.py .
+ruff check --exclude=__init__.py . || DID_FAIL=1
 
 # Check formatting
-ruff format --diff
+ruff format --diff || DID_FAIL=1
+
+exit $DID_FAIL

pmb/chroot/test_apk.py

@@ -21,7 +21,7 @@ def apk_mocks(monkeypatch) -> dict | None:
 
     def _apkindex_package(
         _package: str, _arch: Arch, _must_exist: bool = False, indexes=None
-    ) -> ApkindexBlock:
+    ) -> ApkindexBlock | None:
         if _package == "package1":
             return ApkindexBlock(
                 arch=_arch,
@@ -68,6 +68,8 @@ def apk_mocks(monkeypatch) -> dict | None:
                 version="5.5-r0",
             )
 
+        return None
+
     monkeypatch.setattr(pmb.parse.apkindex, "package", _apkindex_package)
     return None
 

pmb/config/__init__.py

@@ -211,7 +211,11 @@ chroot_outdated = 3600 * 24 * 2
 # for the first time
 # IMPORTANT: the order here matters, it is the order these packages will
 # be built in (if needed). abuild must be first!
-build_packages = ["abuild", "apk-tools", "build-base", "ccache", "git"]
+#
+# NOTE: full hexdump is installed to workaround a bug in busybox,
+# see https://gitlab.postmarketos.org/postmarketOS/pmaports/-/issues/3268. This can be
+# reverted when the bug is properly fixed.
+build_packages = ["abuild", "apk-tools", "build-base", "ccache", "git", "hexdump"]
 
 #
 # PARSE

pmb/core/crosstool.py

-# Copyright 2024 Caleb Connolly
-# SPDX-License-Identifier: GPL-3.0-or-later
-
-import enum
-from pathlib import Path
-
-from pmb.core.chroot import ChrootType
-from pmb.types import PathString
-
-
-class CrossToolTarget(enum.Enum):
-    BUILDROOT = 0
-    ROOTFS = 1
-
-
-class CrossTool:
-    __target: CrossToolTarget
-    __package: str
-    __paths: list[Path]
-
-    def __init__(self, target: CrossToolTarget, package: str, paths: list[PathString]):
-        self.__target = target
-        self.__package = package
-        self.__paths = list(map(lambda p: Path(p) if isinstance(p, str) else p, paths))
-
-    def __repr__(self) -> str:
-        return f"CrossTool({self.__target}, {self.__package}, {self.__paths})"
-
-    @property
-    def package(self) -> str:
-        return self.__package
-
-    @property
-    def paths(self) -> list[Path]:
-        return self.__paths
-
-    def should_install(self, target: ChrootType) -> bool:
-        if target == ChrootType.BUILDROOT and self.__target == CrossToolTarget.BUILDROOT:
-            return True
-        if (
-            target == ChrootType.ROOTFS
-            or target == ChrootType.INSTALLER
-            and self.__target == CrossToolTarget.ROOTFS
-        ):
-            return True
-
-        return False

pmb/parse/apkindex.py

@@ -91,13 +91,23 @@ def parse_next_block(path: Path, lines: list[str]) -> ApkindexBlock | None:
                 ret[key].append(value)
         else:
             ret[key] = []
+    provider_priority = ret.get("provider_priority")
+    if provider_priority:
+        if not provider_priority.isdigit():
+            raise RuntimeError(
+                f"Invalid provider_priority: '{provider_priority}' parsing block {ret}"
+            )
+        provider_priority = int(provider_priority)
+    else:
+        provider_priority = None
+
     return ApkindexBlock(
         arch=Arch.from_str(ret["arch"]),
         depends=ret["depends"],
         origin=ret.get("origin"),
         pkgname=ret["pkgname"],
         provides=ret["provides"],
-        provider_priority=ret.get("provider_priority"),
+        provider_priority=provider_priority,
         timestamp=ret.get("timestamp"),
         version=ret["version"],
     )
@@ -306,7 +316,7 @@ def parse_blocks(path: Path) -> list[ApkindexBlock]:
 
 
 def cache_key(path: Path) -> str:
-    return str(path.relative_to(get_context().config.work))
+    return str(path.relative_to(get_context().config.work, walk_up=True))
 
 
 def clear_cache(path: Path) -> bool:

pmb/parse/test_apkindex.py

+from pmb.core.arch import Arch
+import pytest
+
+from .apkindex import parse as parse_apkindex
+
+example_apkindex = """
+C:Q1p+nGf5oBAmbU9FQvV4MhfEmWqVE=
+P:postmarketos-base-ui-networkmanager
+V:29-r1
+A:aarch64
+S:4185
+I:5376
+T:Meta package for minimal postmarketOS UI base
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+o:postmarketos-base-ui
+m:Clayton Craft <clayton@craftyguy.net>
+t:1729538699
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+D:busctl dnsmasq-dnssec-dbus>=2.90-r1 gojq networkmanager networkmanager-cli networkmanager-tui networkmanager-wifi networkmanager-wwan networkmanager-dnsmasq
+
+C:Q1j8i5C7tOHbhrd5OTtot1u1ZtteU=
+P:postmarketos-base-ui-networkmanager-usb-tethering
+V:29-r1
+A:aarch64
+S:3986
+I:5206
+T:Meta package for minimal postmarketOS UI base
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+o:postmarketos-base-ui
+m:Clayton Craft <clayton@craftyguy.net>
+t:1729538699
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+D:chrony dbus haveged nftables openssh-server-pam pipewire postmarketos-artwork-icons postmarketos-base shadow tzdata util-linux wireless-regdb wireplumber
+i:postmarketos-base-ui-networkmanager=29-r1
+
+C:Q10FDERw8SRwuk6ITfWNyJ4GbVE2I=
+P:postmarketos-base-ui-openrc
+V:29-r1
+A:aarch64
+S:1570
+I:1
+T:Meta package for minimal postmarketOS UI base
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+o:postmarketos-base-ui
+m:Clayton Craft <clayton@craftyguy.net>
+t:1729538699
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+D:chrony-openrc dbus-openrc haveged-openrc util-linux-openrc /bin/sh
+i:postmarketos-base-ui=29-r1 openrc
+
+C:Q1inMHA1+gsS8U50j8odasUNy5TVw=
+P:postmarketos-base-ui-openrc-settingsd
+V:29-r1
+A:aarch64
+S:1824
+I:103
+T:Meta package for minimal postmarketOS UI base
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+o:postmarketos-base-ui
+m:Clayton Craft <clayton@craftyguy.net>
+t:1729538699
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+D:chrony dbus haveged nftables openssh-server-pam pipewire postmarketos-artwork-icons postmarketos-base shadow tzdata util-linux wireless-regdb wireplumber /bin/sh
+i:postmarketos-base-ui=29-r1 openrc-settingsd-openrc
+
+C:Q1iB2PiTO0w29T4kRUSjqLOyKcCwY=
+P:postmarketos-base-ui-qt-tweaks
+V:29-r1
+A:aarch64
+S:1661
+I:47
+T:Meta package for minimal postmarketOS UI base
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+o:postmarketos-base-ui
+m:Clayton Craft <clayton@craftyguy.net>
+t:1729538699
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+D:chrony dbus haveged nftables openssh-server-pam pipewire postmarketos-artwork-icons postmarketos-base shadow tzdata util-linux wireless-regdb wireplumber
+
+C:Q1aQ5UEOC902h2atx0fqDhahpkDY8=
+P:postmarketos-base-ui-qt-wayland
+V:29-r1
+A:aarch64
+S:1687
+I:86
+T:Meta package for minimal postmarketOS UI base
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+o:postmarketos-base-ui
+m:Clayton Craft <clayton@craftyguy.net>
+t:1729538699
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+D:chrony dbus haveged nftables openssh-server-pam pipewire postmarketos-artwork-icons postmarketos-base shadow tzdata util-linux wireless-regdb wireplumber
+
+C:Q16q7k1Z5HAJe5qqeXfTIn62QQoBs=
+P:postmarketos-base-ui-tinydm
+V:29-r1
+A:aarch64
+S:2264
+I:571
+T:Meta package for minimal postmarketOS UI base
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+o:postmarketos-base-ui
+m:Clayton Craft <clayton@craftyguy.net>
+t:1729538699
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+D:chrony dbus haveged nftables openssh-server-pam pipewire postmarketos-artwork-icons postmarketos-base shadow tzdata util-linux wireless-regdb wireplumber
+p:postmarketos-base-tinydm=29-r1
+i:postmarketos-base-ui=29-r1 tinydm-openrc
+
+C:Q11IrmIOlqZX5BzKhLsce+U+S+N3I=
+P:postmarketos-base-ui-wayland
+V:29-r1
+A:aarch64
+S:1361
+I:0
+T:Meta package for minimal postmarketOS UI base
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+o:postmarketos-base-ui
+m:Clayton Craft <clayton@craftyguy.net>
+t:1729538699
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+D:chrony dbus haveged nftables openssh-server-pam pipewire postmarketos-artwork-icons postmarketos-base shadow tzdata util-linux wireless-regdb wireplumber
+i:postmarketos-base-ui=29-r1 wayland-server-libs
+
+C:Q1bDAQG9F9mi7Mi5CT3Csiq4QU52w=
+P:postmarketos-base-ui-wifi-iwd
+V:29-r1
+A:aarch64
+S:1650
+I:26
+T:Use iwd as the WiFi backend (but may not work with all devices)
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+o:postmarketos-base-ui
+m:Clayton Craft <clayton@craftyguy.net>
+t:1729538699
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+k:90
+D:iwd
+p:postmarketos-base-ui-wifi=29-r1
+
+C:Q1s7shLQ3sKNxKABRyQRBQ11RwkuU=
+P:postmarketos-base-ui-wifi-iwd-openrc
+V:29-r1
+A:aarch64
+S:1405
+I:1
+T:Meta package for minimal postmarketOS UI base
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+o:postmarketos-base-ui
+m:Clayton Craft <clayton@craftyguy.net>
+t:1729538699
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+D:iwd-openrc openrc /bin/sh
+i:postmarketos-base-ui-wifi-iwd=29-r1 openrc
+
+C:Q1C23f0J5PBrNAuZiAhzCJhVpNe5c=
+P:postmarketos-base-ui-wifi-wpa_supplicant
+V:29-r1
+A:aarch64
+S:1262
+I:0
+T:Use wpa_supplicant as the WiFi backend.
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+o:postmarketos-base-ui
+m:Clayton Craft <clayton@craftyguy.net>
+t:1729538699
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+k:100
+D:wpa_supplicant
+p:postmarketos-base-ui-wifi=29-r1
+
+C:Q1LfWeOxxgIM3UE/QQBczMpJw17hY=
+P:postmarketos-base-ui-wifi-wpa_supplicant-openrc
+V:29-r1
+A:aarch64
+S:1691
+I:37
+T:Meta package for minimal postmarketOS UI base
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+o:postmarketos-base-ui
+m:Clayton Craft <clayton@craftyguy.net>
+t:1729538699
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+D:wpa_supplicant-openrc openrc /bin/sh
+i:postmarketos-base-ui-wifi-wpa_supplicant=29-r1 openrc
+
+C:Q1yB3CVUFMOjnLOOEAUIUUpJJV8g0=
+P:postmarketos-base-ui-x11
+V:29-r1
+A:aarch64
+S:1587
+I:22
+T:Meta package for minimal postmarketOS UI base
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+o:postmarketos-base-ui
+m:Clayton Craft <clayton@craftyguy.net>
+t:1729538699
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+D:libinput xf86-input-libinput xf86-video-fbdev
+p:postmarketos-base-x11=29-r1
+i:postmarketos-base-ui=29-r1 xorg-server
+
+C:Q1TgE1jgGt1PUb/Zwbi6rDRt3b8go=
+P:postmarketos-initramfs
+V:3.3.5-r2
+A:aarch64
+S:16530
+I:143360
+T:Base files for the postmarketOS initramfs / initramfs-extra
+U:https://postmarketos.org
+L:GPL-2.0-or-later
+o:postmarketos-initramfs
+m:Caleb Connolly <caleb@postmarketos.org>
+t:1728049308
+c:f3a4285732781d5577bad06046b7bbeaf132ce1f-dirty
+k:10
+D:blkid btrfs-progs buffyboard busybox-extras bzip2 cryptsetup device-mapper devicepkg-utils>=0.2.0 dosfstools e2fsprogs e2fsprogs-extra f2fs-tools font-terminus iskey kmod libinput-libs lz4 multipath-tools parted postmarketos-fde-unlocker postmarketos-mkinitfs>=2.2 udev unudhcpd util-linux-misc xz
+p:postmarketos-ramdisk=3.3.5-r2"""
+
+
+def test_apkindex_parse(tmp_path) -> None:
+    tmpfile = tmp_path / "APKINDEX.1"
+    print(tmp_path)
+    f = open(tmpfile, "w")
+    f.write(example_apkindex)
+    f.close()
+
+    blocks = parse_apkindex(tmpfile, True)
+    for k, v in blocks.items():
+        print(f"{k}: {v}")
+
+    # Even though there's only 14 entries, there are some virtual packages
+    assert len(blocks) == 18
+
+    # Check that the postmarketos-ramdisk virtual package is handled correctly
+    # and that it's one provider (postmarketos-initramfs) is declared
+    assert "postmarketos-ramdisk" in blocks.keys()
+    assert "postmarketos-initramfs" in blocks["postmarketos-ramdisk"].keys()
+    assert (
+        blocks["postmarketos-ramdisk"]["postmarketos-initramfs"]
+        == blocks["postmarketos-initramfs"]["postmarketos-initramfs"]
+    )
+
+    initramfs = blocks["postmarketos-initramfs"]["postmarketos-initramfs"]
+    assert initramfs.pkgname == "postmarketos-initramfs"
+    assert initramfs.provides == ["postmarketos-ramdisk"]
+    assert initramfs.provider_priority == 10
+    assert initramfs.depends == [
+        "blkid",
+        "btrfs-progs",
+        "buffyboard",
+        "busybox-extras",
+        "bzip2",
+        "cryptsetup",
+        "device-mapper",
+        "devicepkg-utils",
+        "dosfstools",
+        "e2fsprogs",
+        "e2fsprogs-extra",
+        "f2fs-tools",
+        "font-terminus",
+        "iskey",
+        "kmod",
+        "libinput-libs",
+        "lz4",
+        "multipath-tools",
+        "parted",
+        "postmarketos-fde-unlocker",
+        "postmarketos-mkinitfs",
+        "udev",
+        "unudhcpd",
+        "util-linux-misc",
+        "xz",
+    ]
+
+    tinydm = blocks["postmarketos-base-ui-tinydm"]["postmarketos-base-ui-tinydm"]
+    # Without the version!
+    assert tinydm.provides == ["postmarketos-base-tinydm"]
+    assert tinydm.version == "29-r1"
+    assert tinydm.arch == Arch.aarch64
+
+    wayland = blocks["postmarketos-base-ui-wayland"]["postmarketos-base-ui-wayland"]
+    # Doesn't provide an explicit version
+    assert wayland.provides == []
+    assert wayland.origin == "postmarketos-base-ui"
+
+    networkmanager = blocks["postmarketos-base-ui-networkmanager"][
+        "postmarketos-base-ui-networkmanager"
+    ]
+    assert networkmanager.provider_priority is None
+
+
+def test_apkindex_parse_bad_priority(tmp_path) -> None:
+    tmpfile = tmp_path / "APKINDEX.2"
+    f = open(tmpfile, "w")
+    # A snippet of the above example but with the provider_priority
+    # of postmarketos-initramfs set to a non-integer value
+    f.write(
+        """
+C:Q1yB3CVUFMOjnLOOEAUIUUpJJV8g0=
+P:postmarketos-base-ui-x11
+V:29-r1
+A:aarch64
+S:1587
+I:22
+T:Meta package for minimal postmarketOS UI base
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+o:postmarketos-base-ui
+m:Clayton Craft <clayton@craftyguy.net>
+t:1729538699
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+D:libinput xf86-input-libinput xf86-video-fbdev
+p:postmarketos-base-x11=29-r1
+i:postmarketos-base-ui=29-r1 xorg-server
+
+C:Q1TgE1jgGt1PUb/Zwbi6rDRt3b8go=
+P:postmarketos-initramfs
+V:3.3.5-r2
+A:aarch64
+S:16530
+I:143360
+T:Base files for the postmarketOS initramfs / initramfs-extra
+U:https://postmarketos.org
+L:GPL-2.0-or-later
+o:postmarketos-initramfs
+m:Caleb Connolly <caleb@postmarketos.org>
+t:1728049308
+c:f3a4285732781d5577bad06046b7bbeaf132ce1f-dirty
+k:beep
+D:blkid btrfs-progs buffyboard busybox-extras bzip2 cryptsetup device-mapper devicepkg-utils>=0.2.0 dosfstools e2fsprogs e2fsprogs-extra f2fs-tools font-terminus iskey kmod libinput-libs lz4 multipath-tools parted postmarketos-fde-unlocker postmarketos-mkinitfs>=2.2 udev unudhcpd util-linux-misc xz
+p:postmarketos-ramdisk=3.3.5-r2"""
+    )
+    f.close()
+
+    # We expect a RuntimeError to be raised when provider_priority is not
+    # an integer
+    with pytest.raises(RuntimeError):
+        parse_apkindex(tmpfile, True)
+
+
+def test_apkindex_parse_missing_optionals(tmp_path) -> None:
+    tmpfile = tmp_path / "APKINDEX.3"
+    f = open(tmpfile, "w")
+    # A snippet of the above example but with a missing timestamp
+    # and origin fields
+    f.write(
+        """
+C:Q1yB3CVUFMOjnLOOEAUIUUpJJV8g0=
+P:postmarketos-base-ui-x11
+V:29-r1
+A:aarch64
+S:1587
+I:22
+T:Meta package for minimal postmarketOS UI base
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+m:Clayton Craft <clayton@craftyguy.net>
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+D:libinput xf86-input-libinput xf86-video-fbdev
+p:postmarketos-base-x11=29-r1
+i:postmarketos-base-ui=29-r1 xorg-server"""
+    )
+    f.close()
+
+    # We expect parsing to succeed when the timestamp is missing
+    parse_apkindex(tmpfile, True)
+
+
+def test_apkindex_parse_trailing_newline(tmp_path) -> None:
+    tmpfile = tmp_path / "APKINDEX.4"
+    f = open(tmpfile, "w")
+    # A snippet of the above example but with additional
+    # trailing newlines
+    f.write(
+        """
+C:Q1yB3CVUFMOjnLOOEAUIUUpJJV8g0=
+P:postmarketos-base-ui-x11
+V:29-r1
+A:aarch64
+S:1587
+I:22
+T:Meta package for minimal postmarketOS UI base
+U:https://postmarketos.org
+L:GPL-3.0-or-later
+m:Clayton Craft <clayton@craftyguy.net>
+c:901cb9520450a1e88ded95ac774e83f6b2cfbba3-dirty
+D:libinput xf86-input-libinput xf86-video-fbdev
+p:postmarketos-base-x11=29-r1
+i:postmarketos-base-ui=29-r1 xorg-server
+
+
+"""
+    )
+    f.close()
+
+    # We expect parsing to succeed when the timestamp is missing
+    parse_apkindex(tmpfile, True)