.ci/docs.sh

@@ -14,12 +14,6 @@ if [ "$(id -u)" = 0 ]; then
 	exec su "${TESTUSER:-build}" -c "sh -e $0"
 fi
 
-# Require sphinx to be installed on the host system
-if [ -z "$(command -v sphinx-build)" ]; then
-	echo "ERROR: sphinx-build command not found, make sure it is in your PATH."
-	exit 1
-fi
-
 # Sanity check docs that all modules are documented.
 # Ignore all packages and files named test*
 fail=0

.gitlab-ci.yml

@@ -101,7 +101,7 @@ integration:
   before_script:
     - *global_before_scripts
     - apk upgrade -U
-    - apk add doas git losetup python3 openssl
+    - apk add doas git losetup multipath-tools python3 openssl
     - echo "permit nopass build" > /etc/doas.d/ci-user.conf
   script:
     - ".ci/integration.sh"

README.md

@@ -47,6 +47,8 @@ Issues are being tracked
 * For python3 < 3.11: tomli
 * OpenSSL
 * git
+* kpartx (from multipath-tools)
+* losetup (with --json support, e.g. util-linux version)
 * ps
 * tar
 

pmb/build/_package.py

@@ -480,18 +480,27 @@ def packages(
         channel = pmb.config.pmaports.read_config(aports)["channel"]
         index_data = pmb.parse.apkindex.package(name, arch, False)
         # Make sure we aren't building a package that will never be used! This can happen if
-        # building with --src with an outdated pmaports checkout.
+        # building with --src with an outdated pmaports checkout. Unless --force is used
+        # in which case we assume it was intentional.
         if (
             index_data
             and pmb.parse.version.compare(index_data.version, f"{pkgver}-r{apkbuild['pkgrel']}")
             == 1
         ):
-            raise NonBugError(
-                f"A binary package for {name} has a newer version ({index_data.version})"
-                f" than the source ({pkgver}-{apkbuild['pkgrel']}). Please ensure your pmaports branch is up"
-                " to date and that you don't have a newer version of the package in your local"
-                f" binary repo ({context.config.work / 'packages' / channel / pkg_arch})."
-            )
+            if force:
+                logging.warning(
+                    f"WARNING: A binary package for {name} has a newer version ({index_data.version})"
+                    f" than the source ({pkgver}-{apkbuild['pkgrel']}). The package to be build will"
+                    f" not be installed automatically, use 'apk add {name}={pkgver}-r{apkbuild['pkgrel']}'"
+                    " to install it."
+                )
+            else:
+                raise NonBugError(
+                    f"A binary package for {name} has a newer version ({index_data.version})"
+                    f" than the source ({pkgver}-{apkbuild['pkgrel']}). Please ensure your pmaports branch is up"
+                    " to date and that you don't have a newer version of the package in your local"
+                    f" binary repo ({context.config.work / 'packages' / channel / pkg_arch})."
+                )
         build_queue.append(
             {
                 "name": name,

pmb/build/other.py

@@ -182,7 +182,7 @@ def configure_abuild(chroot: Chroot, verify=False):
         for line in handle:
             if not line.startswith(prefix):
                 continue
-            if line != (prefix + jobs + "\n"):
+            if line != (prefix + str(jobs) + "\n"):
                 if verify:
                     raise RuntimeError(
                         f"Failed to configure abuild: {path}"

pmb/config/__init__.py

@@ -59,6 +59,8 @@ ondev_min_version = "0.2.0"
 # idea is to run almost everything in Alpine chroots.
 required_programs = [
     "git",
+    "kpartx",
+    "losetup",
     "openssl",
     "ps",
     "tar",

pmb/config/init.py

@@ -6,10 +6,12 @@ from pmb.core.config import SystemdConfig
 from pmb.core.context import Context
 from pmb.core.pkgrepo import pkgrepo_default_path
 from pmb.helpers import logging
+from pmb.helpers.exceptions import NonBugError
 import glob
 import json
 import os
 import shutil
+from pathlib import Path
 from typing import Any
 
 import pmb.aportgen
@@ -37,15 +39,34 @@ def require_programs() -> None:
     for program in pmb.config.required_programs:
         if not shutil.which(program):
             missing.append(program)
+
+    losetup_missing_json = False
+
+    if "losetup" not in missing:
+        # Check if losetup supports the --json argument.
+        try:
+            pmb.helpers.run.user(["losetup", "--json"], check=True)
+        except RuntimeError:
+            losetup_missing_json = True
+
+    error_message = ""
+
     if missing:
-        raise RuntimeError(
-            "Can't find all programs required to run"
-            " pmbootstrap. Please install first:"
-            f" {', '.join(missing)}"
-        )
+        error_message += f"Can't find all programs required to run pmbootstrap. Please install the following: {', '.join(missing)}"
 
+    if missing and losetup_missing_json:
+        error_message += "\n\nAdditionally, your"
+    elif losetup_missing_json:
+        error_message += "Your"
 
-def ask_for_username(default_user: str):
+    if losetup_missing_json:
+        error_message += " system's losetup implementation is missing --json support. If you are using BusyBox, try installing losetup from util-linux."
+
+    if error_message:
+        raise NonBugError(error_message)
+
+
+def ask_for_username(default_user: str) -> str:
     """Ask for a reasonable username for the non-root user.
 
     :returns: the username
@@ -62,7 +83,7 @@ def ask_for_username(default_user: str):
         return ret
 
 
-def ask_for_work_path(args: PmbArgs):
+def ask_for_work_path(args: PmbArgs) -> tuple[Path, bool]:
     """Ask for the work path, until we can create it (when it does not exist) and write into it.
 
     :returns: (path, exists)
@@ -110,11 +131,11 @@ def ask_for_work_path(args: PmbArgs):
             return (work, exists)
         except OSError:
             logging.fatal(
-                "ERROR: Could not create this folder, or write" " inside it! Please try again."
+                "ERROR: Could not create this folder, or write inside it! Please try again."
             )
 
 
-def ask_for_channel(config: Config):
+def ask_for_channel(config: Config) -> str:
     """Ask for the postmarketOS release channel.
     The channel dictates, which pmaports branch pmbootstrap will check out,
     and which repository URLs will be used when initializing chroots.
@@ -147,12 +168,10 @@ def ask_for_channel(config: Config):
         ret = pmb.helpers.cli.ask("Channel", None, default, complete=choices)
         if ret in choices:
             return ret
-        logging.fatal(
-            "ERROR: Invalid channel specified, please type in one" " from the list above."
-        )
+        logging.fatal("ERROR: Invalid channel specified, please type in one from the list above.")
 
 
-def ask_for_ui(deviceinfo):
+def ask_for_ui(deviceinfo: Deviceinfo) -> str:
     ui_list = pmb.helpers.ui.list_ui(deviceinfo.arch)
     hidden_ui_count = 0
     device_is_accelerated = deviceinfo.gpu_accelerated == "true"
@@ -187,7 +206,7 @@ def ask_for_ui(deviceinfo):
         if ret in dict(ui_list).keys():
             return ret
         logging.fatal(
-            "ERROR: Invalid user interface specified, please type in" " one from the list above."
+            "ERROR: Invalid user interface specified, please type in one from the list above."
         )
 
 
@@ -221,7 +240,7 @@ def ask_for_systemd(config: Config, ui):
     default_is_systemd = pmb.helpers.ui.check_option(ui, "pmb:systemd")
     not_str = " " if default_is_systemd else " not "
     logging.info(
-        "Based on your UI selection, 'default' will result" f" in{not_str}installing systemd."
+        f"Based on your UI selection, 'default' will result in{not_str}installing systemd."
     )
 
     choices = SystemdConfig.choices()
@@ -235,7 +254,7 @@ def ask_for_systemd(config: Config, ui):
     return answer
 
 
-def ask_for_keymaps(config: Config, deviceinfo: Deviceinfo):
+def ask_for_keymaps(config: Config, deviceinfo: Deviceinfo) -> str:
     if not deviceinfo.keymaps or deviceinfo.keymaps.strip() == "":
         return ""
     options = deviceinfo.keymaps.split(" ")
@@ -247,10 +266,10 @@ def ask_for_keymaps(config: Config, deviceinfo: Deviceinfo):
         ret = pmb.helpers.cli.ask("Keymap", None, config.keymap, True, complete=options)
         if ret in options:
             return ret
-        logging.fatal("ERROR: Invalid keymap specified, please type in" " one from the list above.")
+        logging.fatal("ERROR: Invalid keymap specified, please type in one from the list above.")
 
 
-def ask_for_timezone():
+def ask_for_timezone() -> str:
     localtimes = ["/etc/zoneinfo/localtime", "/etc/localtime"]
     zoneinfo_path = "/usr/share/zoneinfo/"
     for localtime in localtimes:
@@ -269,7 +288,7 @@ def ask_for_timezone():
             logging.info(f"Your host timezone: {tz}")
             if pmb.helpers.cli.confirm("Use this timezone instead of GMT?", default="y"):
                 return tz
-    logging.info("WARNING: Unable to determine timezone configuration on host," " using GMT.")
+    logging.info("WARNING: Unable to determine timezone configuration on host, using GMT.")
     return "GMT"
 
 
@@ -327,11 +346,11 @@ def ask_for_provider_select(apkbuild, providers_cfg) -> None:
                 providers_cfg[select] = providers_short[ret]
                 break
             logging.fatal(
-                "ERROR: Invalid provider specified, please type in" " one from the list above."
+                "ERROR: Invalid provider specified, please type in one from the list above."
             )
 
 
-def ask_for_provider_select_pkg(pkgname, providers_cfg) -> None:
+def ask_for_provider_select_pkg(pkgname: str, providers_cfg: dict[str, str]) -> None:
     """Look up the APKBUILD for the specified pkgname and ask for selectable
     providers that are specified using "_pmb_select".
 
@@ -346,7 +365,7 @@ def ask_for_provider_select_pkg(pkgname, providers_cfg) -> None:
     ask_for_provider_select(apkbuild, providers_cfg)
 
 
-def ask_for_device_kernel(config: Config, device: str):
+def ask_for_device_kernel(config: Config, device: str) -> str:
     """Ask for the kernel that should be used with the device.
 
     :param device: code name, e.g. "lg-mako"
@@ -369,7 +388,7 @@ def ask_for_device_kernel(config: Config, device: str):
     # Ask for kernel (extra message when downstream and upstream are available)
     logging.info("Which kernel do you want to use with your device?")
     if "downstream" in kernels:
-        logging.info("Downstream kernels are typically the outdated Android" " kernel forks.")
+        logging.info("Downstream kernels are typically the outdated Android kernel forks.")
     if "downstream" in kernels and len(kernels) > 1:
         logging.info(
             "Upstream kernels (mainline, stable, ...) get security"
@@ -385,11 +404,11 @@ def ask_for_device_kernel(config: Config, device: str):
         ret = pmb.helpers.cli.ask("Kernel", None, default, True, complete=kernels)
         if ret in kernels.keys():
             return ret
-        logging.fatal("ERROR: Invalid kernel specified, please type in one" " from the list above.")
+        logging.fatal("ERROR: Invalid kernel specified, please type in one from the list above.")
     return ret
 
 
-def ask_for_device(context: Context):
+def ask_for_device(context: Context) -> tuple[str, bool, str]:
     """
     Prompt for the device vendor, model, and kernel.
 
@@ -400,7 +419,7 @@ def ask_for_device(context: Context):
     """
     vendors = sorted(pmb.helpers.devices.list_vendors())
     logging.info(
-        "Choose your target device vendor (either an " "existing one, or a new one for porting)."
+        "Choose your target device vendor (either an existing one, or a new one for porting)."
     )
     logging.info(f"Available vendors ({len(vendors)}): {', '.join(vendors)}")
 
@@ -445,7 +464,7 @@ def ask_for_device(context: Context):
                     " <https://postmarketos.org/renamed>"
                     " to see if it was renamed"
                 )
-            logging.info("You are about to do" f" a new device port for '{device}'.")
+            logging.info(f"You are about to do a new device port for '{device}'.")
             if not pmb.helpers.cli.confirm(default=True):
                 current_vendor = vendor
                 continue
@@ -466,7 +485,7 @@ def ask_for_device(context: Context):
     return (device, device_path is not None, kernel)
 
 
-def ask_for_additional_options(config) -> None:
+def ask_for_additional_options(config: Config) -> None:
     context = pmb.core.context.get_context()
     # Allow to skip additional options
     logging.info(
@@ -505,7 +524,7 @@ def ask_for_additional_options(config) -> None:
     config.boot_size = int(answer)
 
     # Parallel job count
-    logging.info("How many jobs should run parallel on this machine, when" " compiling?")
+    logging.info("How many jobs should run parallel on this machine, when compiling?")
     answer = pmb.helpers.cli.ask("Jobs", None, config.jobs, validation_regex="^[1-9][0-9]*$")
     config.jobs = int(answer)
 
@@ -532,10 +551,10 @@ def ask_for_additional_options(config) -> None:
         " that you'll have to authorize sudo more than once."
     )
     answer = pmb.helpers.cli.confirm(
-        "Enable background timer to prevent" " repeated sudo authorization?",
+        "Enable background timer to prevent repeated sudo authorization?",
         default=context.sudo_timer,
     )
-    config.sudo_timer = str(answer)
+    config.sudo_timer = answer
 
     # Mirrors
     # prompt for mirror change
@@ -550,7 +569,7 @@ def ask_for_additional_options(config) -> None:
         config.mirrors["systemd"] = os.path.join(mirror, "staging/systemd/")
 
 
-def ask_for_mirror():
+def ask_for_mirror() -> str:
     regex = "^[1-9][0-9]*$"  # single non-zero number only
 
     json_path = pmb.helpers.http.download(
@@ -605,7 +624,7 @@ def ask_for_mirror():
     return mirror
 
 
-def ask_for_hostname(default: str | None, device):
+def ask_for_hostname(default: str | None, device: str) -> str:
     if device:
         device = pmb.helpers.other.normalize_hostname(device)
     while True:
@@ -624,7 +643,7 @@ def ask_for_ssh_keys(default: bool) -> bool:
     if not len(glob.glob(os.path.expanduser("~/.ssh/id_*.pub"))):
         return False
     return pmb.helpers.cli.confirm(
-        "Would you like to copy your SSH public" " keys to the device?", default=default
+        "Would you like to copy your SSH public keys to the device?", default=default
     )
 
 
@@ -635,11 +654,11 @@ def ask_build_pkgs_on_install(default: bool) -> bool:
         " changes, reply 'n' for a faster installation."
     )
     return pmb.helpers.cli.confirm(
-        "Build outdated packages during" " 'pmbootstrap install'?", default=default
+        "Build outdated packages during 'pmbootstrap install'?", default=default
     )
 
 
-def get_locales():
+def get_locales() -> list[str]:
     ret = []
     list_path = f"{pmb.config.pmb_src}/pmb/data/locales"
     with open(list_path) as handle:
@@ -666,7 +685,7 @@ def ask_for_locale(current_locale: str) -> str:
         )
         ret = ret.replace(".UTF-8", "")
         if ret not in locales:
-            logging.info("WARNING: this locale is not in the list of known" " valid locales.")
+            logging.info("WARNING: this locale is not in the list of known valid locales.")
             if pmb.helpers.cli.ask() != "y":
                 # Ask again
                 continue

pmb/config/test_config_serde.py

@@ -12,7 +12,7 @@ def test_load(config_file):
     assert config.extra_packages == "neofetch,neovim,reboot-mode"
     assert config.hostname == "qemu-amd64"
     assert not config.is_default_channel
-    assert config.jobs == "8"
+    assert config.jobs == 8
     assert config.kernel == "edge"
     assert config.locale == "C.UTF-8"
     assert config.ssh_keys

pmb/core/config.py

@@ -56,7 +56,7 @@ class Config:
     extra_space: int = 0
     hostname: str = ""
     is_default_channel: bool = True
-    jobs: str = str(multiprocessing.cpu_count() + 1)
+    jobs: int = multiprocessing.cpu_count() + 1
     kernel: str = "stable"
     keymap: str = ""
     locale: str = "en_US.UTF-8"

pmb/helpers/mount.py

@@ -7,7 +7,7 @@ from pmb.core import Chroot
 import pmb.helpers.run
 
 
-def ismount(folder: Path):
+def ismount(folder: Path) -> bool:
     """Ismount() implementation that works for mount --bind.
 
     Workaround for: https://bugs.python.org/issue29707
@@ -23,7 +23,9 @@ def ismount(folder: Path):
     return False
 
 
-def bind(source: Path, destination: Path, create_folders=True, umount=False):
+def bind(
+    source: Path, destination: Path, create_folders: bool = True, umount: bool = False
+) -> None:
     """Mount --bind a folder and create necessary directory structure.
 
     :param umount: when destination is already a mount point, umount it first.
@@ -52,7 +54,7 @@ def bind(source: Path, destination: Path, create_folders=True, umount=False):
         raise RuntimeError(f"Mount failed: {source} -> {destination}")
 
 
-def bind_file(source: Path, destination: Path, create_folders=False):
+def bind_file(source: Path, destination: Path, create_folders: bool = False) -> None:
     """Mount a file with the --bind option, and create the destination file, if necessary."""
     # Skip existing mountpoint
     if ismount(destination):
@@ -93,7 +95,7 @@ def umount_all_list(prefix: Path, source: Path = Path("/proc/mounts")) -> list[P
     return ret
 
 
-def umount_all(folder: Path):
+def umount_all(folder: Path) -> None:
     """Umount all folders that are mounted inside a given folder."""
     for mountpoint in umount_all_list(folder):
         pmb.helpers.run.root(["umount", mountpoint])

pmb/helpers/other.py

@@ -82,10 +82,16 @@ def migrate_work_folder():
     # Read current version
     context = get_context()
     current = 0
+    suffix: str | None = None
     path = context.config.work / "version"
     if os.path.exists(path):
         with open(path) as f:
-            current = int(f.read().rstrip())
+            # pmb 2.3.x added a suffix due to conflicting work versions
+            # We need to be able to handle that going forward
+            version_parts = f.read().rstrip().split("-")
+            current = int(version_parts[0])
+            if len(version_parts) == 2:
+                suffix = version_parts[1]
 
     # Compare version, print warning or do nothing
     required = pmb.config.work_version
@@ -96,7 +102,8 @@ def migrate_work_folder():
         " (from version " + str(current) + " to " + str(required) + ")!"
     )
 
-    if current == 6:
+    # version 6 and version 7 from 2.3.x branch are equivalent for this and we need to migrate
+    if current == 6 or (current == 7 and suffix == "2.x"):
         # Ask for confirmation
         logging.info("Changelog:")
         logging.info("* Major refactor for pmb 3.0.0")

pmb/install/losetup.py

@@ -13,7 +13,7 @@ import pmb.chroot
 from pmb.core import Chroot
 
 
-def init():
+def init() -> None:
     if not Path("/sys/module/loop").is_dir():
         pmb.helpers.run.root(["modprobe", "loop"])
     for loopdevice in Path("/dev/").glob("loop*"):
@@ -22,7 +22,7 @@ def init():
         pmb.helpers.mount.bind_file(loopdevice, Chroot.native() / loopdevice)
 
 
-def mount(img_path: Path, _sector_size: int | None = None):
+def mount(img_path: Path, _sector_size: int | None = None) -> Path:
     """
     :param img_path: Path to the img file inside native chroot.
     """
@@ -56,6 +56,8 @@ def mount(img_path: Path, _sector_size: int | None = None):
                 raise e
             pass
 
+    raise AssertionError("This should never be reached")
+
 
 def device_by_back_file(back_file: Path) -> Path:
     """
@@ -70,12 +72,12 @@ def device_by_back_file(back_file: Path) -> Path:
     # Find the back_file
     losetup = json.loads(losetup_output)
     for loopdevice in losetup["loopdevices"]:
-        if Path(loopdevice["back-file"]) == back_file:
+        if loopdevice["back-file"] is not None and Path(loopdevice["back-file"]) == back_file:
             return Path(loopdevice["name"])
     raise RuntimeError(f"Failed to find loop device for {back_file}")
 
 
-def umount(img_path: Path):
+def umount(img_path: Path) -> None:
     """
     :param img_path: Path to the img file inside native chroot.
     """
@@ -88,7 +90,7 @@ def umount(img_path: Path):
     pmb.chroot.root(["losetup", "-d", device])
 
 
-def detach_all():
+def detach_all() -> None:
     """
     Detach all loop devices used by pmbootstrap
     """
@@ -98,7 +100,6 @@ def detach_all():
     losetup = json.loads(losetup_output)
     work = get_context().config.work
     for loopdevice in losetup["loopdevices"]:
-        print(loopdevice["back-file"])
         if Path(loopdevice["back-file"]).is_relative_to(work):
             pmb.helpers.run.root(["kpartx", "-d", loopdevice["name"]], check=False)
             pmb.helpers.run.root(["losetup", "-d", loopdevice["name"]])

pmb/parse/apkindex.py

@@ -5,7 +5,6 @@ from typing import cast, overload, Any, Literal
 from collections.abc import Sequence
 from pmb.core.apkindex_block import ApkindexBlock
 from pmb.core.arch import Arch
-from pmb.core.context import get_context
 from pmb.helpers import logging
 from pathlib import Path
 import tarfile
@@ -315,8 +314,9 @@ def parse_blocks(path: Path) -> list[ApkindexBlock]:
         ret.append(block)
 
 
-def cache_key(path: Path) -> str:
-    return str(path.relative_to(get_context().config.work, walk_up=True))
+# FIXME: come up with something better here...
+def cache_key(path: Path) -> int:
+    return hash(path)
 
 
 def clear_cache(path: Path) -> bool:

pmb/parse/test_apkindex.py

+from pathlib import Path
 from pmb.core.arch import Arch
 import pytest
 
-from .apkindex import parse as parse_apkindex
+import pmb.parse.apkindex
+
+from .apkindex import parse as parse_apkindex, clear_cache as clear_apkindex_cache
 
 example_apkindex = """
 C:Q1p+nGf5oBAmbU9FQvV4MhfEmWqVE=
@@ -231,13 +234,20 @@ D:blkid btrfs-progs buffyboard busybox-extras bzip2 cryptsetup device-mapper dev
 p:postmarketos-ramdisk=3.3.5-r2"""
 
 
-def test_apkindex_parse(tmp_path) -> None:
+@pytest.fixture
+def valid_apkindex_file(tmp_path) -> Path:
+    # FIXME: use tmpfile fixture from !2453
     tmpfile = tmp_path / "APKINDEX.1"
     print(tmp_path)
     f = open(tmpfile, "w")
     f.write(example_apkindex)
     f.close()
 
+    return tmpfile
+
+
+def test_apkindex_parse(valid_apkindex_file) -> None:
+    tmpfile = valid_apkindex_file
     blocks = parse_apkindex(tmpfile, True)
     for k, v in blocks.items():
         print(f"{k}: {v}")
@@ -409,3 +419,24 @@ i:postmarketos-base-ui=29-r1 xorg-server
 
     # We expect parsing to succeed when the timestamp is missing
     parse_apkindex(tmpfile, True)
+
+
+def test_apkindex_parse_cache_hit(valid_apkindex_file, monkeypatch) -> None:
+    # First parse normally, filling the cache
+    parse_apkindex(valid_apkindex_file)
+
+    # Mock that always asserts when called
+    def mock_parse_next_block(path, lines):
+        assert False
+
+    # parse_next_block() is only called on cache miss
+    monkeypatch.setattr(pmb.parse.apkindex, "parse_next_block", mock_parse_next_block)
+
+    # Now we expect the cache to be hit and thus the mock won't be called, so no assertion error
+    parse_apkindex(valid_apkindex_file)
+
+    # Now we clear the cache, the mock should be called and we'll assert
+    clear_apkindex_cache(valid_apkindex_file)
+
+    with pytest.raises(AssertionError):
+        parse_apkindex(valid_apkindex_file)