.ci/pytest.sh

@@ -23,15 +23,17 @@ fi
 # Use pytest-cov if it is installed to display code coverage
 cov_arg=""
 if python -c "import pytest_cov" >/dev/null 2>&1; then
-	cov_arg="--cov=pmb"
+	cov_arg="--cov=pmb --cov-report term --cov-report xml:coverage.xml"
 fi
 
 echo "Running pytest..."
 echo "NOTE: use 'pmbootstrap log' to see the detailed log if running locally."
+# shellcheck disable=SC2086
 python -m pytest \
 	--color=yes \
 	-vv \
 	-x \
+	--junitxml=junit.xml \
 	$cov_arg \
 		-m "not skip_ci" \
 		"$@"

.gitlab-ci.yml

@@ -17,15 +17,16 @@ before_script: &global_before_scripts
   # Force IPv4 for gitlab.postmarketos.org until it supports IPv6 too, OSUOSL is
   # working on it (infra#195)
   - "echo '140.211.167.182 gitlab.postmarketos.org' >> /etc/hosts"
+  # python 3.10 tests: need to have tomli installed, see pmb/helpers/toml.py
+  - "if grep -q VERSION_ID=3\\.17 /etc/os-release; then apk add py3-tomli; fi"
 
 stages:
-  - lint
-  - deploy
   - test
+  - deploy
   - integration-test
 
 codespell:
-  stage: lint
+  stage: test
   script:
     - ".ci/codespell.sh"
 
@@ -41,28 +42,38 @@ pytest:
     - "cp /home/build/.local/var/pmbootstrap/log_testsuite.txt ."
     - "cp /home/build/.local/var/pmbootstrap/log.txt ."
   artifacts:
-    when: on_failure
+    when: always
     paths:
       - "log_testsuite.txt"
       - "log.txt"
+    reports:
+      coverage_report:
+        coverage_format: cobertura
+        path: coverage.xml
+      junit: junit.xml
+
+# Ensure that tests pass on Python 3.10
+pytest-python3.10:
+  extends: pytest
+  image: alpine:3.17
 
 ruff:
-  stage: lint
+  stage: test
   script:
     - ".ci/ruff.sh"
 
 shellcheck:
-  stage: lint
+  stage: test
   script:
     - ".ci/shellcheck.sh"
 
 vermin:
-  stage: lint
+  stage: test
   script:
     - ".ci/vermin.sh"
 
 mr-settings:
-  stage: lint
+  stage: test
   rules:
     - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
   before_script:
@@ -73,12 +84,12 @@ mr-settings:
     - "python3 ./check_mr_settings.py"
 
 mypy:
-  stage: lint
+  stage: test
   script:
     - ".ci/mypy.sh"
 
 docs:
-  stage: lint
+  stage: test
   script:
     - ".ci/docs.sh"
   artifacts:
@@ -125,6 +136,11 @@ install amd64:
   # ARM64 on AMD64
     - .ci/integration_tests/build_images postmarketos-trailblazer console
 
+# Test that building images works on Python 3.10
+install amd64 python3.10:
+  extends: install amd64
+  image: alpine:3.17
+
 install aarch64:
   extends: .integration
   tags: [arm64]

README.md

@@ -254,11 +254,6 @@ Use `-v` on any action to get verbose logging:
 $ pmbootstrap -v build hello-world
 ```
 
-Parse a single deviceinfo and return it as JSON:
-```
-$ pmbootstrap deviceinfo_parse pine64-pinephone
-```
-
 Parse a single APKBUILD and return it as JSON:
 ```
 $ pmbootstrap apkbuild_parse hello-world

docs/api/pmb.chroot.rst

@@ -12,14 +12,6 @@ pmb.chroot.apk module
    :undoc-members:
    :show-inheritance:
 
-pmb.chroot.apk\_static module
------------------------------
-
-.. automodule:: pmb.chroot.apk_static
-   :members:
-   :undoc-members:
-   :show-inheritance:
-
 pmb.chroot.binfmt module
 ------------------------
 

docs/api/pmb.helpers.rst

@@ -4,6 +4,14 @@ pmb.helpers package
 Submodules
 ----------
 
+pmb.helpers.apk\_static module
+-----------------------------
+
+.. automodule:: pmb.helpers.apk_static
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 pmb.helpers.apk module
 ----------------------
 

docs/chroot.rst

+######
+Chroot
+######
+
+chroot (short for "change root") is a system call and command that changes the apparent root directory.
+This can come in very handy to make some modifications in the generated image file.
+
+
+
+Enter the armhf building chroot:
+
+.. code-block:: shell
+
+  $ pmbootstrap chroot -b armhf
+
+
+Run a command inside a chroot:
+
+.. code-block:: shell
+
+  $ pmbootstrap chroot -- echo test
+
+
+Safely delete all chroots:
+
+.. code-block:: shell
+
+  $ pmbootstrap zap
+
+
+
+Use ``chroot`` to install a specific kernel version from an apk file. 
+
+
+copy your working kernel apk to the chroot dir of pmbootstrap:
+
+.. code-block:: shell
+
+  $ sudo cp /path/of/linux-kernel.apk ~/.local/var/pmbootstrap/chroot_rootfs_oneplus-enchilada/ 
+
+enter chroot
+
+.. code-block:: shell
+
+  $ pmbootstrap chroot -r 
+
+
+and install the package:
+
+.. code-block:: shell
+
+ $ apk add linux-kernel.apk
+
+

docs/debugging.rst

+#########
+Debugging
+#########
+
+pmbootstrap writes all log output and each shell command it runs to log.txt inside the work dir.
+Use the following command to follow the log in a second terminal: 
+
+.. code-block:: shell
+
+  $ pmbootstrap log
+
+
+Use ``-v`` on any action to get verbose logging:
+
+.. code-block:: shell
+
+  $ pmbootstrap -v build hello-world
+
+
+Parse a package from an APKINDEX and return it as JSON:
+
+.. code-block:: shell
+
+  $ pmbootstrap apkindex_parse $WORK/cache_apk_x86_64/APKINDEX.8b865e19.tar.gz hello-world
+
+
+``ccache`` statistics:
+
+.. code-block:: shell
+
+  $ pmbootstrap stats --arch=armhf
+
+

docs/index.rst

@@ -18,6 +18,8 @@ For further information, please check out the `postmarketOS-wiki`_.
 
    installation
    usage
+   chroot
+   debugging
    api/modules
    mirrors
 

docs/usage.rst

@@ -18,6 +18,111 @@ If you already ran this before, run the following to update your local clone of
 
    $ pmbootstrap pull
 
+
+Quick health check and config overview:
+
+.. code-block:: shell
+
+   $ pmbootstrap status
+
+
+After successfully finishing the ``init`` sequence with answering all questions, its time to build the 
+installation:
+
+Devices like the PinePhone, Samsung Galaxy S II, Nokia N900, various laptops etc. can boot from an SD card, USB stick or other external storage. Find the name with lsblk first and make sure it is the right one as you will overwrite everything on it. Use a path without partition number at the end, such as /dev/mmcblk0. If your device is able to boot from SD card without flashing anything (such as the PinePhone), you should then be able to insert SD card into your device and boot it up.
+
+.. code-block:: shell
+
+   $ pmbootstrap install --sdcard=/dev/mmcblk... --fde
+
+
+For devices where you will flash directly to the internal storage, as mostly all sdm845 devices, you can build the installation as:
+
+.. code-block:: shell
+
+   $ pmbootstrap install
+
+
+or with full disk encryption:
+
+.. code-block:: shell
+
+   $ pmbootstrap install --fde
+
+and then flash it with the ``pmbootstrap flasher`` while the device is in ``fastboot`` mode:
+
+.. code-block:: shell
+
+   $ pmbootstrap flasher flash_rootfs
+
+
+and also the kernel:
+
+.. code-block:: shell
+
+   $ pmbootstrap flasher flash_kernel
+
+
+For getting images on the local machine instead of directly flashing them, execute:
+
+.. code-block:: shell
+
+   $ pmbootstrap export
+
+
+To extract the generated initramfs: 
+
+.. code-block:: shell
+
+   $ pmbootstrap initfs extract
+
+
+Build and flash Android recovery zip:
+
+.. code-block:: shell
+
+ $ pmbootstrap install --android-recovery-zip
+ $ pmbootstrap flasher --method=adb sideload
+
+
+Update existing installation on SD card:
+
+.. code-block:: shell
+
+ $ pmbootstrap install --disk=/dev/mmcblk0 --rsync
+
+
+Run the image in QEMU:
+
+.. code-block:: shell
+
+ $ pmbootstrap qemu --image-size=1G
+
+
+**Device Porting Assistance**
+
+Analyze Android boot.img files (also works with recovery OS images like TWRP):
+
+.. code-block:: shell
+
+ $ pmbootstrap bootimg_analyze ~/Downloads/twrp-3.2.1-0-fp2.img
+
+
+Check kernel configs:
+
+.. code-block:: shell
+
+ $ pmbootstrap kconfig check
+
+
+Edit a kernel config:
+
+.. code-block:: shell
+ 
+ $ pmbootstrap kconfig edit
+
+
+
 For further details on the different actions please see below and refer to the wiki-arcticle on `pmbootstrap`_.
 
 .. autoprogram:: pmb.parse:get_parser()

pmb/__init__.py

@@ -14,12 +14,12 @@ if TYPE_CHECKING:
 
 from . import config
 from . import parse
-from . import types
-from .config import init as config_init
+from .config import init as config_init, require_programs
 from .helpers import frontend
 from .helpers import logging
 from .helpers import mount
 from .helpers import other
+from .helpers import status
 from .core import Chroot, Config
 from .core.context import get_context
 from .commands import run_command
@@ -44,6 +44,8 @@ if version < (3, 10):
 
 def print_log_hint() -> None:
     context = get_context(allow_failure=True)
+    if context and context.details_to_stdout:
+        return
     log = context.log if context else Config().work / "log.txt"
     # Hints about the log file (print to stdout only)
     log_hint = "Run 'pmbootstrap log' for details."
@@ -74,6 +76,9 @@ def main() -> int:
         if not args.as_root and os.geteuid() == 0:
             raise RuntimeError("Do not run pmbootstrap as root!")
 
+        # Check for required programs (and find their absolute paths)
+        require_programs()
+
         # Initialize or require config
         if args.action == "init":
             config_init.frontend(args)
@@ -129,10 +134,12 @@ def main() -> int:
 
     except Exception as e:
         # Dump log to stdout when args (and therefore logging) init failed
+        can_print_status = get_context(allow_failure=True) is not None
         if "args" not in locals():
             import logging as pylogging
 
             pylogging.getLogger().setLevel(logging.DEBUG)
+            can_print_status = False
 
         logging.info("ERROR: " + str(e))
         logging.info("See also: <https://postmarketos.org/troubleshooting>")
@@ -145,6 +152,8 @@ def main() -> int:
             "Find the latest version here: https://gitlab.postmarketos.org/postmarketOS/pmbootstrap/-/tags"
         )
         print(f"Your version: {__version__}")
+        if can_print_status:
+            status.print_status()
         return 1
 
     return 0

pmb/aportgen/busybox_static.py

@@ -4,7 +4,6 @@ from pathlib import Path
 import pmb.aportgen.core
 import pmb.build
 import pmb.chroot.apk
-import pmb.chroot.apk_static
 import pmb.helpers.run
 import pmb.parse.apkindex
 from pmb.core import Chroot

pmb/aportgen/grub_efi.py

@@ -4,7 +4,6 @@ from pathlib import Path
 import pmb.aportgen.core
 import pmb.build
 import pmb.chroot.apk
-import pmb.chroot.apk_static
 from pmb.core.arch import Arch
 import pmb.helpers.run
 import pmb.parse.apkindex

pmb/aportgen/musl.py

@@ -4,7 +4,6 @@ from pathlib import Path
 import pmb.aportgen.core
 import pmb.build
 import pmb.chroot.apk
-import pmb.chroot.apk_static
 import pmb.helpers.run
 import pmb.parse.apkindex
 from pmb.core import Chroot

pmb/build/backend.py

 import enum
 from pathlib import Path
-from pmb.core.pkgrepo import pkgrepo_paths
+from pmb.core.pkgrepo import pkgrepo_name, pkgrepo_paths
 import pmb.helpers.run
 import pmb.chroot
 
@@ -126,10 +126,10 @@ def mount_pmaports(chroot: Chroot = Chroot.native()) -> dict[str, Path]:
     """
     dest_paths = {}
     for repo in pkgrepo_paths(skip_extras=True):
-        destination = Path("/mnt") / repo.name
+        destination = Path("/mnt") / pkgrepo_name(repo)
         outside_destination = chroot / destination
         pmb.helpers.mount.bind(repo, outside_destination, umount=True)
-        dest_paths[repo.name] = destination
+        dest_paths[pkgrepo_name(repo)] = destination
 
     return dest_paths
 

pmb/chroot/apk.py

@@ -6,10 +6,8 @@ from __future__ import annotations
 import os
 from pathlib import Path
 import traceback
-import pmb.chroot.apk_static
 from pmb.core.arch import Arch
 from pmb.helpers import logging
-import shlex
 from collections.abc import Sequence
 
 import pmb.build
@@ -30,57 +28,6 @@ from pmb.types import PathString
 from pmb.helpers.exceptions import NonBugError
 
 
-@Cache("chroot", "user_repository", mirrors_exclude=[])
-def update_repository_list(
-    chroot: Chroot,
-    user_repository: bool = False,
-    mirrors_exclude: list[str] = [],
-    check: bool = False,
-) -> None:
-    """
-    Update /etc/apk/repositories, if it is outdated (when the user changed the
-    --mirror-alpine or --mirror-pmOS parameters).
-
-    :param mirrors_exclude: mirrors to exclude from the repository list
-    :param check: This function calls it self after updating the
-                  /etc/apk/repositories file, to check if it was successful.
-                  Only for this purpose, the "check" parameter should be set to
-                  True.
-    """
-    # Read old entries or create folder structure
-    path = chroot / "etc/apk/repositories"
-    lines_old: list[str] = []
-    if path.exists():
-        # Read all old lines
-        lines_old = []
-        with path.open() as handle:
-            for line in handle:
-                lines_old.append(line[:-1])
-    else:
-        pmb.helpers.run.root(["mkdir", "-p", path.parent])
-
-    # Up to date: Save cache, return
-    lines_new = pmb.helpers.repo.urls(
-        user_repository=user_repository, mirrors_exclude=mirrors_exclude
-    )
-    if lines_old == lines_new:
-        return
-
-    # Check phase: raise error when still outdated
-    if check:
-        raise RuntimeError(f"Failed to update: {path}")
-
-    # Update the file
-    logging.debug(f"({chroot}) update /etc/apk/repositories")
-    if path.exists():
-        pmb.helpers.run.root(["rm", path])
-    for line in lines_new:
-        pmb.helpers.run.root(["sh", "-c", "echo " f"{shlex.quote(line)} >> {path}"])
-    update_repository_list(
-        chroot, user_repository=user_repository, mirrors_exclude=mirrors_exclude, check=True
-    )
-
-
 @Cache("chroot")
 def check_min_version(chroot: Chroot = Chroot.native()) -> None:
     """
@@ -163,9 +110,7 @@ def packages_get_locally_built_apks(package_list: list[str], arch: Arch) -> list
         for channel in channels:
             apk_path = get_context().config.work / "packages" / channel / arch / apk_file
             if apk_path.exists():
-                # FIXME: use /mnt/pmb… until MR 2351 is reverted (pmb#2388)
-                # local.append(apk_path)
-                local.append(Path("/mnt/pmbootstrap/packages/") / channel / arch / apk_file)
+                local.append(apk_path)
                 break
 
         # Record all the packages we have visited so far
@@ -233,24 +178,17 @@ def install_run_apk(
         user_repo += ["--repository", context.config.work / "packages" / channel]
 
     for i, command in enumerate(commands):
-        # --no-interactive is a parameter to `add`, so it must be appended or apk
-        # gets confused
-        command += ["--no-interactive"]
         command = user_repo + command
 
         # Ignore missing repos before initial build (bpo#137)
         if os.getenv("PMB_APK_FORCE_MISSING_REPOSITORIES") == "1":
             command = ["--force-missing-repositories"] + command
 
-        if context.offline:
-            command = ["--no-network"] + command
-        if i == 0:
-            pmb.helpers.apk.apk_with_progress(command, chroot)
-        else:
-            # Virtual package related commands don't actually install or remove
-            # packages, but only mark the right ones as explicitly installed.
-            # They finish up almost instantly, so don't display a progress bar.
-            pmb.chroot.root(["apk", "--no-progress"] + command, chroot)
+        # Virtual package related commands don't actually install or remove
+        # packages, but only mark the right ones as explicitly installed.
+        # So only display a progress bar for the "apk add" command which is
+        # always the first one we process (i == 0).
+        pmb.helpers.apk.run(command, chroot, with_progress=(i == 0))
 
 
 def install(packages: list[str], chroot: Chroot, build: bool = True, quiet: bool = False) -> None:

pmb/chroot/init.py

@@ -9,14 +9,16 @@ import os
 import pmb.chroot
 import pmb.chroot.binfmt
 import pmb.chroot.apk
-import pmb.chroot.apk_static
 import pmb.config
 import pmb.config.workdir
+import pmb.helpers.apk_static
+import pmb.helpers.apk
 import pmb.helpers.repo
 import pmb.helpers.run
 import pmb.helpers.other
 from pmb.core import Chroot, ChrootType
 from pmb.core.context import get_context
+from pmb.types import PathString
 
 
 class UsrMerge(enum.Enum):
@@ -136,33 +138,27 @@ def init(chroot: Chroot, usr_merge: UsrMerge = UsrMerge.AUTO) -> None:
     mark_in_chroot(chroot)
     if chroot.exists():
         copy_resolv_conf(chroot)
-        pmb.chroot.apk.update_repository_list(chroot)
+        pmb.helpers.apk.update_repository_list(chroot.path)
         warn_if_chroots_outdated()
         return
 
-    # Require apk-tools-static
-    pmb.chroot.apk_static.init()
+    # Fetch apk.static
+    pmb.helpers.apk_static.init()
 
     logging.info(f"({chroot}) Creating chroot")
 
-    # Initialize cache
-    apk_cache = config.work / f"cache_apk_{arch}"
-    pmb.helpers.run.root(["ln", "-s", "-f", "/var/cache/apk", chroot / "etc/apk/cache"])
-
     # Initialize /etc/apk/keys/, resolv.conf, repositories
     init_keys()
     copy_resolv_conf(chroot)
-    pmb.chroot.apk.update_repository_list(chroot)
+    pmb.helpers.apk.update_repository_list(chroot.path)
 
     pmb.config.workdir.chroot_save_init(chroot)
 
     # Install alpine-base
     pmb.helpers.repo.update(arch)
     pkgs = ["alpine-base"]
-    cmd = ["--root", chroot.path, "--cache-dir", apk_cache, "--initdb", "--arch", arch]
-    for channel in pmb.config.pmaports.all_channels():
-        cmd += ["--repository", config.work / "packages" / channel]
-    pmb.chroot.apk_static.run(cmd + ["add", *pkgs])
+    cmd: list[PathString] = ["--initdb"]
+    pmb.helpers.apk.run(cmd + ["add", *pkgs], chroot)
 
     # Merge /usr
     if usr_merge is UsrMerge.AUTO and pmb.config.is_systemd_selected(config):

pmb/chroot/initfs_hooks.py

@@ -51,7 +51,7 @@ def delete(hook: str, suffix: Chroot) -> None:
     if hook not in list_chroot(suffix):
         raise RuntimeError("There is no such hook installed!")
     prefix = pmb.config.initfs_hook_prefix
-    pmb.chroot.root(["apk", "del", f"{prefix}{hook}"], suffix)
+    pmb.helpers.apk.run(["del", f"{prefix}{hook}"], suffix)
 
 
 def update(suffix: Chroot) -> None:

pmb/chroot/run.py

@@ -2,7 +2,6 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 import os
 from pathlib import Path, PurePath
-import shutil
 import subprocess
 from collections.abc import Sequence
 from typing import overload, Literal
@@ -23,23 +22,6 @@ from pmb.types import (
 )
 
 
-def executables_absolute_path():
-    """
-    Get the absolute paths to the sh and chroot executables.
-    """
-    ret = {}
-    for binary in ["sh", "chroot"]:
-        path = shutil.which(binary, path=pmb.config.chroot_host_path)
-        if not path:
-            raise RuntimeError(
-                f"Could not find the '{binary}'"
-                " executable. Make sure that it is in"
-                " your current user's PATH."
-            )
-        ret[binary] = path
-    return ret
-
-
 def rootm(
     cmds: Sequence[Sequence[PathString]],
     chroot: Chroot = Chroot.native(),
@@ -97,8 +79,8 @@ def rootm(
     # cmd: ["echo", "test"]
     # cmd_chroot: ["/sbin/chroot", "/..._native", "/bin/sh", "-c", "echo test"]
     # cmd_sudo: ["sudo", "env", "-i", "sh", "-c", "PATH=... /sbin/chroot ..."]
-    executables = executables_absolute_path()
-    cmd_chroot = [
+    executables = pmb.config.required_programs
+    cmd_chroot: list[PathString] = [
         executables["chroot"],
         chroot.path,
         "/bin/sh",

pmb/chroot/zap.py

@@ -11,6 +11,7 @@ import pmb.config.workdir
 import pmb.chroot
 import pmb.config.pmaports
 import pmb.config.workdir
+import pmb.helpers.apk
 import pmb.helpers.cli
 import pmb.helpers.pmaports
 import pmb.helpers.run
@@ -107,7 +108,7 @@ def zap(
     pmb.config.workdir.clean()
 
     # Chroots were zapped, so no repo lists exist anymore
-    pmb.chroot.apk.update_repository_list.cache_clear()
+    pmb.helpers.apk.update_repository_list.cache_clear()
     # Let chroot.init be called again
     pmb.chroot.init.cache_clear()
 
@@ -183,18 +184,6 @@ def zap_pkgs_online_mismatch(confirm=True, dry=False):
     # Iterate over existing apk caches
     for path in paths:
         arch = Arch.from_str(path.name.split("_", 2)[2])
-        if arch.is_native():
-            chroot = Chroot.native()
-        else:
-            chroot = Chroot.buildroot(arch)
-
-        # Skip if chroot does not exist
-        # FIXME: should we init the buildroot to do it anyway?
-        # what if we run apk.static with --arch instead?
-        if not chroot.exists():
-            continue
-
-        # Clean the cache with apk
-        logging.info(f"({chroot}) apk -v cache clean")
+
         if not dry:
-            pmb.chroot.root(["apk", "-v", "cache", "clean"], chroot)
+            pmb.helpers.apk.cache_clean(arch)

pmb/commands/__init__.py

@@ -47,7 +47,6 @@ unmigrated_commands = [
     "install",
     "checksum",
     "build",
-    "deviceinfo_parse",
     "apkbuild_parse",
     "apkindex_parse",
     "config",