.ci/docs.sh

@@ -4,10 +4,11 @@
 # https://postmarketos.org/pmb-ci
 
 
-# Ensure sphinx_rtd_theme is installed
+# Install sphinx + extensions when running in CI
 if [ "$(id -u)" = 0 ]; then
 	set -x
 	apk -q add \
+		py3-myst-parser \
 		py3-sphinx_rtd_theme \
 		py3-sphinxcontrib-autoprogram
 	exec su "${TESTUSER:-build}" -c "sh -e $0"

.ci/vermin.sh

@@ -10,7 +10,7 @@ fi
 
 # shellcheck disable=SC2046
 vermin \
-	-t=3.9- \
+	-t=3.10- \
 	--backport argparse \
 	--backport configparser \
 	--backport enum \

README.md

@@ -43,8 +43,8 @@ Issues are being tracked
   * [Linux kernel 3.17 or higher](https://postmarketos.org/oldkernel)
   * Note: kernel versions between 5.8.8 and 6.0 might 
     [have issues with parted](https://gitlab.com/postmarketOS/pmbootstrap/-/issues/2309).
-* Python 3.9+
-* For python3 <= 3.10: tomli
+* Python 3.10+
+* For python3 < 3.11: tomli
 * OpenSSL
 * git
 * ps

docs/conf.py

@@ -34,6 +34,7 @@ version = ".".join(release.split(".")[:3])
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
 extensions = [
+    "myst_parser",
     "sphinx.ext.autodoc",
     "sphinx.ext.autosummary",
     "sphinx.ext.doctest",
@@ -42,6 +43,7 @@ extensions = [
 
 templates_path = ["_templates"]
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+source_suffix = [".rst", ".md"]
 
 # -- Options for HTML output -------------------------------------------------
 html_theme = "sphinx_rtd_theme"

docs/index.rst

@@ -19,6 +19,7 @@ For further information, please check out the `postmarketOS-wiki`_.
    installation
    usage
    api/modules
+   mirrors
 
 
 

docs/mirrors.md

+# Mirror Configuration
+
+A typical postmarketOS installation has one Alpine Linux mirror configured as well as one
+postmarketOS mirror. As Alpine's CDN mirror is used by default, it should be suitable for most
+users. The postmarketOS mirror can be configured interactively with `pmbootstrap init`, under
+"additional options".
+
+Find the currently selected mirrors in the output of `pmbootstrap status`, as well as in
+`/etc/apk/repositories` for initialized chroots and finished installations.
+
+## Advanced
+
+Some advanced use cases are supported by configuring the mirrors directly, either by editing
+`pmbootstrap_v3.cfg` or running `pmbootstrap config`. Find the lists of mirrors at
+[mirrors.alpinelinux.org](https://mirrors.alpinelinux.org) and
+[mirrors.postmarketos.org](https://mirrors.postmarketos.org).
+
+### Change the Alpine Linux mirror
+
+```
+$ pmbootstrap config mirrors.alpine http://uk.alpinelinux.org/alpine/
+```
+
+### Disable the postmarketOS mirror
+
+This is useful to test bootstrapping from pure Alpine:
+
+```
+$ pmbootstrap config mirrors.pmaports none
+$ pmbootstrap config mirrors.systemd none
+```
+
+### Use `_custom` mirrors
+
+For all repositories, it is possible to add `_custom` entries, for example
+`pmaports_custom` in addition to `pmaports`. If these are set, then pmbootstrap
+creates addition entries infront of the real mirrors in
+`/etc/apk/repositories`. This is used by [BPO](https://build.postmarketos.org)
+to build packages with a WIP repository enabled in addition to the final
+repository, but could also be used if you have another custom repository that
+you want to use in addition to the postmarketOS binary package repository.
+
+```
+$ pmbootstrap config mirrors.pmaports_custom http://custom-repository-here
+```

docs/usage.rst

@@ -41,8 +41,8 @@ pmbootstrap requires the following:
        Kernel version 5.8 - 6.0 might have issues with loop-devices
 
 
-  * Python 3.9+
-  * For python3 <= 3.10: tomli
+  * Python 3.10+
+  * For python3 < 3.11: tomli
   * OpenSSL
   * git
   * ps

pmb/__init__.py

@@ -28,9 +28,16 @@ from .commands import run_command
 __version__ = "3.0.0_alpha"
 
 # Python version check
+# === CHECKLIST FOR UPGRADING THE REQUIRED PYTHON VERSION ===
+# * .ci/vermin.sh
+# * README.md
+# * docs/usage.rst
+# * pmb/__init__.py (you are here)
+# * pyproject.toml
+# * when upgrading to python 3.11: pmb/helpers/toml.py and remove this line
 version = sys.version_info
-if version < (3, 9):
-    print("You need at least Python 3.9 to run pmbootstrap")
+if version < (3, 10):
+    print("You need at least Python 3.10 to run pmbootstrap")
     print("(You are running it with Python " + str(version.major) + "." + str(version.minor) + ")")
     sys.exit()
 

pmb/build/_package.py

 # Copyright 2023 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
 import datetime
-from typing import Any, Callable, Optional, TypedDict
+from typing import Any, TypedDict
+from collections.abc import Callable
 from pmb.build.other import BuildStatus
 from pmb.core.arch import Arch
 from pmb.core.context import Context
@@ -149,9 +150,14 @@ def finish(apkbuild, channel, arch, output: Path, chroot: Chroot, strict=False):
 
     logging.info(f"@YELLOW@=>@END@ @BLUE@{channel}/{apkbuild['pkgname']}@END@: Done!")
 
-    if apkbuild["pkgname"] == "abuild":
-        logging.info("NOTE: re-installing abuild since it was just built")
-        pmb.chroot.apk.install(["abuild"], chroot, build=False)
+    # If we just built a package which is used to build other packages, then
+    # update the buildroot to use the newly built version.
+    if apkbuild["pkgname"] in pmb.config.build_packages:
+        logging.info(
+            f"NOTE: Updating package {apkbuild['pkgname']} in buildroot since it's"
+            " used for building..."
+        )
+        pmb.chroot.apk.install([apkbuild["pkgname"]], chroot, build=False, quiet=True)
 
 
 _package_cache: dict[str, list[str]] = {}
@@ -210,7 +216,7 @@ def process_package(
     context: Context,
     queue_build: Callable,
     pkgname: str,
-    arch: Optional[Arch],
+    arch: Arch | None,
     fallback_arch: Arch,
     force: bool,
 ) -> list[str]:
@@ -237,11 +243,6 @@ def process_package(
     # Add the package to the build queue
     base_depends = get_depends(context, base_apkbuild)
 
-    # We use a fork of abuild for systemd, make sure it gets built if necessary by adding
-    # it as a dependency of the first package
-    if "abuild" not in base_depends:
-        base_depends.insert(0, "abuild")
-
     depends = base_depends.copy()
 
     base_build_status = BuildStatus.NEW if force else BuildStatus.UNNECESSARY
@@ -321,12 +322,12 @@ def process_package(
 def packages(
     context: Context,
     pkgnames: list[str],
-    arch: Optional[Arch] = None,
+    arch: Arch | None = None,
     force=False,
     strict=False,
     src=None,
     bootstrap_stage=BootstrapStage.NONE,
-    log_callback: Optional[Callable] = None,
+    log_callback: Callable | None = None,
 ) -> list[str]:
     """
     Build a package and its dependencies with Alpine Linux' abuild.
@@ -348,6 +349,11 @@ def packages(
     build_queue: list[BuildQueueItem] = []
     built_packages: set[str] = set()
 
+    # We want to build packages in the order they're given here. Due to how we
+    # walk the package dependencies, reverse the list so that when we later
+    # reverse the build queue we'll be back in the right order.
+    pkgnames.reverse()
+
     # Add a package to the build queue, fetch it's dependency, and
     # add record build helpers to installed (e.g. sccache)
     def queue_build(
@@ -414,6 +420,26 @@ def packages(
             context, queue_build, pkgname, arch, fallback_arch, force
         )
 
+    # If any of our common build packages need to be built and are missing, then add them
+    # to the queue so they're built first. This is necessary so that our abuild fork is
+    # always built first (for example). For now assume that if building in strict mode we
+    # should skip this step, but we might want to revisit this later.
+    if not src:
+        for pkgname in pmb.config.build_packages:
+            if pkgname not in pkgnames:
+                aport, apkbuild = get_apkbuild(pkgname)
+                if not aport:
+                    continue
+                bstatus = pmb.build.get_status(arch, apkbuild)
+                if bstatus.necessary():
+                    if strict:
+                        raise RuntimeError(
+                            f"Strict mode enabled and build package {pkgname} needs building."
+                            " Please build it manually first or build without --strict to build"
+                            " it automatically."
+                        )
+                    queue_build(aport, apkbuild, get_depends(context, apkbuild))
+
     if not len(build_queue):
         return []
 
@@ -426,6 +452,12 @@ def packages(
     for item in build_queue:
         logging.info(f"   @BLUE@*@END@ {item['channel']}/{item['name']}")
 
+    if len(build_queue) > 1 and src:
+        raise RuntimeError(
+            "Additional packages need building, please build them first and then"
+            " build the package with --src again."
+        )
+
     cross = None
 
     for pkg in build_queue:

pmb/build/autodetect.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 from pmb.core.arch import Arch
 from pmb.helpers import logging
-from typing import Any, Optional, Union
+from typing import Any
 
 import pmb.config
 import pmb.chroot.apk
@@ -15,7 +15,7 @@ from pmb.types import CrossCompileType
 
 
 # FIXME (#2324): type hint Arch
-def arch_from_deviceinfo(pkgname, aport: Path) -> Optional[Arch]:
+def arch_from_deviceinfo(pkgname, aport: Path) -> Arch | None:
     """
     The device- packages are noarch packages. But it only makes sense to build
     them for the device's architecture, which is specified in the deviceinfo
@@ -39,7 +39,7 @@ def arch_from_deviceinfo(pkgname, aport: Path) -> Optional[Arch]:
 
 
 @Cache("package")
-def arch(package: Union[str, dict[str, Any]]):
+def arch(package: str | dict[str, Any]):
     """
     Find a good default in case the user did not specify for which architecture
     a package should be built.

pmb/build/init.py

@@ -15,16 +15,19 @@ from pmb.core import Chroot
 from pmb.core.context import get_context
 
 
-def init_abuild_minimal(chroot: Chroot = Chroot.native(), additional_pkgs: list[str] = []):
+def init_abuild_minimal(chroot: Chroot = Chroot.native(), build_pkgs: list[str] = []):
     """Initialize a minimal chroot with abuild where one can do 'abuild checksum'."""
     marker = chroot / "tmp/pmb_chroot_abuild_init_done"
     if os.path.exists(marker):
         return
 
+    if not build_pkgs:
+        build_pkgs = pmb.config.build_packages
+
     # pigz is multithreaded and makes compression must faster, we install it in the native
     # chroot and then symlink it into the buildroot so we aren't running it through QEMU.
     # pmb.chroot.apk.install(["pigz"], Chroot.native(), build=False)
-    pmb.chroot.apk.install(["abuild"] + additional_pkgs, chroot, build=False)
+    pmb.chroot.apk.install(build_pkgs, chroot, build=False)
 
     # Fix permissions
     pmb.chroot.root(["chown", "root:abuild", "/var/cache/distfiles"], chroot)
@@ -47,7 +50,7 @@ def init(chroot: Chroot = Chroot.native()) -> bool:
     # Initialize chroot, install packages
     pmb.chroot.init(Chroot.native())
     pmb.chroot.init(chroot)
-    init_abuild_minimal(chroot, additional_pkgs=pmb.config.build_packages)
+    init_abuild_minimal(chroot)
 
     # Generate package signing keys
     if not os.path.exists(get_context().config.work / "config_abuild/abuild.conf"):

pmb/chroot/apk.py

@@ -238,7 +238,7 @@ def install_run_apk(to_add: list[str], to_add_local: list[Path], to_del: list[st
             pmb.chroot.root(["apk", "--no-progress"] + command, chroot)
 
 
-def install(packages, chroot: Chroot, build=True):
+def install(packages, chroot: Chroot, build=True, quiet: bool = False):
     """
     Install packages from pmbootstrap's local package index or the pmOS/Alpine
     binary package mirrors. Iterate over all dependencies recursively, and
@@ -272,7 +272,8 @@ def install(packages, chroot: Chroot, build=True):
 
     to_add_local = packages_get_locally_built_apks(to_add, arch)
 
-    logging.info(f"({chroot}) install {' '.join(packages)}")
+    if not quiet:
+        logging.info(f"({chroot}) install {' '.join(packages)}")
     install_run_apk(to_add, to_add_local, to_del, chroot)
 
 

pmb/chroot/init.py

@@ -154,14 +154,10 @@ def init(chroot: Chroot, usr_merge=UsrMerge.AUTO):
     # Install alpine-base
     pmb.helpers.repo.update(arch)
     pkgs = ["alpine-base"]
-    # install apk static in the native chroot so we can run it
-    # we have a forked apk for systemd and this is the easiest
-    # way to install/run it.
-    if chroot.type == ChrootType.NATIVE:
-        pkgs += ["apk-tools-static"]
-    pmb.chroot.apk_static.run(
-        ["--root", chroot.path, "--cache-dir", apk_cache, "--initdb", "--arch", arch, "add"] + pkgs
-    )
+    cmd = ["--root", chroot.path, "--cache-dir", apk_cache, "--initdb", "--arch", arch]
+    for channel in pmb.config.pmaports.all_channels():
+        cmd += ["--repository", config.work / "packages" / channel]
+    pmb.chroot.apk_static.run(cmd + ["add", *pkgs])
 
     # Merge /usr
     if usr_merge is UsrMerge.AUTO and pmb.config.is_systemd_selected(config):
@@ -182,13 +178,3 @@ def init(chroot: Chroot, usr_merge=UsrMerge.AUTO):
                 pmb.chroot.root(["mkdir", "-p", target], chroot)
             pmb.chroot.user(["ln", "-s", target, link_name], chroot)
             pmb.chroot.root(["chown", "pmos:pmos", target], chroot)
-
-    # Upgrade packages in the chroot, in case alpine-base, apk, etc. have been
-    # built from source with pmbootstrap
-    command = ["--no-network", "upgrade", "-a"]
-
-    # Ignore missing repos before initial build (bpo#137)
-    if os.getenv("PMB_APK_FORCE_MISSING_REPOSITORIES") == "1":
-        command = ["--force-missing-repositories"] + command
-
-    pmb.chroot.root(["apk"] + command, chroot)

pmb/commands/__init__.py

@@ -64,7 +64,7 @@ def run_command(args: PmbArgs):
     command: Command
     # Would be nice to use match case but we support Python 3.8
     if args.action == "log":
-        command = Log(args.clear_log, int(args.lines))
+        command = Log(args.clear_log, args.lines)
     elif args.action == "index":
         # FIXME: should index support --arch?
         command = Index()

pmb/commands/repo_bootstrap.py

 # Copyright 2024 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
-from typing import Optional
 from pmb.core.arch import Arch
 from pmb.core.chroot import Chroot, ChrootType
 from pmb.core.context import Context
@@ -43,7 +42,7 @@ class RepoBootstrap(commands.Command):
             f"Couldn't find section 'repo:{self.repo}' in pmaports.cfg of" " current branch"
         )
 
-    def __init__(self, arch: Optional[Arch], repository: str):
+    def __init__(self, arch: Arch | None, repository: str):
         context = get_context()
         if arch:
             self.arch = arch

pmb/config/__init__.py

@@ -209,7 +209,9 @@ chroot_outdated = 3600 * 24 * 2
 
 # Packages that will be installed in a chroot before it builds packages
 # for the first time
-build_packages = ["abuild", "build-base", "ccache", "git"]
+# IMPORTANT: the order here matters, it is the order these packages will
+# be built in (if needed). abuild must be first!
+build_packages = ["abuild", "apk-tools", "build-base", "ccache", "git"]
 
 #
 # PARSE

pmb/config/file.py

@@ -29,16 +29,6 @@ def load(path: Path) -> Config:
         # default values won't be set in the config file
         if key not in cfg["pmbootstrap"]:
             continue
-        elif key == "mirror_alpine":
-            # DEPRCATED
-            config.mirrors["alpine"] = cfg["pmbootstrap"]["mirror_alpine"]
-            continue
-        # Handle whacky type conversions
-        elif key == "mirrors_postmarketos":
-            mirrors = cfg["pmbootstrap"]["mirrors_postmarketos"].split(",")
-            if len(mirrors) > 1:
-                logging.warning("Multiple mirrors are not supported, using the last one")
-            config.mirrors["pmaports"] = mirrors[-1].strip("/master")
         # Convert strings to paths
         elif type(getattr(Config, key)) is PosixPath:
             setattr(config, key, Path(cfg["pmbootstrap"][key]))
@@ -57,11 +47,6 @@ def load(path: Path) -> Config:
         elif key in cfg["pmbootstrap"]:
             setattr(config, key, cfg["pmbootstrap"][key])
 
-    # One time migration "mirror_alpine" -> mirrors.alpine
-    if "mirror_alpine" in cfg["pmbootstrap"] or "mirrors_postmarketos" in cfg["pmbootstrap"]:
-        logging.info("Migrating config file to 3.0 format.")
-        save(path, config)
-
     return config
 
 
@@ -85,9 +70,6 @@ def serialize(config: Config, skip_defaults=True) -> configparser.ConfigParser:
         # we wouldn't be able to tell if the user overwrote it.
         if skip_defaults and Config.get_default(key) == getattr(config, key):
             continue
-        if key == "mirror_alpine" or key == "mirrors_postmarketos":
-            # DEPRECATED: skip these
-            continue
         if key == "providers":
             cfg["providers"] = config.providers
         elif key.startswith("mirrors."):

pmb/config/init.py

@@ -10,7 +10,7 @@ import glob
 import json
 import os
 import shutil
-from typing import Any, Optional
+from typing import Any
 
 import pmb.aportgen
 import pmb.config
@@ -539,10 +539,13 @@ def ask_for_additional_options(config):
 
     # Mirrors
     # prompt for mirror change
-    logging.info("Selected mirror:" f" {','.join(context.config.mirrors_postmarketos)}")
+    logging.info("Selected mirror:" f" {context.config.mirrors['pmaports']}")
     if pmb.helpers.cli.confirm("Change mirror?", default=False):
         mirror = ask_for_mirror()
         config.mirrors["pmaports"] = mirror
+        # FIXME: this path will change once the systemd repository is
+        # integrated into bpo (fixing this is a tasks in bpo#140)
+        config.mirrors["systemd"] = os.path.join(mirror, "staging/systemd/")
 
 
 def ask_for_mirror():
@@ -600,7 +603,7 @@ def ask_for_mirror():
     return mirror
 
 
-def ask_for_hostname(default: Optional[str], device):
+def ask_for_hostname(default: str | None, device):
     while True:
         ret = pmb.helpers.cli.ask(
             "Device hostname (short form, e.g. 'foo')", None, (default or device), True
@@ -672,8 +675,16 @@ def frontend(args: PmbArgs):
 
     # Work folder (needs to be first, so we can create chroots early)
     config = pmb.config.load(args.config)
+    # Update context to point to new config
+    get_context().config = config
+
     config.work, work_exists = ask_for_work_path(args)
 
+    # If the work dir is not the default, reset aports and make
+    # it relative to the work dir
+    if not config.aports[0].is_relative_to(config.work):
+        config.aports = [config.work / "cache_git/pmaports"]
+
     # Update args and save config (so chroots and 'pmbootstrap log' work)
     # pmb.helpers.args.update_work(args, config.work)
     pmb.config.save(args.config, config)

pmb/config/pmaports.py

@@ -2,7 +2,6 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 import configparser
 from pathlib import Path
-from typing import Optional
 from pmb.core.pkgrepo import pkgrepo_default_path, pkgrepo_paths, pkgrepo_relative_path
 from pmb.helpers import logging
 import os
@@ -89,7 +88,7 @@ def read_config_repos():
 
 
 @Cache("aports")
-def read_config(aports: Optional[Path] = None):
+def read_config(aports: Path | None = None):
     """Read and verify pmaports.cfg. If aports is not
     specified and systemd is enabled, the returned channel
     will be the systemd one (e.g. systemd-edge instead of edge)

pmb/config/sudo.py

@@ -3,11 +3,10 @@
 import os
 import shutil
 from functools import lru_cache
-from typing import Optional
 
 
 @lru_cache
-def which_sudo() -> Optional[str]:
+def which_sudo() -> str | None:
     """Return a command required to run commands as root, if any.
 
     Find whether sudo or doas is installed for commands that require root.