From dcc4137ee881e794b670bfcc9ee66241236ce5f8 Mon Sep 17 00:00:00 2001
From: Caleb Connolly <caleb@postmarketos.org>
Date: Sun, 16 Mar 2025 15:10:30 +0000
Subject: [PATCH] FIXUP: build: abstract CrossCompile type logic
* Make CrossCompile a proper enum type rather than a string literal,
* Introduce methods to get the correct host/build chroots depending on the
cross compile type and target architecture.
* Remove autodetect.chroot() since it doesn't do what we expect, adjust
all users to use cross.build_chroot() instead.
* Refactor package building to correctly use cross.host_chroot() and
cross.build_chroot().
Signed-off-by: Caleb Connolly <caleb@postmarketos.org>
Part-of: https://gitlab.postmarketos.org/postmarketOS/pmbootstrap/-/merge_requests/2568
---
pmb/build/_package.py | 46 +++++++++++++++------------------
pmb/build/autodetect.py | 27 ++++++-------------
pmb/build/backend.py | 46 ++++++++++++++++++---------------
pmb/build/envkernel.py | 5 +---
pmb/build/init.py | 8 +++---
pmb/build/kconfig.py | 2 +-
pmb/types.py | 57 +++++++++++++++++++++++++++++++++++------
7 files changed, 107 insertions(+), 84 deletions(-)
diff --git a/pmb/build/_package.py b/pmb/build/_package.py
index 0ad7fb0da..401a815fd 100644
--- a/pmb/build/_package.py
+++ b/pmb/build/_package.py
@@ -10,7 +10,7 @@ from pmb.core.arch import Arch
from pmb.core.context import Context
from pmb.core.pkgrepo import pkgrepo_relative_path
from pmb.helpers import logging
-from pmb.types import Apkbuild, CrossCompileType
+from pmb.types import Apkbuild, CrossCompile
from pathlib import Path
import pmb.build
@@ -217,8 +217,7 @@ class BuildQueueItem(TypedDict):
output_path: Path
channel: str
depends: list[str]
- cross: CrossCompileType
- chroot: Chroot
+ cross: CrossCompile
def has_cyclical_dependency(
@@ -493,7 +492,7 @@ def packages(
aports: Path,
apkbuild: dict[str, Any],
depends: list[str],
- cross: CrossCompileType = "autodetect",
+ cross: CrossCompile | None = None,
) -> list[str]:
# Skip if already queued
name = apkbuild["pkgname"]
@@ -540,7 +539,6 @@ def packages(
),
"channel": channel,
"depends": depends,
- "chroot": chroot,
"cross": cross,
}
)
@@ -613,16 +611,19 @@ def packages(
" build the package with --src again."
)
- cross = "autodetect"
- prev_cross = "autodetect"
+ cross = None
+ prev_cross = None
hostchroot = None # buildroot for the architecture we're building for
total_pkgs = len(build_queue)
count = 0
for pkg in build_queue:
count += 1
- hostchroot = chroot = pkg["chroot"]
+ prev_cross = cross
+ cross = pkg["cross"]
pkg_arch = pkg["arch"]
+ hostchroot = cross.host_chroot(pkg_arch)
+ buildchroot = cross.build_chroot(pkg_arch)
apkbuild = pkg["apkbuild"]
channel = pkg["channel"]
@@ -648,25 +649,19 @@ def packages(
)
)
- # (re)-initialize the cross compiler stuff when cross method changes
- prev_cross = cross
- cross = pmb.build.autodetect.crosscompile(pkg["apkbuild"], pkg_arch)
- if cross == "cross-native2" or cross == "cross-native":
- chroot = Chroot.native()
-
# One time chroot initialization
- if hostchroot != chroot:
+ if hostchroot != buildchroot:
pmb.build.init(hostchroot)
- if pmb.build.init(chroot):
- pmb.build.other.configure_abuild(chroot)
- pmb.build.other.configure_ccache(chroot)
+ if pmb.build.init(buildchroot):
+ pmb.build.other.configure_abuild(buildchroot)
+ pmb.build.other.configure_ccache(buildchroot)
if "rust" in all_dependencies or "cargo" in all_dependencies:
- pmb.chroot.apk.install(["sccache"], chroot)
+ pmb.chroot.apk.install(["sccache"], buildchroot)
- if cross != prev_cross and cross not in ["unnecessary", "qemu-only"]:
+ if cross != prev_cross and cross.enabled():
pmb.build.init_compiler(context, pkg_depends, cross, pkg_arch)
- if cross == "crossdirect":
- pmb.chroot.mount_native_into_foreign(chroot)
+ if cross == CrossCompile.CROSSDIRECT:
+ pmb.chroot.mount_native_into_foreign(buildchroot)
depends_build: list[str] = []
depends_host: list[str] = []
@@ -702,11 +697,11 @@ def packages(
logging.info("*** Install build dependencies")
if src:
depends_build.append("rsync")
- pmb.chroot.apk.install(depends_build, chroot, build=False)
+ pmb.chroot.apk.install(depends_build, buildchroot, build=False)
# Build and finish up
msg = f"@YELLOW@=>@END@ @BLUE@{channel}/{pkg['name']}@END@: Building package"
- if cross != "unnecessary":
+ if cross != CrossCompile.UNNECESSARY:
msg += f" (cross compiling: {cross})"
logging.info(msg)
@@ -720,13 +715,12 @@ def packages(
cross,
strict,
force,
- hostchroot,
src,
bootstrap_stage,
)
except RuntimeError:
raise BuildFailedError(f"Couldn't build {output}!")
- finish(pkg["apkbuild"], channel, pkg_arch, output, chroot, strict)
+ finish(pkg["apkbuild"], channel, pkg_arch, output, buildchroot, strict)
# Clear package cache for the next run
_package_cache = {}
diff --git a/pmb/build/autodetect.py b/pmb/build/autodetect.py
index b42df7b52..b0bde3925 100644
--- a/pmb/build/autodetect.py
+++ b/pmb/build/autodetect.py
@@ -7,10 +7,9 @@ from pmb.helpers import logging
import pmb.config
import pmb.chroot.apk
import pmb.helpers.pmaports
-from pmb.core import Chroot
from pmb.core.context import get_context
from pmb.meta import Cache
-from pmb.types import Apkbuild, CrossCompileType
+from pmb.types import Apkbuild, CrossCompile
def arch_from_deviceinfo(pkgname: str, aport: Path) -> Arch | None:
@@ -82,26 +81,16 @@ def arch(package: str | Apkbuild) -> Arch:
return Arch.native()
-def chroot(apkbuild: Apkbuild, arch: Arch) -> Chroot:
- if arch == Arch.native():
- return Chroot.native()
-
- if "pmb:cross-native" in apkbuild["options"]:
- return Chroot.native()
-
- return Chroot.buildroot(arch)
-
-
-def crosscompile(apkbuild: Apkbuild, arch: Arch) -> CrossCompileType:
+def crosscompile(apkbuild: Apkbuild, arch: Arch) -> CrossCompile:
"""Decide the type of compilation necessary to build a given APKBUILD."""
if not get_context().cross:
- return "qemu-only"
+ return CrossCompile.QEMU_ONLY
if not arch.cpu_emulation_required():
- return "unnecessary"
+ return CrossCompile.UNNECESSARY
if "pmb:cross-native" in apkbuild["options"]:
- return "cross-native"
+ return CrossCompile.CROSS_NATIVE
if arch.is_native() or "pmb:cross-native2" in apkbuild["options"]:
- return "cross-native2"
+ return CrossCompile.CROSS_NATIVE2
if "!pmb:crossdirect" in apkbuild["options"]:
- return "qemu-only"
- return "crossdirect"
+ return CrossCompile.QEMU_ONLY
+ return CrossCompile.CROSSDIRECT
diff --git a/pmb/build/backend.py b/pmb/build/backend.py
index 752c2c97c..ad426039c 100644
--- a/pmb/build/backend.py
+++ b/pmb/build/backend.py
@@ -11,7 +11,7 @@ from pmb.core import Context
from pmb.core.arch import Arch
from pmb.core.chroot import Chroot
from pmb.helpers import logging
-from pmb.types import Apkbuild, CrossCompileType, Env
+from pmb.types import Apkbuild, CrossCompile, Env
class BootstrapStage(enum.IntEnum):
@@ -190,10 +190,9 @@ def run_abuild(
pkgver: str,
channel: str,
arch: Arch,
- cross: CrossCompileType,
+ cross: CrossCompile,
strict: bool = False,
force: bool = False,
- hostchroot: Chroot = Chroot.native(),
src: str | None = None,
bootstrap_stage: int = BootstrapStage.NONE,
) -> None:
@@ -210,18 +209,23 @@ def run_abuild(
the environment variables dict generated in this function.
"""
# Sanity check
- if cross == "cross-native" and "!tracedeps" not in apkbuild["options"]:
+ if cross == CrossCompile.CROSS_NATIVE and "!tracedeps" not in apkbuild["options"]:
logging.warning(
"WARNING: Option !tracedeps is not set, but cross compiling with"
" cross-native (version 1). This will probably fail!"
)
+ hostchroot = cross.host_chroot(arch)
+ buildchroot = cross.build_chroot(arch)
+
# For cross-native2 compilation, bindmount the "host" rootfs to /mnt/sysroot
# it will be used as the "sysroot"
- if cross == "cross-native2":
- pmb.mount.bind(hostchroot.path, Chroot.native() / "/mnt/sysroot", umount=True)
-
- chroot = Chroot.native() if cross == "cross-native2" else hostchroot
+ if cross == CrossCompile.CROSS_NATIVE2:
+ if buildchroot != Chroot.native():
+ raise ValueError(
+ "Trying to use cross-native2 build buildchroot != native! This is a bug"
+ )
+ pmb.mount.bind(hostchroot.path, buildchroot / "/mnt/sysroot", umount=True)
pkgdir = context.config.work / "packages" / channel
if not pkgdir.exists():
@@ -241,16 +245,16 @@ def run_abuild(
["rm", "-f", "/home/pmos/packages/pmos"],
["ln", "-sf", f"/mnt/pmbootstrap/packages/{channel}", "/home/pmos/packages/pmos"],
],
- chroot,
+ buildchroot,
)
# Environment variables
env: Env = {"SUDO_APK": "abuild-apk --no-progress"}
- if cross == "cross-native":
+ if cross == CrossCompile.CROSS_NATIVE:
hostspec = arch.alpine_triple()
env["CROSS_COMPILE"] = hostspec + "-"
env["CC"] = hostspec + "-gcc"
- if cross == "cross-native2":
+ if cross == CrossCompile.CROSS_NATIVE2:
env["CHOST"] = str(arch)
env["CBUILDROOT"] = "/mnt/sysroot"
env["CFLAGS"] = "-Wl,-rpath-link=/mnt/sysroot/usr/lib"
@@ -261,7 +265,7 @@ def run_abuild(
except ValueError:
logging.debug(f"Not setting $GOARCH for {arch}")
- elif cross == "crossdirect":
+ elif cross == CrossCompile.CROSSDIRECT:
env["PATH"] = ":".join([f"/native/usr/lib/crossdirect/{arch}", pmb.config.chroot_path])
else:
env["CARCH"] = str(arch)
@@ -269,7 +273,7 @@ def run_abuild(
env["CCACHE_DISABLE"] = "1"
# Use sccache without crossdirect (crossdirect uses it via rustc.sh)
- if context.ccache and cross != "crossdirect":
+ if context.ccache and cross != CrossCompile.CROSSDIRECT:
env["RUSTC_WRAPPER"] = "/usr/bin/sccache"
# Cache binary objects from go in this path (like ccache)
@@ -300,16 +304,16 @@ def run_abuild(
cmd += ["-K"]
# Copy the aport to the chroot and build it
- pmb.build.copy_to_buildpath(apkbuild["pkgname"], chroot, no_override=strict)
+ pmb.build.copy_to_buildpath(apkbuild["pkgname"], buildchroot, no_override=strict)
if src and strict:
- logging.debug(f"({chroot}) Ensuring previous build artifacts are removed")
- pmb.chroot.root(["rm", "-rf", "/tmp/pmbootstrap-local-source-copy"], chroot)
- override_source(apkbuild, pkgver, src, chroot)
- link_to_git_dir(chroot)
+ logging.debug(f"({buildchroot}) Ensuring previous build artifacts are removed")
+ pmb.chroot.root(["rm", "-rf", "/tmp/pmbootstrap-local-source-copy"], buildchroot)
+ override_source(apkbuild, pkgver, src, buildchroot)
+ link_to_git_dir(buildchroot)
try:
- pmb.chroot.user(cmd, chroot, Path("/home/pmos/build"), env=env)
+ pmb.chroot.user(cmd, buildchroot, Path("/home/pmos/build"), env=env)
finally:
- handle_csum_failure(apkbuild, chroot)
+ handle_csum_failure(apkbuild, buildchroot)
- pmb.helpers.run.root(["umount", Chroot.native() / "/mnt/sysroot"], output="null", check=False)
+ pmb.helpers.run.root(["umount", buildchroot / "/mnt/sysroot"], output="null", check=False)
diff --git a/pmb/build/envkernel.py b/pmb/build/envkernel.py
index 095995c40..a16ab2a9c 100644
--- a/pmb/build/envkernel.py
+++ b/pmb/build/envkernel.py
@@ -247,10 +247,7 @@ def package_kernel(args: PmbArgs) -> None:
if not kbuild_out:
kbuild_out = ".output"
- if "pmb:cross-native" in apkbuild["options"]:
- chroot = Chroot.native()
- else:
- chroot = pmb.build.autodetect.chroot(apkbuild, arch)
+ chroot = Chroot.native()
# Install package dependencies
depends = pmb.build.get_depends(context, apkbuild)
diff --git a/pmb/build/init.py b/pmb/build/init.py
index 3089aa1dd..58e15535b 100644
--- a/pmb/build/init.py
+++ b/pmb/build/init.py
@@ -13,7 +13,7 @@ import pmb.chroot.apk
import pmb.helpers.run
from pmb.core import Chroot
from pmb.core.context import get_context
-from pmb.types import CrossCompileType
+from pmb.types import CrossCompile
def init_abuild_minimal(chroot: Chroot = Chroot.native(), build_pkgs: list[str] = []) -> None:
@@ -112,9 +112,7 @@ def init(chroot: Chroot = Chroot.native()) -> bool:
return True
-def init_compiler(
- context: Context, depends: list[str], cross: CrossCompileType, arch: Arch
-) -> None:
+def init_compiler(context: Context, depends: list[str], cross: CrossCompile, arch: Arch) -> None:
arch_str = str(arch)
cross_pkgs = ["ccache-cross-symlinks", "abuild"]
if "gcc4" in depends:
@@ -125,7 +123,7 @@ def init_compiler(
cross_pkgs += ["gcc-" + arch_str, "g++-" + arch_str]
if "clang" in depends or "clang-dev" in depends:
cross_pkgs += ["clang"]
- if cross == "crossdirect":
+ if cross == CrossCompile.CROSSDIRECT:
cross_pkgs += ["crossdirect"]
if "rust" in depends or "cargo" in depends:
if context.ccache:
diff --git a/pmb/build/kconfig.py b/pmb/build/kconfig.py
index 0dd5ce47b..d5da0bd53 100644
--- a/pmb/build/kconfig.py
+++ b/pmb/build/kconfig.py
@@ -177,8 +177,8 @@ def _init(pkgname: str, arch: Arch | None) -> tuple[str, Arch, Any, Chroot, Env]
if arch is None:
arch = get_arch(apkbuild)
- chroot = pmb.build.autodetect.chroot(apkbuild, arch)
cross = pmb.build.autodetect.crosscompile(apkbuild, arch)
+ chroot = cross.build_chroot(arch)
hostspec = arch.alpine_triple()
# Set up build tools and makedepends
diff --git a/pmb/types.py b/pmb/types.py
index bfbe59d4c..2edaa3b29 100644
--- a/pmb/types.py
+++ b/pmb/types.py
@@ -1,21 +1,62 @@
# Copyright 2024 Caleb Connolly
# SPDX-License-Identifier: GPL-3.0-or-later
+import enum
import subprocess
from argparse import Namespace
from pathlib import Path
from typing import Any, Literal, TypedDict
from pmb.core.arch import Arch
+from pmb.core.chroot import Chroot
+
+
+class CrossCompile(enum.Enum):
+ # Cross compilation isn't needed for this package
+ UNNECESSARY = "unnecessary"
+ # Cross compilation disabled, only use QEMU
+ QEMU_ONLY = "qemu-only"
+ # Cross compilation will use crossdirect
+ CROSSDIRECT = "crossdirect"
+ # Cross compilation will use cross-native
+ CROSS_NATIVE = "cross-native"
+ # Cross compilation will use cross-native2
+ CROSS_NATIVE2 = "cross-native2"
+
+ def __str__(self) -> str:
+ return self.value
+
+ def enabled(self) -> bool:
+ """Are we cross-compiling for this value of cross?"""
+ return self not in [CrossCompile.UNNECESSARY, CrossCompile.QEMU_ONLY]
+
+ def host_chroot(self, arch: Arch) -> Chroot:
+ """Chroot for the package target architecture (the "host" machine).
+ Cross native (v1) is the exception, since we exclusively use the native
+ chroot for that."""
+ if arch == Arch.native():
+ return Chroot.native()
+
+ match self:
+ case CrossCompile.CROSS_NATIVE:
+ return Chroot.native()
+ case _:
+ return Chroot.buildroot(arch)
+
+ def build_chroot(self, arch: Arch) -> Chroot:
+ """Chroot for the package build architecture (the "build" machine)."""
+ if arch == Arch.native():
+ return Chroot.native()
+
+ match self:
+ case CrossCompile.CROSSDIRECT | CrossCompile.QEMU_ONLY:
+ return Chroot.buildroot(arch)
+ # FIXME: are there cases where we're building for a different arch
+ # but don't need to cross compile?
+ case CrossCompile.UNNECESSARY | CrossCompile.CROSS_NATIVE | CrossCompile.CROSS_NATIVE2:
+ return Chroot.native()
+
-CrossCompileType = Literal[
- "autodetect",
- "unnecessary",
- "qemu-only",
- "crossdirect",
- "cross-native",
- "cross-native2",
-]
RunOutputTypeDefault = Literal["log", "stdout", "interactive", "tui", "null"]
RunOutputTypePopen = Literal["background", "pipe"]
RunOutputType = RunOutputTypeDefault | RunOutputTypePopen
--
GitLab