CI: expand integration tests (MR 2453)

Refactor the integration test script to support running a variety of
tests, and add CI jobs for them.

Additionally, move the integration tests to their own stage so we don't
potentially waste a bunch of CI time running them on broken branches.

Signed-off-by: Caleb Connolly <caleb@postmarketos.org>

.ci/integration.sh

 #!/bin/sh -e
 
-set -x
+echo "\$@:" "$@"
 
 if [ "$(id -u)" = 0 ]; then
-	exec su "${TESTUSER:-build}" -c "sh -e $0"
+	exec su "${TESTUSER:-build}" -c "sh -ec '$0 $*'"
 fi
 
+test="$(basename "$0")"
+
+usage() {
+	echo "Usage: $test $1"
+	exit 1
+}
+
 pmbootstrap() {
-	./pmbootstrap.py --details-to-stdout "$@"
+	printf "\033[0;32m\$ pmbootstrap %s\033[0m\n" "$*"
+	./pmbootstrap.py --details-to-stdout -y "$@"
 }
 
 # Make sure that the work folder format is up to date, and that there are no
@@ -18,17 +26,93 @@ yes '' | ./pmbootstrap.py --details-to-stdout init
 pmbootstrap work_migrate
 pmbootstrap -q shutdown
 
-# TODO: make device configurable?
-device="qemu-amd64"
-# TODO: make UI configurable?
-ui="phosh"
-pmbootstrap config device "$device"
-pmbootstrap config ui "$ui"
+# Default for tests where the device doesn't matter
+pmbootstrap config device qemu-amd64
+
+# A test that builds normal and FDE images for the given device/ui
+build_images() {
+	device="$1"
+	ui="$2"
+	if [ -z "$ui" ] || [ -z "$device" ]; then
+		usage "<device> <ui>"
+	fi
+	pmbootstrap config device "$device"
+	pmbootstrap config ui "$ui"
+
+	# NOTE: --no-image is used because building images makes pmb try to
+	# "modprobe loop". This fails in CI.
+	echo "Building $ui image for $device"
+	pmbootstrap install --zap --password 147147 --no-image
+
+	echo "Building $ui image for $device, with FDE"
+	pmbootstrap install --zap --password 147147 --fde --no-image
+}
+
+force_build() {
+	arch="$1"
+	shift
+	packages="$*"
+	if [ -z "$arch" ] || [ -z "$packages" ]; then
+		usage "<arch> <packages...>"
+	fi
+
+	echo "Force building $packages for $arch"
+	# shellcheck disable=SC2086
+	pmbootstrap build --force --arch "$arch" $packages
+}
+
+strict_build() {
+	arch="$1"
+	shift
+	packages="$*"
+	if [ -z "$arch" ] || [ -z "$packages" ]; then
+		usage "<arch> <packages...>"
+	fi
+
+	echo "Strict building $packages for $arch"
+	# shellcheck disable=SC2086
+	pmbootstrap build --strict --arch "$arch" $packages
+}
+
+force_strict_build() {
+	arch="$1"
+	shift
+	packages="$*"
+	if [ -z "$arch" ] || [ -z "$packages" ]; then
+		usage "<arch> <packages...>"
+	fi
+
+	echo "Force building $packages for $arch"
+	# shellcheck disable=SC2086
+	pmbootstrap build --force --strict --arch "$arch" $packages
+}
+
+bump_autobuild() {
+	device="$1"
+	package="$2"
+	if [ -z "$device" ] || [ -z "$package" ]; then
+		usage "<device> <package>"
+	fi
+
+	pmbootstrap config device "$device"
 
-# NOTE: --no-image is used because building images makes pmb try to
-# "modprobe loop". This fails in CI.
-echo "Building $ui image for $device"
-pmbootstrap -y install --zap --password 147147 --no-image
+	echo "Bumping pkgrel of $package"
+	# shellcheck disable=SC2086
+	pmbootstrap pkgrel_bump $package
+
+	echo "Ensuring package is built during install"
+	pmbootstrap config ui none
+	# shellcheck disable=SC2086
+	pmbootstrap install --no-image --password 147147 --add $package
+	pkgs="$(find "$(./pmbootstrap.py config work)/packages/" -type f)"
+	if ! echo "$pkgs" | grep -q "$package"; then
+		echo "Package $package not found in built packages:"
+		echo "$pkgs"
+		exit 1
+	fi
+}
 
-echo "Building $ui image for $device, with FDE"
-pmbootstrap -y install --zap --password 147147 --fde --no-image
+# Run the test
+echo "Running $test $*"
+"$test" "$@"
+echo "Test $test passed!"

.ci/integration_tests/build_images

+../integration.sh
\ No newline at end of file

.ci/integration_tests/bump_autobuild

+../integration.sh
\ No newline at end of file

.ci/integration_tests/force_build

+../integration.sh
\ No newline at end of file

.ci/integration_tests/force_strict_build

+../integration.sh
\ No newline at end of file

.ci/integration_tests/strict_build

+../integration.sh
\ No newline at end of file

.gitlab-ci.yml

@@ -22,6 +22,7 @@ stages:
   - lint
   - deploy
   - test
+  - integration-test
 
 codespell:
   stage: lint
@@ -99,12 +100,82 @@ deploy:
   environment:
     name: deploy
 
-integration:
-  stage: test
+.integration:
+  stage: integration-test
   before_script:
     - *global_before_scripts
     - apk upgrade -U
     - apk add doas git losetup multipath-tools python3 openssl
     - echo "permit nopass build" > /etc/doas.d/ci-user.conf
+  # Add built packages as artifacts so they can be inspected if needed.
+  after_script:
+    - mkdir packages
+    - mv /home/build/.local/var/pmbootstrap/packages/* packages/
+  artifacts:
+    paths: [packages]
+    expire_in: 1 week
+
+# Test that we can generate an install rootfs on AMD64 for native
+# and arm64 QEMU targets
+install amd64:
+  extends: .integration
   script:
-    - ".ci/integration.sh"
+  # Native AMD64 on AMD64
+    - .ci/integration_tests/build_images qemu-amd64 console
+  # ARM64 on AMD64
+    - .ci/integration_tests/build_images postmarketos-trailblazer console
+
+install aarch64:
+  extends: .integration
+  tags: [arm64]
+  script:
+  # Native ARM64 on ARM64
+    - .ci/integration_tests/build_images postmarketos-trailblazer console
+  # AMD64 on ARM64
+  # This is currently broken on the ARM64 runners, see infra#202
+  # so disable it until OSUOSL can fix it.
+    # - .ci/integration_tests/build_images qemu-amd64 console
+
+# Test building packages with --force on AMD64 for AMD64, aarch64, and armv7
+force build amd64:
+  extends: .integration
+  script:
+  # Packages chosen for being small, unlikely to break, and for language coverage.
+    - .ci/integration_tests/force_build x86_64 hello-world postmarketos-mkinitfs
+    - .ci/integration_tests/force_build aarch64 hello-world postmarketos-mkinitfs
+    - .ci/integration_tests/force_build armv7 hello-world postmarketos-mkinitfs
+
+# Same again but on the arm64 runner
+force build aarch64:
+  extends: .integration
+  tags: [arm64]
+  script:
+  # Packages chosen for being small, unlikely to break, and for language coverage.
+  # The ARM64 runner seems to have QEMU binfmt issues.
+    # - .ci/integration_tests/force_build x86_64 hello-world pbsplash #postmarketos-mkinitfs
+    - .ci/integration_tests/force_build aarch64 hello-world postmarketos-mkinitfs
+    # - .ci/integration_tests/force_build armv7 hello-world postmarketos-mkinitfs
+
+# Test building packages with --strict on AMD64 for ARM64
+# since the cross-build codepath is the more complicated one.
+strict build amd64:
+  extends: .integration
+  script:
+  # Packages chosen for being small, unlikely to break, and for language coverage.
+    - .ci/integration_tests/strict_build aarch64 hello-world postmarketos-mkinitfs
+
+# Test building with --force --strict, for diversity run this on the ARM64 runner
+force_strict build aarch64:
+  extends: .integration
+  tags: [arm64]
+  script:
+  # Packages chosen for being small, unlikely to break, and for language coverage.
+  # The ARM64 runner seems to have QEMU binfmt issues.
+    # - .ci/integration_tests/force_strict_build x86_64 hello-world postmarketos-mkinitfs
+    - .ci/integration_tests/force_strict_build aarch64 hello-world postmarketos-mkinitfs
+
+pkgrel_bump amd64:
+  extends: .integration
+  script:
+    - .ci/integration_tests/bump_autobuild qemu-amd64 hello-world
+