CI: adjust .ci/common.py users to pmbootstrap ci (MR 3615)

Adjust all CI scripts that make use of .ci/common.py to
'pmbootstrap ci'. Move all scripts that are not direct entry points to
running CI scripts to .ci/lib.

Comment out the dtb check, as it is failing. Apparently it didn't run
properly before. Let's fix this after this CI change is done.

.ci/apkbuild-lint.sh

+#!/bin/sh -e
+# Description: run apkbuild-lint on modified APKBUILDs
+# Options: native
+# Use 'native' because it requires git commit history.
+# https://postmarktos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh"
+	sh ./install_pmbootstrap.sh
+	exec su "${TESTUSER:-pmos}" -c "sh -e $0"
+fi
+
+# Wrap pmbootstrap to use this repository for --aports
+pmaports="$(cd $(dirname $0)/..; pwd -P)"
+_pmbootstrap="$(command -v pmbootstrap)"
+pmbootstrap() {
+	"$_pmbootstrap" --aports="$pmaports" "$@"
+}
+
+.ci/lib/apkbuild_linting.py
+

.ci/build-aarch64.sh

+lib/build_changed_aports.sh
\ No newline at end of file

.ci/build-armhf.sh

+lib/build_changed_aports.sh
\ No newline at end of file

.ci/build-armv7.sh

+lib/build_changed_aports.sh
\ No newline at end of file

.ci/build-riscv64.sh

+lib/build_changed_aports.sh
\ No newline at end of file

.ci/build-x86.sh

+lib/build_changed_aports.sh
\ No newline at end of file

.ci/build-x86_64.sh

+lib/build_changed_aports.sh
\ No newline at end of file

.ci/build.sh

-#!/bin/sh -e
-# Convenience wrapper for short arch-specific build jobs in .gitlab-ci.yml
-
-export PYTHONUNBUFFERED=1
-JOB_ARCH="${CI_JOB_NAME#build-}"
-
-set -x
-su pmos -c ".ci/build_changed_aports.py $JOB_ARCH"

.ci/check-dtb-install-location.sh

-#!/bin/sh -e
-
-if grep -r 'INSTALL_DTBS_PATH="$pkgdir"/usr/share/dtb'; then
-	echo 'Please do not install dtbs to /usr/share/dtb!'
-	echo 'Unless you have a good reason not to, please put them in /boot/dtbs'
-	exit 1
-fi

.ci/commits.sh

+#!/bin/sh -e
+# Copyright 2022 Oliver Smith
+# SPDX-License-Identifier: GPL-3.0-or-later
+# Description: check pkgver/pkgrel bumps, amount of changed pkgs etc
+# Options: native
+# Use 'native' because it requires git commit history.
+# https://postmarktos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	# In .gitlab-ci.yml currently .ci/pytest.sh runs before this and
+	# already downloads and runs install_pmbootstrap.sh.
+	if ! [ -e install_pmbootstrap.sh ]; then
+		wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh"
+		sh ./install_pmbootstrap.sh
+	fi
+	exec su "${TESTUSER:-pmos}" -c "sh -e $0"
+fi
+
+.ci/lib/check_changed_aports_versions.py

.ci/grep.sh

@@ -14,3 +14,11 @@ if grep -qr '(CHANGEME!)' *; then
 	grep --color=always -r '(CHANGEME!)' *
 	exit 1
 fi
+
+# DTBs installed to /usr/share/db
+# FIXME: doesn't pass currently, fix it and enable it afterwards
+# if grep -r 'INSTALL_DTBS_PATH="$pkgdir"/usr/share/dtb'; then
+# 	echo 'Please do not install dtbs to /usr/share/dtb!'
+# 	echo 'Unless you have a good reason not to, please put them in /boot/dtbs'
+# 	exit 1
+# fi

.ci/lib/add_pmbootstrap_to_import_path

+../testcases/add_pmbootstrap_to_import_path
\ No newline at end of file

.ci/build_changed_aports.py → .ci/lib/build_changed_aports.py

@@ -7,7 +7,7 @@ import sys
 import common
 
 # pmbootstrap
-import testcases.add_pmbootstrap_to_import_path
+import add_pmbootstrap_to_import_path
 import pmb.parse
 import pmb.parse._apkbuild
 import pmb.helpers.pmaports

.ci/lib/build_changed_aports.sh

+#!/bin/sh -e
+# Description: build modified packages for this architecture
+# Options: native slow
+# https://postmarktos.org/pmb-ci
+
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh"
+	sh ./install_pmbootstrap.sh
+	exec su "${TESTUSER:-pmos}" -c "sh -e $0"
+fi
+
+export PYTHONUNBUFFERED=1
+
+# Get the architecture from the symlink we are running
+arch="$(echo "$0" | cut -d '-' -f 2 | cut -d '.' -f 1)"
+
+set -x
+.ci/lib/build_changed_aports.py "$arch"

.ci/check_changed_aports_versions.py → .ci/lib/check_changed_aports_versions.py

@@ -11,7 +11,7 @@ import subprocess
 import common
 
 # pmbootstrap
-import testcases.add_pmbootstrap_to_import_path  # noqa
+import add_pmbootstrap_to_import_path  # noqa
 import pmb.parse
 import pmb.parse.version
 import pmb.helpers.logging

.ci/common.py → .ci/lib/common.py

@@ -17,7 +17,7 @@ def get_pmaports_dir():
     global cache
     if "pmaports_dir" in cache:
         return cache["pmaports_dir"]
-    ret = os.path.realpath(os.path.join(os.path.dirname(__file__) + "/.."))
+    ret = os.path.realpath(os.path.join(os.path.dirname(__file__) + "/../.."))
     cache["pmaports_dir"] = ret
     return ret
 

.ci/run_testcases.sh → .ci/pytest.sh

 #!/bin/sh -e
-# Copyright 2021 Oliver Smith
+# Copyright 2022 Oliver Smith
 # SPDX-License-Identifier: GPL-3.0-or-later
+# Description: lint with various python tests
+# Options: native
+# Use 'native' because it requires pmbootstrap.
+# https://postmarktos.org/pmb-ci
 
-# Require pmbootstrap
-if ! command -v pmbootstrap > /dev/null; then
-	echo "ERROR: pmbootstrap needs to be installed."
+if [ "$(id -u)" = 0 ]; then
+	set -x
+	wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh"
+	sh ./install_pmbootstrap.sh pytest
+	exec su "${TESTUSER:-pmos}" -c "sh -e $0"
+fi
+
+# Require pytest to be installed on the host system
+if [ -z "$(command -v pytest)" ]; then
+	echo "ERROR: pytest command not found, make sure it is in your PATH."
 	exit 1
 fi
 

.gitlab-ci.yml

@@ -70,16 +70,13 @@ editor-config:
     - .ci/ec.sh
 
 # aports checks (generic)
-aports-static:
+pytest-commits:
   stage: lint
   <<: *only-default
-  before_script:
-    - wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh"
-    - sh ./install_pmbootstrap.sh pytest
   script:
-    - su pmos -c ".ci/run_testcases.sh"
-    - su pmos -c ".ci/check_changed_aports_versions.py"
-    - su pmos -c ".ci/check-dtb-install-location.sh"
+    - .ci/lib/gitlab_prepare_ci.sh
+    - .ci/pytest.sh
+    - .ci/commits.sh
   artifacts:
     when: on_failure
     paths:
@@ -103,11 +100,9 @@ distfile-check:
 # APKBUILD linting
 aport-lint:
   stage: lint
-  before_script:
-    - wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh"
-    - sh ./install_pmbootstrap.sh
   script:
-    - su pmos -c ".ci/apkbuild-linting.py"
+    - .ci/lib/gitlab_prepare_ci.sh
+    - .ci/apkbuild-lint.sh
   only:
     - merge_requests
   allow_failure: true
@@ -145,10 +140,8 @@ mr-settings:
   stage: build
   <<: *only-default
   before_script:
-    - wget "https://gitlab.com/postmarketOS/ci-common/-/raw/master/install_pmbootstrap.sh"
-    - sh ./install_pmbootstrap.sh
-  script:
-    - .ci/build.sh
+    - .ci/lib/gitlab_prepare_ci.sh
+  after_script:
     - cp -r /home/pmos/.local/var/pmbootstrap/packages/ packages/ || true
   artifacts:
     expire_in: 1 week
@@ -158,18 +151,30 @@ mr-settings:
 
 build-x86_64:
   extends: .build
+  script:
+    - .ci/build-x86_64.sh
 
 build-x86:
   extends: .build
+  script:
+    - .ci/build-x86.sh
 
 build-aarch64:
   extends: .build
+  script:
+    - .ci/build-aarch64.sh
 
 build-armv7:
   extends: .build
+  script:
+    - .ci/build-armv7.sh
 
 build-armhf:
   extends: .build
+  script:
+    - .ci/build-armhf.sh
 
 build-riscv64:
   extends: .build
+  script:
+    - .ci/build-riscv64.sh