From 40233cb6de28c2df84b2877604ce24edb2f70383 Mon Sep 17 00:00:00 2001
From: David Wronek <david@mainlining.org>
Date: Fri, 28 Feb 2025 09:31:02 +0000
Subject: [PATCH 1/4] jobs: send BPO_KERNEL to API (MR 121)
Send the kernel variant to the API to allow adding a new field the JSON index.
Signed-off-by: David Wronek <david@mainlining.org>
---
bpo/jobs/build_image.py | 61 ++++++++++++++++++++++++++++++++------
bpo/jobs/build_package.py | 1 +
bpo/jobs/get_depends.py | 1 +
bpo/jobs/repo_bootstrap.py | 1 +
bpo/jobs/sign_index.py | 1 +
helpers/submit.py | 2 ++
6 files changed, 58 insertions(+), 9 deletions(-)
diff --git a/bpo/jobs/build_image.py b/bpo/jobs/build_image.py
index b5fc06e..1bb317e 100644
--- a/bpo/jobs/build_image.py
+++ b/bpo/jobs/build_image.py
@@ -115,14 +115,23 @@ def run(device, branch, ui):
if [ -e {arg_work_rootfs}/{arg_device}.img ]; then
sudo mv {arg_work_rootfs}/{arg_device}.img \\
"out/$IMG_PREFIX.img"
+ if [ -n {arg_kernel} ]; then
+ echo "out/$IMG_PREFIX.img" >> kernel-{arg_kernel}
+ fi
else
# Boot and root partitions in separate files (pmbootstrap!1871)
# Name the second file -bootpart.img instead of -boot.img to
# avoid confusion with Android boot.img files.
sudo mv {arg_work_rootfs}/{arg_device}-root.img \\
"out/$IMG_PREFIX.img"
+ if [ -n {arg_kernel} ]; then
+ echo "out/$IMG_PREFIX.img" >> kernel-{arg_kernel}
+ fi
sudo mv {arg_work_rootfs}/{arg_device}-boot.img \\
"out/$IMG_PREFIX-bootpart.img"
+ if [ -n {arg_kernel} ]; then
+ echo "out/$IMG_PREFIX-bootpart.img" >> kernel-{arg_kernel}
+ fi
fi
ls -lh out
"""
@@ -139,6 +148,9 @@ def run(device, branch, ui):
for i in {arg_work_boot}/boot.img*; do
if [ -e "$i" ]; then
sudo mv "$i" "out/$IMG_PREFIX-boot.img"
+ if [ -n {arg_kernel} ]; then
+ echo "out/$IMG_PREFIX-boot.img" >> kernel-{arg_kernel}
+ fi
fi
done
@@ -154,6 +166,9 @@ def run(device, branch, ui):
for i in {arg_work_boot}/lk2nd.img; do
if [ -e "$i" ]; then
sudo mv "$i" "out/$IMG_PREFIX-lk2nd.img"
+ if [ -n {arg_kernel} ]; then
+ echo "out/$IMG_PREFIX-lk2nd.img" >> kernel-{arg_kernel}
+ fi
fi
done
@@ -163,9 +178,19 @@ def run(device, branch, ui):
tasks["compress"] = """
sudo chown "$(id -u):$(id -g)" out/*.img
- for i in out/*.img; do
- xz -0 -T0 "$i"
- done
+ kernels="$(find * -name 'kernel-*' -maxdepth 0)"
+ if [ -z "$kernels" ]; then
+ for i in out/*.img; do
+ xz -0 -T0 "$i"
+ done
+ else
+ for kernel in ${kernels}; do
+ for i in $(cat $kernel); do
+ xz -0 -T0 "$i"
+ sed -i "s:$i:$i.xz:" "$kernel"
+ done
+ done
+ fi
ls -lh out
"""
@@ -190,6 +215,10 @@ def run(device, branch, ui):
sudo mv {arg_work}/chroot_*/var/lib/postmarketos-android-recovery-installer/pmos-{arg_device}.zip \
"$OUTPUT_FILE"
+ if [ -n {arg_kernel} ]; then
+ echo "$OUTPUT_FILE" >> kernel-{arg_kernel}
+ fi
+
sudo chown "$(id -u):$(id -g)" "$OUTPUT_FILE"
ls -lh "$OUTPUT_FILE"
"""
@@ -215,12 +244,26 @@ def run(device, branch, ui):
# Upload one file at a time
prev=""
- for i in out/*; do
- export BPO_PAYLOAD_FILES_PREVIOUS="$prev"
- export BPO_PAYLOAD_FILES="$i"
- build.postmarketos.org/helpers/submit.py
- prev="$prev$(basename "$i")#"
- done
+ kernels="$(find * -name 'kernel-*' -maxdepth 0)"
+ if [ -z "$kernels" ]; then
+ for i in out/*; do
+ export BPO_KERNEL=""
+ export BPO_PAYLOAD_FILES_PREVIOUS="$prev"
+ export BPO_PAYLOAD_FILES="$i"
+ build.postmarketos.org/helpers/submit.py
+ prev="$prev$(basename "$i")#"
+ done
+ else
+ for kernel in ${{kernels}}; do
+ for file in "$(cat $kernel)"; do
+ export BPO_KERNEL="${{kernel#'kernel-'}}"
+ export BPO_PAYLOAD_FILES_PREVIOUS="$prev"
+ export BPO_PAYLOAD_FILES="$file"
+ build.postmarketos.org/helpers/submit.py
+ prev="$prev$(basename "$file")#"
+ done
+ done
+ fi
# Finalize upload
export BPO_PAYLOAD_FILES_PREVIOUS="$prev"
diff --git a/bpo/jobs/build_package.py b/bpo/jobs/build_package.py
index 0563525..fb56d77 100644
--- a/bpo/jobs/build_package.py
+++ b/bpo/jobs/build_package.py
@@ -119,6 +119,7 @@ def run(arch, pkgname, branch, splitrepo):
export BPO_BRANCH={shlex.quote(branch)}
export BPO_DEVICE=""
packages="$(pmbootstrap -q config work)/packages"
+ export BPO_KERNEL=""
export BPO_PAYLOAD_FILES="$(find "$packages" -name '*.apk')"
export BPO_PAYLOAD_FILES_PREVIOUS=""
export BPO_PAYLOAD_IS_JSON="0"
diff --git a/bpo/jobs/get_depends.py b/bpo/jobs/get_depends.py
index 583b1c7..45e5feb 100644
--- a/bpo/jobs/get_depends.py
+++ b/bpo/jobs/get_depends.py
@@ -54,6 +54,7 @@ def run(branch):
export BPO_ARCH=""
export BPO_BRANCH=""" + shlex.quote(branch) + """
export BPO_DEVICE=""
+ export BPO_KERNEL=""
export BPO_PAYLOAD_FILES="$(ls -1 depends.*.json)"
export BPO_PAYLOAD_FILES_PREVIOUS=""
export BPO_PAYLOAD_IS_JSON="0"
diff --git a/bpo/jobs/repo_bootstrap.py b/bpo/jobs/repo_bootstrap.py
index f194d64..5ed389c 100644
--- a/bpo/jobs/repo_bootstrap.py
+++ b/bpo/jobs/repo_bootstrap.py
@@ -65,6 +65,7 @@ def run(session, rb, test_pmaports_cfg=None):
export BPO_BRANCH={shlex.quote(rb.branch)}
export BPO_SPLITREPO={shlex.quote(rb.dir_name)}
export BPO_DEVICE=""
+ export BPO_KERNEL=""
packages="$(pmbootstrap -q config work)/packages"
export BPO_PAYLOAD_FILES="$(find "$packages" -name '*.apk')"
export BPO_PAYLOAD_FILES_PREVIOUS=""
diff --git a/bpo/jobs/sign_index.py b/bpo/jobs/sign_index.py
index 36abd5d..faca0b5 100644
--- a/bpo/jobs/sign_index.py
+++ b/bpo/jobs/sign_index.py
@@ -67,6 +67,7 @@ def run(arch, branch, splitrepo):
export BPO_ARCH={shlex.quote(arch)}
export BPO_BRANCH={shlex.quote(branch)}
export BPO_DEVICE=""
+ export BPO_KERNEL=""
export BPO_PAYLOAD_FILES="APKINDEX.tar.gz"
export BPO_PAYLOAD_FILES_PREVIOUS=""
export BPO_PAYLOAD_IS_JSON="0"
diff --git a/helpers/submit.py b/helpers/submit.py
index 8c48faa..f6dc6be 100755
--- a/helpers/submit.py
+++ b/helpers/submit.py
@@ -17,6 +17,7 @@ for key in ["BPO_API_ENDPOINT",
"BPO_DEVICE",
"BPO_JOB_ID",
"BPO_JOB_NAME",
+ "BPO_KERNEL",
"BPO_PAYLOAD_FILES_PREVIOUS", # separated by '#'
"BPO_PAYLOAD_IS_JSON", # set to "1" to enable
"BPO_PKGNAME",
@@ -54,6 +55,7 @@ headers = {"X-BPO-Arch": os.environ["BPO_ARCH"],
"X-BPO-Device": os.environ["BPO_DEVICE"],
"X-BPO-Job-Id": os.environ["BPO_JOB_ID"],
"X-BPO-Job-Name": os.environ["BPO_JOB_NAME"],
+ "X-BPO-Kernel": os.environ["BPO_KERNEL"],
"X-BPO-Payload-Files-Previous":
os.environ["BPO_PAYLOAD_FILES_PREVIOUS"],
"X-BPO-Pkgname": os.environ["BPO_PKGNAME"],
--
GitLab
From f80d70f87180542fd356f257c5df87b7cd504f59 Mon Sep 17 00:00:00 2001
From: David Wronek <david@mainlining.org>
Date: Fri, 28 Feb 2025 09:51:08 +0000
Subject: [PATCH 2/4] bpo.images.ui.write_index_json: Add kernel variant if set
(MR 121)
With this field, desktop installers like the UBports Installer can list
different kernel variants without having to do fuzzy searching.
Signed-off-by: David Wronek <david@mainlining.org>
---
bpo/api/job_callback/build_image.py | 7 ++++---
bpo/ui/images.py | 8 +++++---
2 files changed, 9 insertions(+), 6 deletions(-)
diff --git a/bpo/api/job_callback/build_image.py b/bpo/api/job_callback/build_image.py
index af334bb..66d39a6 100644
--- a/bpo/api/job_callback/build_image.py
+++ b/bpo/api/job_callback/build_image.py
@@ -98,7 +98,7 @@ def upload_new_files(path_temp, files):
return f"got {count} file(s)"
-def upload_finish(session, image, path_temp, dir_name):
+def upload_finish(session, image, path_temp, dir_name, kernel):
# Create target dir
path = bpo.images.path(image.branch, image.device, image.ui, dir_name)
os.makedirs(path, exist_ok=True)
@@ -123,7 +123,7 @@ def upload_finish(session, image, path_temp, dir_name):
# Generate HTML files (for all dirs in the images path, including the path
# of this image and its potentially new parent directories)
- bpo.ui.images.write_index_all()
+ bpo.ui.images.write_index_all(kernel)
# Start next build job
bpo.repo.build()
@@ -137,6 +137,7 @@ def job_callback_build_image():
device = bpo.api.get_header(request, "Device")
dir_name = get_dir_name(request)
job_id = bpo.api.get_header(request, "Job-Id")
+ kernel = bpo.api.get_header(request, "Kernel")
ui = bpo.api.get_header(request, "Ui")
session = bpo.db.session()
@@ -148,4 +149,4 @@ def job_callback_build_image():
if files:
return upload_new_files(path_temp, files)
- return upload_finish(session, image, path_temp, dir_name)
+ return upload_finish(session, image, path_temp, dir_name, kernel)
diff --git a/bpo/ui/images.py b/bpo/ui/images.py
index bbd9502..081bd0d 100644
--- a/bpo/ui/images.py
+++ b/bpo/ui/images.py
@@ -165,7 +165,7 @@ def parse_files_from_disk():
return index
-def write_index_json():
+def write_index_json(kernel):
""" Write an index.json file, which can be used by a desktop installer to
list and download available images. """
logging.info("Writing index.json to images dir")
@@ -244,6 +244,8 @@ def write_index_json():
"size": i["size"],
"url": i["url"]
}
+ if kernel:
+ image["kernel"] = kernel
# these are optional
for s in ["sha256", "sha512"]:
if s in i:
@@ -283,6 +285,6 @@ def write_index_html():
write_index_file_list(path, "05_files.html")
-def write_index_all():
- write_index_json()
+def write_index_all(kernel = ""):
+ write_index_json(kernel)
write_index_html()
--
GitLab
From 0f8515641d76e648f6eebebc38f3edcc1a09562b Mon Sep 17 00:00:00 2001
From: David Wronek <david@mainlining.org>
Date: Wed, 5 Mar 2025 10:27:22 +0000
Subject: [PATCH 3/4] fixup! jobs: send BPO_KERNEL to API (MR 121)
---
bpo/jobs/build_image.py | 43 ++++++++++++++++++++++++++++-------------
1 file changed, 30 insertions(+), 13 deletions(-)
diff --git a/bpo/jobs/build_image.py b/bpo/jobs/build_image.py
index 1bb317e..57febfc 100644
--- a/bpo/jobs/build_image.py
+++ b/bpo/jobs/build_image.py
@@ -185,10 +185,10 @@ def run(device, branch, ui):
done
else
for kernel in ${kernels}; do
- for i in $(cat $kernel); do
+ while read -r i; do
xz -0 -T0 "$i"
sed -i "s:$i:$i.xz:" "$kernel"
- done
+ done < "$kernel"
done
fi
@@ -225,10 +225,25 @@ def run(device, branch, ui):
tasks["checksums"] = """
cd out
- for i in *; do
- sha256sum "$i" | tee "$i.sha256"
- sha512sum "$i" | tee "$i.sha512"
- done
+
+ kernels="$(cd ..; find * -name 'kernel-*' -maxdepth 0)"
+ if [ -z "$kernels" ]; then
+ for i in *; do
+ sha256sum "$i" | tee "$i.sha256"
+ sha512sum "$i" | tee "$i.sha512"
+ done
+ else
+ for kernel in ${kernels}; do
+ while read -r file; do
+ sha256sum "$file" | tee "$file.sha256"
+ sha512sum "$file" | tee "$file.sha512"
+ echo "$file.sha256" >> ../tmp-"$kernel"
+ echo "$file.sha512" >> ../tmp-"$kernel"
+ done < ../"$kernel"
+ cat ../tmp-"$kernel" >> ../"$kernel"
+ rm ../tmp-"$kernel"
+ done
+ fi
"""
tasks["submit"] = f"""
@@ -255,13 +270,15 @@ def run(device, branch, ui):
done
else
for kernel in ${{kernels}}; do
- for file in "$(cat $kernel)"; do
- export BPO_KERNEL="${{kernel#'kernel-'}}"
- export BPO_PAYLOAD_FILES_PREVIOUS="$prev"
- export BPO_PAYLOAD_FILES="$file"
- build.postmarketos.org/helpers/submit.py
- prev="$prev$(basename "$file")#"
- done
+ while read -r file; do
+ if [ -f "$file" ]; then
+ export BPO_KERNEL="${{kernel#'kernel-'}}"
+ export BPO_PAYLOAD_FILES_PREVIOUS="$prev"
+ export BPO_PAYLOAD_FILES="$file"
+ build.postmarketos.org/helpers/submit.py
+ prev="$prev$(basename "$file")#"
+ fi
+ done < "$kernel"
done
fi
--
GitLab
From 3096feaf40185ebd1589a48e1d4a34610cd4ca53 Mon Sep 17 00:00:00 2001
From: David Wronek <david@mainlining.org>
Date: Wed, 5 Mar 2025 11:11:41 +0000
Subject: [PATCH 4/4] fixup! jobs: send BPO_KERNEL to API (MR 121)
---
bpo/jobs/build_image.py | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/bpo/jobs/build_image.py b/bpo/jobs/build_image.py
index 57febfc..45d5f85 100644
--- a/bpo/jobs/build_image.py
+++ b/bpo/jobs/build_image.py
@@ -235,10 +235,10 @@ def run(device, branch, ui):
else
for kernel in ${kernels}; do
while read -r file; do
- sha256sum "$file" | tee "$file.sha256"
- sha512sum "$file" | tee "$file.sha512"
- echo "$file.sha256" >> ../tmp-"$kernel"
- echo "$file.sha512" >> ../tmp-"$kernel"
+ sha256sum $(basename "$file") | tee "$(basename $file).sha256"
+ sha512sum $(basename "$file") | tee "$(basename $file).sha512"
+ echo "out/$file.sha256" >> ../tmp-"$kernel"
+ echo "out/$file.sha512" >> ../tmp-"$kernel"
done < ../"$kernel"
cat ../tmp-"$kernel" >> ../"$kernel"
rm ../tmp-"$kernel"
--
GitLab