mrhlpr: make GitLabOrigin a real class, and have the download_json method

This resolves a cyclic dependency in pipeline, and is also a more
programatic pattern

mrhlpr/gitlab.py

@@ -7,10 +7,8 @@ import urllib.parse
 import urllib.request
 import os
 import shutil
-import json
 import logging
 import re
-from typing import Any
 
 from mrhlpr.origin import GitLabOrigin
 from . import git
@@ -18,53 +16,6 @@ from . import runner
 from .pipeline import PipelineMetadata
 
 
-def download_json(origin: GitLabOrigin, pathname: str, no_cache: bool = False) -> Any:
-    """Download and parse JSON from an API, with a cache.
-
-    :param origin: gitlab origin information, see gitlab.parse_git_origin()
-    :param pathname: gitlab URL pathname (without the usual prefix)
-    :param no_cache: download again, even if already cached
-    :returns: parsed JSON"""
-    url = origin.api + pathname
-
-    home_path = os.getenv("HOME")
-    if home_path is None:
-        msg = "$HOME was not set"
-        raise RuntimeError(msg)
-    # Prepare cache
-    cache_dir = home_path + "/.cache/mrhlpr/http"
-    cache_key = hashlib.sha256(url.encode("utf-8")).hexdigest()
-    cache_file = cache_dir + "/" + cache_key
-    os.makedirs(cache_dir, exist_ok=True)
-
-    # Check the cache
-    if os.path.exists(cache_file) and not no_cache:
-        logging.debug("Download " + url + " (cached)")
-    else:
-        print("Download " + url)
-        # Save to temp file
-        temp_file = cache_file + ".tmp"
-        with urllib.request.urlopen(url) as response:
-            with open(temp_file, "wb") as handle:
-                shutil.copyfileobj(response, handle)
-
-        # Pretty print JSON (easier debugging)
-        with open(temp_file, "r") as handle:
-            parsed = json.load(handle)
-        with open(temp_file, "w") as handle:
-            handle.write(json.dumps(parsed, indent=4))
-
-        # Replace cache file
-        if os.path.exists(cache_file):
-            os.remove(cache_file)
-        os.rename(temp_file, cache_file)
-
-    # Parse JSON from the cache file
-    logging.debug(" -> " + cache_file)
-    with open(cache_file, "r") as handle:
-        return json.load(handle)
-
-
 def download_artifacts_zip(api: str, pipeline: PipelineMetadata, no_cache: bool = False) -> str:
     """Download the job artifacts zip file, with a cache.
 

mrhlpr/mr.py

@@ -75,14 +75,14 @@ def get_status(
     if not origin:
         origin = gitlab.parse_git_origin()
     url_mr = "/projects/{}/merge_requests/{}".format(origin.api_project_id, mr_id)
-    api = gitlab.download_json(origin, url_mr, no_cache)
+    api = origin.download_json(url_mr, no_cache)
 
     # Query source project/repository
     # https://docs.gitlab.com/ee/api/projects.html
     # Always cache this, since we don't expect the "path_with_namespace" to
     # ever change (and even if, we can keep using the old one locally).
     url_project = "/projects/" + str(api["source_project_id"])
-    api_source = gitlab.download_json(origin, url_project)
+    api_source = origin.download_json(url_project)
 
     # Allow maintainer to push
     allow_push = False
@@ -168,7 +168,7 @@ def get_artifacts_repo_urls(
         origin.api_project_id,
         mr_id,
     )
-    api = gitlab.download_json(origin, url_mr, no_cache=True)
+    api = origin.download_json(url_mr, no_cache=True)
 
     host_arch_job_web_url = pipeline_metadata.host_arch_job["web_url"]
     if alpine_mr:

mrhlpr/origin.py

@@ -3,19 +3,78 @@
 """Origin information about gitlab instances relevant to mrtest, in the format
 needed for mrhlpr.gitlab.parse_git_origin()."""
 
-from dataclasses import dataclass
-from typing import Optional
+import json
+import logging
+import os
+import shutil
+import urllib
+import hashlib
+from typing import Any, Optional
 
 
-@dataclass
 class GitLabOrigin:
-    api: str
-    api_project_id: str
-    full: str
-    project: str
-    project_id: str
-    host: str
-    username: Optional[str]
+    def __init__(
+        self,
+        api: str,
+        api_project_id: str,
+        full: str,
+        project: str,
+        project_id: str,
+        host: str,
+        username: Optional[str],
+    ):
+        self.api: str = api
+        self.api_project_id: str = api_project_id
+        self.full: str = full
+        self.project: str = project
+        self.project_id: str = project_id
+        self.host: str = host
+        self.username: Optional[str] = username
+
+    def download_json(self, pathname: str, no_cache: bool = False) -> Any:
+        """Download and parse JSON from an API, with a cache.
+
+        :param pathname: gitlab URL pathname (without the usual prefix)
+        :param no_cache: download again, even if already cached
+        :returns: parsed JSON"""
+        url = self.api + pathname
+
+        home_path = os.getenv("HOME")
+        if home_path is None:
+            msg = "$HOME was not set"
+            raise RuntimeError(msg)
+        # Prepare cache
+        cache_dir = home_path + "/.cache/mrhlpr/http"
+        cache_key = hashlib.sha256(url.encode("utf-8")).hexdigest()
+        cache_file = cache_dir + "/" + cache_key
+        os.makedirs(cache_dir, exist_ok=True)
+
+        # Check the cache
+        if os.path.exists(cache_file) and not no_cache:
+            logging.debug("Download " + url + " (cached)")
+        else:
+            print("Download " + url)
+            # Save to temp file
+            temp_file = cache_file + ".tmp"
+            with urllib.request.urlopen(url) as response:
+                with open(temp_file, "wb") as handle:
+                    shutil.copyfileobj(response, handle)
+
+            # Pretty print JSON (easier debugging)
+            with open(temp_file, "r") as handle:
+                parsed = json.load(handle)
+            with open(temp_file, "w") as handle:
+                handle.write(json.dumps(parsed, indent=4))
+
+            # Replace cache file
+            if os.path.exists(cache_file):
+                os.remove(cache_file)
+            os.rename(temp_file, cache_file)
+
+        # Parse JSON from the cache file
+        logging.debug(" -> " + cache_file)
+        with open(cache_file, "r") as handle:
+            return json.load(handle)
 
 
 pmaports = GitLabOrigin(

mrhlpr/pipeline.py

@@ -6,14 +6,12 @@ import logging
 import platform
 
 from mrhlpr.origin import GitLabOrigin
-from . import gitlab
 
 
 class PipelineMetadata:
     def __init__(self, mr_id: int, no_cache: bool, origin: GitLabOrigin):
         # Get the latest pipeline (without cache so we don't miss newer ones)
-        api = gitlab.download_json(
-            origin,
+        api = origin.download_json(
             f"/projects/{origin.api_project_id}/merge_requests/{mr_id}",
             no_cache=True,
         )
@@ -32,8 +30,7 @@ class PipelineMetadata:
             )
 
         if origin.project == "alpine":
-            trigger_jobs = gitlab.download_json(
-                origin,
+            trigger_jobs = origin.download_json(
                 f"/projects/{pipeline_project_id}/pipelines/{pipeline_id}/bridges",
                 True,
             )
@@ -47,8 +44,7 @@ class PipelineMetadata:
 
         # Query the jobs of the pipeline (without cache, the result may change e.g.
         # if this ran earlier while the pipeline was not successful and now it is)
-        api_pipeline_jobs = gitlab.download_json(
-            origin,
+        api_pipeline_jobs = origin.download_json(
             f"/projects/{pipeline_project_id}/pipelines/{pipeline_id}/jobs",
             True,
         )