polishing

.gitlab-ci.yml

@@ -135,7 +135,7 @@ release_binary_windows:
     - .\pyinstaller\build-win-ci.bat $CI_COMMIT_TAG
     - python ./utils/github.py upload ./pyinstaller/release/specterd-$CI_COMMIT_TAG-win64.zip
     - cd ./pyinstaller/release
-    - python ..\..\utils\release-helper.py sha256sums specterd-$CI_COMMIT_TAG-win64.zip > SHA256SUMS-windows
+    - python ..\..\utils\release_helper.py sha256sums specterd-$CI_COMMIT_TAG-win64.zip > SHA256SUMS-windows
     - type SHA256SUMS-windows
     - echo $GPG_PASSPHRASE | c:\Program` Files` `(x86`)\GnuPg\bin\gpg --detach-sign --armor --no-tty --batch --yes --passphrase-fd 0  --pinentry-mode loopback SHA256SUMS-windows
   artifacts:
@@ -222,14 +222,14 @@ release_signatures:
     - pip3 install -e ".[test]"
     - ./utils/artifact_signer.sh init # prepare .gnupg
   script:
-    - python3 -m utils.release-helper download          # downloads the job-artifacts from gitlab
-    - python3 -m utils.release-helper downloadgithub    # downloads additional artifacts from github (if not there and is they have SHA256SUMS-something)
-    - python3 -m utils.release-helper checksigs         # checks the signatures of all SHA256SUMM*.asc files
-    - python3 -m utils.release-helper checkhashes       # checks all SHA256SUM* files (might modify files on the fly due to windows line endings)
-    - python3 -m utils.release-helper create            # creates a SHA256SUM
+    - python3 -m utils.release_helper download          # downloads the job-artifacts from gitlab
+    - python3 -m utils.release_helper downloadgithub    # downloads additional artifacts from github (if not there and is they have SHA256SUMS-something)
+    - python3 -m utils.release_helper checksigs         # checks the signatures of all SHA256SUMM*.asc files
+    - python3 -m utils.release_helper checkhashes       # checks all SHA256SUM* files (might modify files on the fly due to windows line endings)
+    - python3 -m utils.release_helper create            # creates a SHA256SUM
     - ./utils/artifact_signer.sh sign --artifact ./signing_dir/SHA256SUMS # Signs the SHA256SUM
-    - python3 -m utils.release-helper upload_shasums    # uploads SHA256SUMS to github
-    - python3 -m utils.release-helper upload_shasumssig # uploads SHA256SUMS.asc to github
+    - python3 -m utils.release_helper upload_shasums    # uploads SHA256SUMS to github
+    - python3 -m utils.release_helper upload_shasumssig # uploads SHA256SUMS.asc to github
 
 release_docker:
   stage: post_releasing

pyinstaller/build-win-ci.bat

@@ -19,7 +19,7 @@ pip3 install build==0.10.0
 python -m build
 
 echo "    --> Installing pypi package"
-python .\utils\release-helper.py install_wheel %1%
+python .\utils\release_helper.py install_wheel %1%
 
 cd pyinstaller
 

utils/build-osx.sh

@@ -66,7 +66,7 @@ function make_hash_if_necessary {
     # Download and check the intel one
     # this needs some env-vars to be set
     rm -rf signing_dir/*
-    PYTHONPATH=../.. python3 -m utils.release-helper downloadgithub
+    PYTHONPATH=../.. python3 -m utils.release_helper downloadgithub
     ret_code=$?
     if [ $ret_code -ne 0 ]; then
       echo "Downloading and verifying x64 specterd failed with exit code $ret_code"

utils/release-helper.py → utils/release_helper.py

@@ -18,6 +18,7 @@
 import subprocess
 import hashlib
 import logging
+import gitlab
 
 logger = logging.getLogger(__name__)
 
@@ -212,14 +213,16 @@ class ReleaseHelper:
 
     def __init__(self):
         self.target_dir = "signing_dir"
+        Path(self.target_dir).mkdir(parents=True, exist_ok=True)
 
-    def init_gitlab(self):
+    @property
+    def gl(self):
         # https://python-gitlab.readthedocs.io/en/stable/api-usage.html
         import gitlab
 
         if os.environ.get("GITLAB_PRIVATE_TOKEN"):
             logger.info("Using GITLAB_PRIVATE_TOKEN")
-            self.gl = gitlab.Gitlab(
+            gl = gitlab.Gitlab(
                 "http://gitlab.com",
                 private_token=os.environ.get("GITLAB_PRIVATE_TOKEN"),
             )
@@ -232,88 +235,137 @@ def init_gitlab(self):
             raise Exception(
                 "Can't authenticate against Gitlab ( export GITLAB_PRIVATE_TOKEN )"
             )
+        return gl
 
-        if os.environ.get("CI_PROJECT_ROOT_NAMESPACE"):
-            project_root_namespace = os.environ.get("CI_PROJECT_ROOT_NAMESPACE")
-            logger.info(
-                f"Using project_root_namespace: {project_root_namespace} ( export CI_PROJECT_ROOT_NAMESPACE={project_root_namespace} )"
-            )
-        else:
-            raise Exception(
-                "no CI_PROJECT_ROOT_NAMESPACE given ( export CI_PROJECT_ROOT_NAMESPACE=k9ert )"
-            )
-
-        if os.environ.get("CI_PROJECT_ID"):
-            self.project_id = os.environ.get("CI_PROJECT_ID")
-            self.github_project = f"{project_root_namespace}/specter-desktop"
-        else:
-            logger.error("No Project given. choose one:")
-            for project in self.gl.projects.list(search="specter-desktop"):
-                logger.info(
-                    f"     export CI_PROJECT_ID={project.id}  # {project.name_with_namespace}"
-                )
-            exit(1)
-
-        logger.info(f"Using project_id: {self.project_id} ")
-        logger.info(f"Using github_project: {self.github_project}")
+    @property
+    def gitlab_project(self):
+        if hasattr(self, "_gitlab_project"):
+            return self._gitlab_project
         try:
             from gitlab.v4.objects import Project
 
-            self.project: Project = self.gl.projects.get(self.project_id)
+            self._gitlab_project: Project = self.gl.projects.get(self.ci_project_id)
         except gitlab.exceptions.GitlabAuthenticationError as e:
             logger.fatal(e)
             logger.error("Your token might be expired or wrong. Get a new one here:")
             logger.error("  https://gitlab.com/-/profile/personal_access_tokens")
             exit(2)
 
-        if self.project.attributes["namespace"]["path"] != project_root_namespace:
+        if (
+            self._gitlab_project.attributes["namespace"]["path"]
+            != self.ci_project_root_namespace
+        ):
             logger.fatal(
-                f"project_root_namespace ({ project_root_namespace }) does not match namespace of Project ({self.project.attributes['namespace']['path']}) "
+                f"project_root_namespace ({ self.ci_project_root_namespace }) does not match namespace of Project ({self._gitlab_project.attributes['namespace']['path']}) "
             )
             logger.error("You might want to: unset CI_PROJECT_ID")
             exit(2)
+        return self._gitlab_project
 
+    @property
+    def ci_project_id(self):
+        if hasattr(self, "_ci_project_id"):
+            return self._ci_project_id
+        if os.environ.get("CI_PROJECT_ID"):
+            self._ci_project_id = os.environ.get("CI_PROJECT_ID")
+            logger.info(f"Using ci_project_id: {self.ci_project_id} ")
+        else:
+            logger.error("No Project given. choose one:")
+            for project in self.gl.projects.list(search="specter-desktop"):
+                logger.info(
+                    f"     export CI_PROJECT_ID={project.id}  # {project.name_with_namespace}"
+                )
+                if project.name_with_namespace.startswith("cryptoadvance"):
+                    self._ci_project_id = project.id
+            logger.warn("{self._ci_project_id} has been chosen as self._ci_project_id")
+        return self._ci_project_id
+
+    @property
+    def github_project(self):
+        if hasattr(self, "_github_project"):
+            return self._github_project
+        self._github_project = f"{self.ci_project_root_namespace}/specter-desktop"
+        logger.info(f"Using github_project: {self._github_project}")
+        return self._github_project
+
+    @property
+    def ci_commit_tag(self):
+        if hasattr(self, "_ci_commit_tag"):
+            return self._ci_commit_tag
         if os.environ.get("CI_COMMIT_TAG"):
-            self.tag = os.environ.get("CI_COMMIT_TAG")
+            self._ci_commit_tag = os.environ.get("CI_COMMIT_TAG")
         else:
             raise Exception("no tag given ( export CI_COMMIT_TAG=v0.0.0.0-pre13 )")
-        logger.info(f"Using tag: {self.tag}")
+        logger.info(f"Using tag: {self._ci_commit_tag}")
+        return self._ci_commit_tag
+
+    @property
+    def ci_project_root_namespace(self):
+        if hasattr(self, "_ci_project_root_namespace"):
+            return self._ci_project_root_namespace
+        if os.environ.get("CI_PROJECT_ROOT_NAMESPACE"):
+            self._ci_project_root_namespace = os.environ.get(
+                "CI_PROJECT_ROOT_NAMESPACE"
+            )
+            logger.info(
+                f"Using project_root_namespace: {project_root_namespace} ( export CI_PROJECT_ROOT_NAMESPACE={project_root_namespace} )"
+            )
+        else:
+            self._ci_project_root_namespace = "cryptoadvance"
+            logger.warn(
+                f"Using project_root_namespace: {self._ci_project_root_namespace} ( export CI_PROJECT_ROOT_NAMESPACE={self._ci_project_root_namespace} )"
+            )
+        return self._ci_project_root_namespace
+
+    @property
+    def ci_pipeline_id(self):
+        if hasattr(self, "_ci_pipeline_id"):
+            return self._ci_pipeline_id
 
         if os.environ.get("CI_PIPELINE_ID"):
-            self.pipeline_id = os.environ.get("CI_PIPELINE_ID")
-            self.pipeline = self.project.pipelines.get(self.pipeline_id)
+            self._ci_pipeline_id = os.environ.get("CI_PIPELINE_ID")
         else:
             logger.info(
                 "no CI_PIPELINE_ID given, trying to find an appropriate one ..."
             )
-            pipelines = self.project.pipelines.list()
+            pipelines = self.gitlab_project.pipelines.list()
             for pipeline in pipelines:
-                if pipeline.ref == self.tag:
-                    self.pipeline = pipeline
+                if pipeline.ref == self.ci_commit_tag:
+                    self._ci_pipeline_id = pipeline.id
+                    self._ci_pipeline = pipeline
                     logger.info(f"Found matching pipeline: {pipeline}")
-            if not hasattr(self, "pipeline"):
-                logger.error(f"Could not find tag {self.tag} in the pipeline-refs:")
-                for pipeline in self.project.pipelines.list():
+            if not hasattr(self, "_ci_pipeline"):
+                logger.error(
+                    f"Could not find tag {self.ci_commit_tag} in the pipeline-refs:"
+                )
+                for pipeline in self.gitlab_project.pipelines.list():
                     logger.error(pipeline.ref)
                 raise Exception(
                     "no CI_PIPELINE_ID given ( export CI_PIPELINE_ID= ) or maybe you're on the wrong project ( export CI_PROJECT_ROOT_NAMESPACE= )"
                 )
 
-        logger.info(f"Using pipeline_id: {self.pipeline.id}")
-        Path(self.target_dir).mkdir(parents=True, exist_ok=True)
+        logger.info(f"Using pipeline_id: {self.ci_pipeline.id}")
+        return self._ci_pipeline_id
+
+    @property
+    def ci_pipeline(self):
+        if hasattr(self, "_ci_pipeline"):
+            return self._ci_pipeline
+        self._ci_pipeline = self.gitlab_project.pipelines.get(self.ci_pipeline_id)
+        return self._ci_pipeline
 
     def download_and_unpack_all_artifacts(self):
         if os.path.isdir(self.target_dir):
             logger.info(f"First purging {self.target_dir}")
             shutil.rmtree(self.target_dir)
-        for job in self.pipeline.jobs.list():
+        for job in self.ci_pipeline.jobs.list():
             if job.name in [
                 "release_electron_linux_windows",
                 "release_binary_windows",
                 "release_pip",
             ]:
                 zipfn = f"/tmp/_artifacts_{job.name}.zip"
-                job_obj = self.project.jobs.get(job.id, lazy=True)
+                job_obj = self.gitlab_project.jobs.get(job.id, lazy=True)
 
                 if not os.path.isfile(zipfn):
                     logger.info(f"Downloading artifacts for {job.name}")
@@ -334,7 +386,7 @@ def download_and_unpack_all_artifacts(self):
     def download_and_unpack_new_artifacts_from_github(self):
 
         gc = github.GithubConnection(self.github_project)
-        release = gc.fetch_existing_release(self.tag)
+        release = gc.fetch_existing_release(self.ci_commit_tag)
         assets = gc.list_assets(release)
         for asset in assets:
             if not asset.name.startswith("SHA256"):
@@ -346,10 +398,11 @@ def download_and_unpack_new_artifacts_from_github(self):
             logger.info("iterating file " + asset.name)
             shasumfile = Sha256sumFile(asset.name)
             if not shasumfile.is_in_target_dir():
-                shasumfile.download_from_tag(self.tag, gc)
-                shasumfile.download_hashed_files(self.tag, gc)
+                shasumfile.download_from_tag(self.ci_commit_tag, gc)
+                shasumfile.download_hashed_files(self.ci_commit_tag, gc)
                 shasumfile.check_hashes()
                 shasumfile.check_sig()
+        logger.info("All files have valid signatures")
 
     def create_sha256sum_file(self):
         with open(f"{self.target_dir}/SHA256SUMS", "w") as shafile:
@@ -383,17 +436,20 @@ def check_all_hashes(self):
                 )
                 if returncode != 0:
                     raise Exception(f"Could not validate hashes for file {file}")
+        logger.info("All files SHA256SUM* (not .asc) has valid hashes")
 
     def check_all_sigs(self):
         for file in os.listdir(self.target_dir):
             if file.endswith(".asc"):
+                logger.info(f"Checking signature for {file}")
                 returncode = subprocess.call(
                     ["gpg", "--verify", file], cwd=self.target_dir
                 )
                 if returncode != 0:
                     raise Exception(
                         f"Could not validate signature of file {file}: {subprocess.run(['gpg', '--verify', file], cwd=self.target_dir)}"
                     )
+        logger.info("All files *.asc has valid signatures")
 
     def calculate_publish_params(self):
         if not "CI_PROJECT_ROOT_NAMESPACE" in os.environ:
@@ -418,14 +474,16 @@ def upload_sha256sum_file(self):
         artifact = os.path.join("signing_dir", "SHA256SUMS")
         self.calculate_publish_params()
 
-        if github.artifact_exists(self.github_project, self.tag, Path(artifact).name):
+        if github.artifact_exists(
+            self.github_project, self.ci_commit_tag, Path(artifact).name
+        ):
             logger.info(f"Github artifact {artifact} existing. Skipping upload.")
             exit(0)
         else:
             logger.info(f"Github artifact {artifact} does not exist. Let's upload!")
         github.publish_release_from_tag(
             self.github_project,
-            self.tag,
+            self.ci_commit_tag,
             [artifact],
             "gitlab_upload_release_binaries",
             self.password,
@@ -435,14 +493,16 @@ def upload_sha256sumsig_file(self):
         artifact = os.path.join("signing_dir", "SHA256SUMS.asc")
         self.calculate_publish_params()
 
-        if github.artifact_exists(self.github_project, self.tag, Path(artifact).name):
+        if github.artifact_exists(
+            self.github_project, self.ci_commit_tag, Path(artifact).name
+        ):
             logger.info(f"Github artifact {artifact} existing. Skipping upload.")
             exit(0)
         else:
             logger.info(f"Github artifact {artifact} does not exist. Let's upload!")
         github.publish_release_from_tag(
             self.github_project,
-            self.tag,
+            self.ci_commit_tag,
             [artifact],
             "gitlab_upload_release_binaries",
             self.password,
@@ -493,13 +553,12 @@ def sha256sum(filenames):
         exit(0)
 
     rh = ReleaseHelper()
-    rh.init_gitlab()
     try:
         from utils import github
     except Exception as e:
         logger.fatal(e)
         logger.error("You might have called this script wrong. Execute it like:")
-        logger.error("python3 -m utils.release-helper ...")
+        logger.error("python3 -m utils.release_helper ...")
 
     if "download" in sys.argv:
         rh.download_and_unpack_all_artifacts()