Chore: Release process improvements (#1434)

* Bugfix: unknown kex fixes for SpecterMigrator * create sha256sums * fix * kick * kick * kick * kick * kick * a script for signing * kick * kick * lots of stuff * improve caching * improve caching * reasonable wget progress output * kick * kick * kick * hash and signing for windows * sha256impl in py for win incl refactoring * kick * kick * kick * kick * kick * kick * kick * kick * small test-cypress change
cryptoadvance · Oct 19, 2021 · f814bbf · f814bbf
1 parent 6055337
commit f814bbf
Show file tree

Hide file tree

Showing 13 changed files with 567 additions and 50 deletions.
diff --git a/.gitignore b/.gitignore
@@ -32,3 +32,4 @@ tests/bitcoin*
 token.sh
 src/cryptoadvance/specter/translations/**/messages.mo
 tests/elements
+signing_dir
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -8,15 +8,19 @@ variables:
 
 cache:
   # enable per-job and per-branch caching
-  key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
-  paths:
-    - .cache/pip
-    - .env/
-    - ./tests/elements
+  - key: 
+      files: 
+        - ./requirements.txt
+        - ./test_requirements.txt
+      prefix: "$CI_JOB_NAME"
+    paths:
+      - .cache/pip
+      - .env
 
 stages:
   - testing
   - releasing
+  - post_releasing
 
 before_script:
   - docker info || echo "no docker-command found" # Print out docker version for debugging
@@ -86,7 +90,7 @@ release_pip:
     - pip3 install -r test_requirements.txt
     - python3 setup.py install
     # verifying the version number follows vx.y.z (e.g. "v1.2.3")
-    - if ! [[ $CI_COMMIT_TAG =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(\+([0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*))?$ ]]; then exit 1; fi
+    - if ! [[ $CI_COMMIT_TAG =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-pre(0|[1-9][0-9]*))? ]]; then exit 1; fi
     # set version number in setup.py
     - echo Releasing $CI_COMMIT_TAG
     - sed -i "s/version=\".*/version=\"$CI_COMMIT_TAG\",/" setup.py
@@ -97,8 +101,13 @@ release_pip:
     # Either testing it or doing the real thing depending on which gitlab-project we're running:
     - if ! [[ ${CI_PROJECT_ROOT_NAMESPACE} = "cryptoadvance" ]]; then  python3 -m twine upload --verbose --user __token__ dist/* --repository-url https://test.pypi.org/legacy/ ; fi
     - if [[ ${CI_PROJECT_ROOT_NAMESPACE} = "cryptoadvance" ]]; then  python3 -m twine upload --verbose --user __token__ dist/* ; fi
-    - sha256sum dist/cryptoadvance.specter-*.tar.gz > ./dist/SHA256SUMS-pip.txt
-    - python ./utils/github.py upload ./dist/SHA256SUMS-pip.txt 
+    - cd dist
+    - sha256sum cryptoadvance.specter-*.tar.gz > SHA256SUMS-pip
+    - ../utils/sign_artifact.sh --artifact ./SHA256SUMS-pip
+    - cd ..
+    - cat ./dist/SHA256SUMS-pip
+    #- python ./utils/github.py upload ./dist/SHA256SUMS-pip
+    #- python ./utils/github.py upload ./dist/SHA256SUMS-pip.asc
     - python ./utils/github.py upload ./dist/cryptoadvance.specter-*.tar.gz
   artifacts:
     when: always
@@ -117,6 +126,7 @@ release_binary_windows:
     - pip3 install virtualenv
     - virtualenv --python=python3 .env
     - .\.env\Scripts\activate
+    - pip3 install -r test_requirements.txt
   script:
     # This script won't execute if the script before that fails
     # No need to check the version-scheme again
@@ -125,11 +135,23 @@ release_binary_windows:
     - cd pyinstaller 
     - .\build-win-ci.bat $CI_COMMIT_TAG
     - python ../utils/github.py upload ./release/specterd-$CI_COMMIT_TAG-win64.zip
+    - cd release
+    - python ..\..\utils\release-helper.py sha256sums specterd-$CI_COMMIT_TAG-win64.zip > SHA256SUMS-windows
+    - type SHA256SUMS-windows
+    - echo $GPG_PASSPHRASE | c:\Program` Files` `(x86`)\GnuPg\bin\gpg --detach-sign --armor --no-tty --batch --yes --passphrase-fd 0  --pinentry-mode loopback SHA256SUMS-windows
+    - dir
   artifacts:
     when: always
     paths:
       - pyinstaller/release/*
     expire_in: 1 day
+  cache:
+    key:
+        files:
+          - ./pyinstaller/electron/package-lock.json
+        prefix: $CI_JOB_NAME
+    paths:
+      - ./pyinstaller/electron/node_modules
 
 release_electron_linux_windows:
   image: registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest
@@ -151,21 +173,70 @@ release_electron_linux_windows:
     - ./build-ci.sh $CI_COMMIT_TAG "make-hash"
     - ls -l release-linux
     - ls -l release-win
-    - sha256sum  ./release-linux/specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip ./release-linux/specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz ./release-win/Specter-Setup-${CI_COMMIT_TAG}.exe > ./SHA256SUMS.txt
-    - cat ./SHA256SUMS.txt
-    - if [[ -f /credentials/private.key ]]; then gpg --import --no-tty --batch --yes /credentials/private.key; echo $GPG_PASSPHRASE | gpg --detach-sign  --no-tty --batch --yes --passphrase-fd 0  --pinentry-mode loopback ./SHA256SUMS.txt || true ; fi
-    - python ../utils/github.py upload ./release-win/Specter-Setup-${CI_COMMIT_TAG}.exe
-    - python ../utils/github.py upload ./release-linux/specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip 
-    - python ../utils/github.py upload ./release-linux/specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz 
-    - python ../utils/github.py upload ./SHA256SUMS.txt
-    - if [[ -f ./SHA256SUMS.txt.sig ]]; then python3 ../utils/github.py upload ./SHA256SUMS.txt.sig ; fi
+    - cd release-linux
+    - sha256sum  specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz > ./SHA256SUMS-linux
+    - cat ./SHA256SUMS-linux
+    - cd ..
+    - cd release-win
+    - sha256sum  Specter-Setup-${CI_COMMIT_TAG}.exe > ./SHA256SUMS-win
+    - cat ./SHA256SUMS-win
+    - cd ..
+    - ../utils/sign_artifact.sh --artifact ./release-win/SHA256SUMS-win
+    - ../utils/sign_artifact.sh --artifact ./release-linux/SHA256SUMS-linux
+    - python3 ../utils/github.py upload ./release-win/Specter-Setup-${CI_COMMIT_TAG}.exe
+    - python3 ../utils/github.py upload ./release-linux/specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip 
+    - python3 ../utils/github.py upload ./release-linux/specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz 
+    #- python3 ../utils/github.py upload ./release-linux/SHA256SUMS-linux
+    #- python3 ../utils/github.py upload ./release-linux/SHA256SUMS-linux.asc
+    #- python3 ../utils/github.py upload ./release-win/SHA256SUMS-win
+    #- python3 ../utils/github.py upload ./release-win/SHA256SUMS-win.asc
+  cache:
+    key:
+        files:
+          - ./pyinstaller/electron/package-lock.json
+        prefix: $CI_JOB_NAME
+    paths:
+      - ./pyinstaller/electron/node_modules
+
   artifacts:
     when: always
     paths:
-      - pyinstaller/release-linux/*
-      - pyinstaller/release-win/*
-      - pyinstaller/SHA256SUMS.txt
+      - pyinstaller/release-win/Specter-Setup-${CI_COMMIT_TAG}.exe
+      - pyinstaller/release-linux/specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip
+      - pyinstaller/release-linux/specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz
+      - pyinstaller/release-linux/SHA256SUMS-linux
+      - pyinstaller/release-linux/SHA256SUMS-linux.asc
+      - pyinstaller/release-win/SHA256SUMS-win
+      - pyinstaller/release-win/SHA256SUMS-win.asc
     expire_in: 1 day
 
+release_signatures:
+  stage: post_releasing
+  only: 
+    - tags
+  before_script:
+    - python -V  # Print out python version for debugging
+    - pip3 install --upgrade virtualenv
+    - virtualenv --python=python3 .env
+    - source .env/bin/activate
+    - pip3 install -r test_requirements.txt
+    - ./utils/create-gitlab-cli-cfg.sh
+    - gpg --import --no-tty --batch --yes /credentials/private.key
+  script:
+    - python3 -m utils.release-helper download          # downloads the job-artifacts from gitlab
+    - python3 -m utils.release-helper downloadgithub    # downloads additional artifacts from github (if not there and is they have SHA256SUMS-something)
+    - python3 -m utils.release-helper checkhashes       # checks all SHA256SUM* files
+    - python3 -m utils.release-helper checksig          # checks the signatures of all SHA256SUMM*.asc files
+    - python3 -m utils.release-helper create            # creates a SHA256SUM.txt
+    - python3 -m utils.release-helper upload            # uploads it to github
+    - ./utils/sign_artifact.sh --artifact ./signing_dir/SHA256SUMS
+    - python3 ./utils/github.py upload ./signing_dir/SHA256SUMS.asc
 
-
+release_docker:
+  stage: post_releasing
+  only: 
+    - tags
+  before_script:
+    - echo "Triggering Docker Release"
+  script:
+    - ./utils/trigger_docker_build.sh
diff --git a/README.md b/README.md
@@ -92,8 +92,8 @@ After that, Specter will be available at [http://127.0.0.1:25441/](http://127.0.
 The above installation-method is quite easy but you have to trust pypi. If you want to verify the software completely yourself while still installing via pip3, you can do something like this:
 ```
 wget https://github.com/cryptoadvance/specter-desktop/releases/download/v1.4.6/cryptoadvance.specter-1.4.6.tar.gz
-wget https://github.com/cryptoadvance/specter-desktop/releases/download/v1.4.6/SHA256SUMS-pip.txt
-sha256sum --check SHA256SUMS-pip.txt 
+wget https://github.com/cryptoadvance/specter-desktop/releases/download/v1.4.6/SHA256SUMS-pip
+sha256sum --check SHA256SUMS-pip 
 # Do your usual GPG-check here
 # Now, let's extract the requirements-file and install all requirements with require-hashes
 tar -xvzf cryptoadvance.specter-1.4.6.tar.gz cryptoadvance.specter-1.4.6/requirements.txt

diff --git a/pyinstaller/build-ci.sh b/pyinstaller/build-ci.sh
@@ -54,7 +54,7 @@ rm -rf dist
 mkdir dist
 cd dist
 echo "    --> Downloading the windows-version of specterd for version $1"
-wget https://github.com/${CI_PROJECT_ROOT_NAMESPACE}/specter-desktop/releases/download/$1/specterd-$1-win64.zip -O ./specterd.zip
+wget --progress=dot -e dotbytes=10M https://github.com/${CI_PROJECT_ROOT_NAMESPACE}/specter-desktop/releases/download/$1/specterd-$1-win64.zip -O ./specterd.zip
 unzip specterd.zip
 cd ../electron
 rm -rf dist/

diff --git a/src/cryptoadvance/specter/util/specter_migrator.py b/src/cryptoadvance/specter/util/specter_migrator.py
@@ -249,7 +249,7 @@ def set_Execution_log_error_msg(self, id, msg):
 
     def _find_exec_log(self, id):
         for migration in self.migration_executions:
-            if migration["migration_id"] == id:
+            if migration.get("migration_id") == id:
                 return migration
         raise SpecterError(f"Can't find migration_execution with id {id}")
 
@@ -259,7 +259,7 @@ def migration_executions(self):
 
     def has_migration_executed(self, migration_id):
         executed_list = [
-            migration_execution["migration_id"]
+            migration_execution.get("migration_id")
             for migration_execution in self.migration_executions
         ]
         logger.debug(f"Executed migration_classes ids: {executed_list}")

diff --git a/test_requirements.txt b/test_requirements.txt
@@ -7,4 +7,5 @@ pytest==6.1.2
 PySocks==1.7.1
 pytest-cov==2.10.1
 mock==4.0.2
-babel==2.9.1
+babel==2.9.1
+python-gitlab==2.10.1
diff --git a/utils/__init__.py b/utils/__init__.py
diff --git a/utils/create-gitlab-cli-cfg.sh b/utils/create-gitlab-cli-cfg.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+cat > ~/.python-gitlab.cfg << EOF
+[global]
+default = specterdesktop
+ssl_verify = true
+timeout = 5
+
+[specterdesktop]
+url = https://gitlab.com
+#private_token = ${CI_JOB_TOKEN}
+job_token =${CI_JOB_TOKEN}
+api_version = 4
+
+
+EOF
diff --git a/utils/github.py b/utils/github.py
@@ -43,6 +43,11 @@
 logger.setLevel(logging.DEBUG)
 
 
+github_api_root_url = f"https://api.github.com"
+
+github_username = "gitlab_upload_release_binaries"
+
+
 def main():
     if sys.argv[1] != "upload":
         # Maybe something more fancy in the future:
@@ -69,7 +74,7 @@ def main():
         logger.info("Github artifact existing. Skipping upload.")
         exit(0)
     else:
-        logger.info("Github artifact does not exist. Let's upload!")
+        logger.info(f"Github artifact {artifact} does not exist. Let's upload!")
 
     if not "GH_BIN_UPLOAD_PW" in os.environ:
         logger.error("GH_BIN_UPLOAD_PW not found.")
@@ -80,8 +85,7 @@ def main():
         project,
         tag,
         [artifact],
-        "github.com",
-        "gitlab_upload_release_binaries",
+        github_username,
         password,
     )
 
@@ -202,11 +206,101 @@ def get_mimetype(filepath: str) -> str:
     return mime_type
 
 
+def strip_asset_upload_url(asset_upload_url_with_get_params: str) -> str:
+    match_obj = re.match(r"([^{]+)(?:\{.*\})?", asset_upload_url_with_get_params)
+    if not match_obj:
+        raise InvalidUploadUrlError(
+            'The upload url "{}" is not in the expected format.'.format(
+                asset_upload_url_with_get_params
+            )
+        )
+    asset_upload_url = match_obj.group(1)  # type: str
+    return asset_upload_url
+
+
+class GithubConnection:
+    def __init__(self, project):
+        self.github_api_root_url = github_api_root_url
+        self.project = project
+        self.username = github_username
+        self.password = os.environ["GH_BIN_UPLOAD_PW"]
+
+    def fetch_existing_release(self, tag) -> Optional[Release]:
+        try:
+            release_query_url = "{}/repos/{}/releases/tags/{}".format(
+                self.github_api_root_url, self.project, tag
+            )
+            response = requests.get(
+                release_query_url,
+                auth=(self.username, self.password),
+                headers={"Accept": "application/json"},
+            )
+            response.raise_for_status()
+            logger.info(
+                'Fetched the existing release "%s" in the GitHub repository "%s"',
+                tag,
+                self.project,
+            )
+            response_json = response.json()
+            asset_upload_url_with_get_params = response_json["upload_url"]
+            asset_upload_url = strip_asset_upload_url(asset_upload_url_with_get_params)
+            release = Release(response_json["id"], asset_upload_url)
+            return release
+        except requests.HTTPError as e:
+            if e.response.status_code == 404:
+                return None
+            raise HTTPError(
+                'Could not fetch the release "{}" due to a severe HTTP error.'.format(
+                    tag
+                )
+            )
+
+    def list_assets(self, release: Release) -> List[Asset]:
+        try:
+            asset_list_url = "{}/repos/{}/releases/{}/assets".format(
+                self.github_api_root_url, self.project, release.id
+            )
+            response = requests.get(asset_list_url, auth=(self.username, self.password))
+            response.raise_for_status()
+            assets = [
+                Asset(asset_dict["id"], asset_dict["name"])
+                for asset_dict in response.json()
+            ]
+            return assets
+        except requests.HTTPError:
+            raise HTTPError(
+                'Could not get a list of assets for project "{}".'.format(self.project)
+            )
+        except json.decoder.JSONDecodeError:
+            raise JSONError("Got an invalid json string.")
+        except KeyError as e:
+            raise JSONError(
+                'Got an unexpected json object missing the key "{}".'.format(e.args[0])
+            )
+
+    def download_artifact(self, tag, artifact, target_dir="."):
+        artifact_url = (
+            f"https://github.com/{self.project}/releases/download/{tag}/{artifact}"
+        )
+        response = requests.get(artifact_url)
+
+        # If the HTTP GET request can be served
+        if response.status_code == 200:
+
+            # Write the file contents in the response to a file specified by local_file_path
+            with open(os.path.join(target_dir, artifact), "wb") as local_file:
+                for chunk in response.iter_content(chunk_size=128):
+                    local_file.write(chunk)
+        else:
+            raise Exception(
+                f"Status-code {response.status_code} for url {artifact_url}"
+            )
+
+
 def publish_release_from_tag(
     project: str,
     tag: Optional[str],
     asset_filepaths: List[str],
-    github_server: str,
     username: str,
     password: str,
     dry_run: bool = False,
@@ -216,8 +310,6 @@ def publish_release_from_tag(
             'The "requests" package is missing. Please install and run again.'
         )
 
-    github_api_root_url = "https://api.{}".format(github_server)
-
     def fetch_latest_tag() -> str:
         try:
             tags_url = "{}/repos/{}/tags".format(github_api_root_url, project)
@@ -253,19 +345,6 @@ def fetch_latest_tag() -> str:
             )
 
     def publish_release(tag: str) -> Release:
-        def strip_asset_upload_url(asset_upload_url_with_get_params: str) -> str:
-            match_obj = re.match(
-                r"([^{]+)(?:\{.*\})?", asset_upload_url_with_get_params
-            )
-            if not match_obj:
-                raise InvalidUploadUrlError(
-                    'The upload url "{}" is not in the expected format.'.format(
-                        asset_upload_url_with_get_params
-                    )
-                )
-            asset_upload_url = match_obj.group(1)  # type: str
-            return asset_upload_url
-
         def fetch_existing_release() -> Optional[Release]:
             try:
                 release_query_url = "{}/repos/{}/releases/tags/{}".format(