diff --git a/.gitignore b/.gitignore index 0637f05305..a1e4ffeab8 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ tests/bitcoin* token.sh src/cryptoadvance/specter/translations/**/messages.mo tests/elements +signing_dir diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fc052fe381..3c7eda987d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,15 +8,19 @@ variables: cache: # enable per-job and per-branch caching - key: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG" - paths: - - .cache/pip - - .env/ - - ./tests/elements + - key: + files: + - ./requirements.txt + - ./test_requirements.txt + prefix: "$CI_JOB_NAME" + paths: + - .cache/pip + - .env stages: - testing - releasing + - post_releasing before_script: - docker info || echo "no docker-command found" # Print out docker version for debugging @@ -86,7 +90,7 @@ release_pip: - pip3 install -r test_requirements.txt - python3 setup.py install # verifying the version number follows vx.y.z (e.g. "v1.2.3") - - if ! [[ $CI_COMMIT_TAG =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(\+([0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*))?$ ]]; then exit 1; fi + - if ! [[ $CI_COMMIT_TAG =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-pre(0|[1-9][0-9]*))? ]]; then exit 1; fi # set version number in setup.py - echo Releasing $CI_COMMIT_TAG - sed -i "s/version=\".*/version=\"$CI_COMMIT_TAG\",/" setup.py @@ -97,8 +101,13 @@ release_pip: # Either testing it or doing the real thing depending on which gitlab-project we're running: - if ! [[ ${CI_PROJECT_ROOT_NAMESPACE} = "cryptoadvance" ]]; then python3 -m twine upload --verbose --user __token__ dist/* --repository-url https://test.pypi.org/legacy/ ; fi - if [[ ${CI_PROJECT_ROOT_NAMESPACE} = "cryptoadvance" ]]; then python3 -m twine upload --verbose --user __token__ dist/* ; fi - - sha256sum dist/cryptoadvance.specter-*.tar.gz > ./dist/SHA256SUMS-pip.txt - - python ./utils/github.py upload ./dist/SHA256SUMS-pip.txt + - cd dist + - sha256sum cryptoadvance.specter-*.tar.gz > SHA256SUMS-pip + - ../utils/sign_artifact.sh --artifact ./SHA256SUMS-pip + - cd .. + - cat ./dist/SHA256SUMS-pip + #- python ./utils/github.py upload ./dist/SHA256SUMS-pip + #- python ./utils/github.py upload ./dist/SHA256SUMS-pip.asc - python ./utils/github.py upload ./dist/cryptoadvance.specter-*.tar.gz artifacts: when: always @@ -117,6 +126,7 @@ release_binary_windows: - pip3 install virtualenv - virtualenv --python=python3 .env - .\.env\Scripts\activate + - pip3 install -r test_requirements.txt script: # This script won't execute if the script before that fails # No need to check the version-scheme again @@ -125,11 +135,23 @@ release_binary_windows: - cd pyinstaller - .\build-win-ci.bat $CI_COMMIT_TAG - python ../utils/github.py upload ./release/specterd-$CI_COMMIT_TAG-win64.zip + - cd release + - python ..\..\utils\release-helper.py sha256sums specterd-$CI_COMMIT_TAG-win64.zip > SHA256SUMS-windows + - type SHA256SUMS-windows + - echo $GPG_PASSPHRASE | c:\Program` Files` `(x86`)\GnuPg\bin\gpg --detach-sign --armor --no-tty --batch --yes --passphrase-fd 0 --pinentry-mode loopback SHA256SUMS-windows + - dir artifacts: when: always paths: - pyinstaller/release/* expire_in: 1 day + cache: + key: + files: + - ./pyinstaller/electron/package-lock.json + prefix: $CI_JOB_NAME + paths: + - ./pyinstaller/electron/node_modules release_electron_linux_windows: image: registry.gitlab.com/cryptoadvance/specter-desktop/electron-builder:latest @@ -151,21 +173,70 @@ release_electron_linux_windows: - ./build-ci.sh $CI_COMMIT_TAG "make-hash" - ls -l release-linux - ls -l release-win - - sha256sum ./release-linux/specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip ./release-linux/specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz ./release-win/Specter-Setup-${CI_COMMIT_TAG}.exe > ./SHA256SUMS.txt - - cat ./SHA256SUMS.txt - - if [[ -f /credentials/private.key ]]; then gpg --import --no-tty --batch --yes /credentials/private.key; echo $GPG_PASSPHRASE | gpg --detach-sign --no-tty --batch --yes --passphrase-fd 0 --pinentry-mode loopback ./SHA256SUMS.txt || true ; fi - - python ../utils/github.py upload ./release-win/Specter-Setup-${CI_COMMIT_TAG}.exe - - python ../utils/github.py upload ./release-linux/specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip - - python ../utils/github.py upload ./release-linux/specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz - - python ../utils/github.py upload ./SHA256SUMS.txt - - if [[ -f ./SHA256SUMS.txt.sig ]]; then python3 ../utils/github.py upload ./SHA256SUMS.txt.sig ; fi + - cd release-linux + - sha256sum specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz > ./SHA256SUMS-linux + - cat ./SHA256SUMS-linux + - cd .. + - cd release-win + - sha256sum Specter-Setup-${CI_COMMIT_TAG}.exe > ./SHA256SUMS-win + - cat ./SHA256SUMS-win + - cd .. + - ../utils/sign_artifact.sh --artifact ./release-win/SHA256SUMS-win + - ../utils/sign_artifact.sh --artifact ./release-linux/SHA256SUMS-linux + - python3 ../utils/github.py upload ./release-win/Specter-Setup-${CI_COMMIT_TAG}.exe + - python3 ../utils/github.py upload ./release-linux/specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip + - python3 ../utils/github.py upload ./release-linux/specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz + #- python3 ../utils/github.py upload ./release-linux/SHA256SUMS-linux + #- python3 ../utils/github.py upload ./release-linux/SHA256SUMS-linux.asc + #- python3 ../utils/github.py upload ./release-win/SHA256SUMS-win + #- python3 ../utils/github.py upload ./release-win/SHA256SUMS-win.asc + cache: + key: + files: + - ./pyinstaller/electron/package-lock.json + prefix: $CI_JOB_NAME + paths: + - ./pyinstaller/electron/node_modules + artifacts: when: always paths: - - pyinstaller/release-linux/* - - pyinstaller/release-win/* - - pyinstaller/SHA256SUMS.txt + - pyinstaller/release-win/Specter-Setup-${CI_COMMIT_TAG}.exe + - pyinstaller/release-linux/specterd-${CI_COMMIT_TAG}-x86_64-linux-gnu.zip + - pyinstaller/release-linux/specter_desktop-${CI_COMMIT_TAG}-x86_64-linux-gnu.tar.gz + - pyinstaller/release-linux/SHA256SUMS-linux + - pyinstaller/release-linux/SHA256SUMS-linux.asc + - pyinstaller/release-win/SHA256SUMS-win + - pyinstaller/release-win/SHA256SUMS-win.asc expire_in: 1 day +release_signatures: + stage: post_releasing + only: + - tags + before_script: + - python -V # Print out python version for debugging + - pip3 install --upgrade virtualenv + - virtualenv --python=python3 .env + - source .env/bin/activate + - pip3 install -r test_requirements.txt + - ./utils/create-gitlab-cli-cfg.sh + - gpg --import --no-tty --batch --yes /credentials/private.key + script: + - python3 -m utils.release-helper download # downloads the job-artifacts from gitlab + - python3 -m utils.release-helper downloadgithub # downloads additional artifacts from github (if not there and is they have SHA256SUMS-something) + - python3 -m utils.release-helper checkhashes # checks all SHA256SUM* files + - python3 -m utils.release-helper checksig # checks the signatures of all SHA256SUMM*.asc files + - python3 -m utils.release-helper create # creates a SHA256SUM.txt + - python3 -m utils.release-helper upload # uploads it to github + - ./utils/sign_artifact.sh --artifact ./signing_dir/SHA256SUMS + - python3 ./utils/github.py upload ./signing_dir/SHA256SUMS.asc - +release_docker: + stage: post_releasing + only: + - tags + before_script: + - echo "Triggering Docker Release" + script: + - ./utils/trigger_docker_build.sh \ No newline at end of file diff --git a/README.md b/README.md index b1b3bdf0a3..774f9da807 100755 --- a/README.md +++ b/README.md @@ -92,8 +92,8 @@ After that, Specter will be available at [http://127.0.0.1:25441/](http://127.0. The above installation-method is quite easy but you have to trust pypi. If you want to verify the software completely yourself while still installing via pip3, you can do something like this: ``` wget https://github.com/cryptoadvance/specter-desktop/releases/download/v1.4.6/cryptoadvance.specter-1.4.6.tar.gz -wget https://github.com/cryptoadvance/specter-desktop/releases/download/v1.4.6/SHA256SUMS-pip.txt -sha256sum --check SHA256SUMS-pip.txt +wget https://github.com/cryptoadvance/specter-desktop/releases/download/v1.4.6/SHA256SUMS-pip +sha256sum --check SHA256SUMS-pip # Do your usual GPG-check here # Now, let's extract the requirements-file and install all requirements with require-hashes tar -xvzf cryptoadvance.specter-1.4.6.tar.gz cryptoadvance.specter-1.4.6/requirements.txt diff --git a/pyinstaller/build-ci.sh b/pyinstaller/build-ci.sh index 9b7f6f366b..20bf19f827 100755 --- a/pyinstaller/build-ci.sh +++ b/pyinstaller/build-ci.sh @@ -54,7 +54,7 @@ rm -rf dist mkdir dist cd dist echo " --> Downloading the windows-version of specterd for version $1" -wget https://github.com/${CI_PROJECT_ROOT_NAMESPACE}/specter-desktop/releases/download/$1/specterd-$1-win64.zip -O ./specterd.zip +wget --progress=dot -e dotbytes=10M https://github.com/${CI_PROJECT_ROOT_NAMESPACE}/specter-desktop/releases/download/$1/specterd-$1-win64.zip -O ./specterd.zip unzip specterd.zip cd ../electron rm -rf dist/ diff --git a/src/cryptoadvance/specter/util/specter_migrator.py b/src/cryptoadvance/specter/util/specter_migrator.py index c5e594475d..ebc29368eb 100644 --- a/src/cryptoadvance/specter/util/specter_migrator.py +++ b/src/cryptoadvance/specter/util/specter_migrator.py @@ -249,7 +249,7 @@ def set_Execution_log_error_msg(self, id, msg): def _find_exec_log(self, id): for migration in self.migration_executions: - if migration["migration_id"] == id: + if migration.get("migration_id") == id: return migration raise SpecterError(f"Can't find migration_execution with id {id}") @@ -259,7 +259,7 @@ def migration_executions(self): def has_migration_executed(self, migration_id): executed_list = [ - migration_execution["migration_id"] + migration_execution.get("migration_id") for migration_execution in self.migration_executions ] logger.debug(f"Executed migration_classes ids: {executed_list}") diff --git a/test_requirements.txt b/test_requirements.txt index 7c27414788..5971fb02a9 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -7,4 +7,5 @@ pytest==6.1.2 PySocks==1.7.1 pytest-cov==2.10.1 mock==4.0.2 -babel==2.9.1 \ No newline at end of file +babel==2.9.1 +python-gitlab==2.10.1 \ No newline at end of file diff --git a/utils/__init__.py b/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/utils/create-gitlab-cli-cfg.sh b/utils/create-gitlab-cli-cfg.sh new file mode 100755 index 0000000000..399b239d1d --- /dev/null +++ b/utils/create-gitlab-cli-cfg.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +cat > ~/.python-gitlab.cfg << EOF +[global] +default = specterdesktop +ssl_verify = true +timeout = 5 + +[specterdesktop] +url = https://gitlab.com +#private_token = ${CI_JOB_TOKEN} +job_token =${CI_JOB_TOKEN} +api_version = 4 + + +EOF diff --git a/utils/github.py b/utils/github.py index 24f5f6cafd..181264acc0 100644 --- a/utils/github.py +++ b/utils/github.py @@ -43,6 +43,11 @@ logger.setLevel(logging.DEBUG) +github_api_root_url = f"https://api.github.com" + +github_username = "gitlab_upload_release_binaries" + + def main(): if sys.argv[1] != "upload": # Maybe something more fancy in the future: @@ -69,7 +74,7 @@ def main(): logger.info("Github artifact existing. Skipping upload.") exit(0) else: - logger.info("Github artifact does not exist. Let's upload!") + logger.info(f"Github artifact {artifact} does not exist. Let's upload!") if not "GH_BIN_UPLOAD_PW" in os.environ: logger.error("GH_BIN_UPLOAD_PW not found.") @@ -80,8 +85,7 @@ def main(): project, tag, [artifact], - "github.com", - "gitlab_upload_release_binaries", + github_username, password, ) @@ -202,11 +206,101 @@ def get_mimetype(filepath: str) -> str: return mime_type +def strip_asset_upload_url(asset_upload_url_with_get_params: str) -> str: + match_obj = re.match(r"([^{]+)(?:\{.*\})?", asset_upload_url_with_get_params) + if not match_obj: + raise InvalidUploadUrlError( + 'The upload url "{}" is not in the expected format.'.format( + asset_upload_url_with_get_params + ) + ) + asset_upload_url = match_obj.group(1) # type: str + return asset_upload_url + + +class GithubConnection: + def __init__(self, project): + self.github_api_root_url = github_api_root_url + self.project = project + self.username = github_username + self.password = os.environ["GH_BIN_UPLOAD_PW"] + + def fetch_existing_release(self, tag) -> Optional[Release]: + try: + release_query_url = "{}/repos/{}/releases/tags/{}".format( + self.github_api_root_url, self.project, tag + ) + response = requests.get( + release_query_url, + auth=(self.username, self.password), + headers={"Accept": "application/json"}, + ) + response.raise_for_status() + logger.info( + 'Fetched the existing release "%s" in the GitHub repository "%s"', + tag, + self.project, + ) + response_json = response.json() + asset_upload_url_with_get_params = response_json["upload_url"] + asset_upload_url = strip_asset_upload_url(asset_upload_url_with_get_params) + release = Release(response_json["id"], asset_upload_url) + return release + except requests.HTTPError as e: + if e.response.status_code == 404: + return None + raise HTTPError( + 'Could not fetch the release "{}" due to a severe HTTP error.'.format( + tag + ) + ) + + def list_assets(self, release: Release) -> List[Asset]: + try: + asset_list_url = "{}/repos/{}/releases/{}/assets".format( + self.github_api_root_url, self.project, release.id + ) + response = requests.get(asset_list_url, auth=(self.username, self.password)) + response.raise_for_status() + assets = [ + Asset(asset_dict["id"], asset_dict["name"]) + for asset_dict in response.json() + ] + return assets + except requests.HTTPError: + raise HTTPError( + 'Could not get a list of assets for project "{}".'.format(self.project) + ) + except json.decoder.JSONDecodeError: + raise JSONError("Got an invalid json string.") + except KeyError as e: + raise JSONError( + 'Got an unexpected json object missing the key "{}".'.format(e.args[0]) + ) + + def download_artifact(self, tag, artifact, target_dir="."): + artifact_url = ( + f"https://github.com/{self.project}/releases/download/{tag}/{artifact}" + ) + response = requests.get(artifact_url) + + # If the HTTP GET request can be served + if response.status_code == 200: + + # Write the file contents in the response to a file specified by local_file_path + with open(os.path.join(target_dir, artifact), "wb") as local_file: + for chunk in response.iter_content(chunk_size=128): + local_file.write(chunk) + else: + raise Exception( + f"Status-code {response.status_code} for url {artifact_url}" + ) + + def publish_release_from_tag( project: str, tag: Optional[str], asset_filepaths: List[str], - github_server: str, username: str, password: str, dry_run: bool = False, @@ -216,8 +310,6 @@ def publish_release_from_tag( 'The "requests" package is missing. Please install and run again.' ) - github_api_root_url = "https://api.{}".format(github_server) - def fetch_latest_tag() -> str: try: tags_url = "{}/repos/{}/tags".format(github_api_root_url, project) @@ -253,19 +345,6 @@ def fetch_latest_tag() -> str: ) def publish_release(tag: str) -> Release: - def strip_asset_upload_url(asset_upload_url_with_get_params: str) -> str: - match_obj = re.match( - r"([^{]+)(?:\{.*\})?", asset_upload_url_with_get_params - ) - if not match_obj: - raise InvalidUploadUrlError( - 'The upload url "{}" is not in the expected format.'.format( - asset_upload_url_with_get_params - ) - ) - asset_upload_url = match_obj.group(1) # type: str - return asset_upload_url - def fetch_existing_release() -> Optional[Release]: try: release_query_url = "{}/repos/{}/releases/tags/{}".format( diff --git a/utils/release-helper.py b/utils/release-helper.py new file mode 100644 index 0000000000..04f7aa30fc --- /dev/null +++ b/utils/release-helper.py @@ -0,0 +1,299 @@ +import hashlib +import logging +import os +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path + + +logging.basicConfig(format="%(levelname)s:%(message)s", level=logging.INFO) +logger = logging.getLogger(__name__) +logger.setLevel(logging.DEBUG) + + +class Sha256sumFile: + def __init__(self, name, target_dir="./signing_dir"): + self.name = name + self.target_dir = target_dir + self.hashed_files = {} + + def is_in_target_dir(self): + return os.path.isfile(os.path.join(self.target_dir, self.name)) + + def download_from_tag(self, tag, gc): + gc.download_artifact(tag, self.name, target_dir=self.target_dir) + gc.download_artifact(tag, self.name + ".asc", target_dir=self.target_dir) + self.read() + + def download_hashed_files(self, gc): + for file in self.hashed_files.keys(): + logger.info(f"Downloading {file} from {tag}") + gc.download_artifact(tag, file, target_dir=self.target_dir) + + def read(self): + with open(os.path.join(self.target_dir, self.name), "r") as file: + line = file.readline() + while line: + line = line.split(maxsplit=2) + self.hashed_files[line[1]] = line[0] + line = file.readline() + + def print(self): + for hashed_file, hash in self.hashed_files.items(): + print(f"{hash} {hashed_file}") + + def write(self): + with open(os.path.join(self.target_dir, self.name), "w") as file: + for hashed_file, hash in self.hashed_files.items(): + file.write(f"{hash} {hashed_file}") + + def add_file(self, file): + self.hashed_files[file] = Sha256sumFile.sha256_checksum(file, self.target_dir) + + def check_hashes(self): + returncode = subprocess.call( + ["sha256sum", "-c", self.name], cwd=self.target_dir + ) + if returncode != 0: + raise Exception( + f"Could not validate hashes for file {self.name}: {subprocess.run(['sha256sum', '-c', self.name], cwd=self.target_dir)}" + ) + + def check_sig(self): + returncode = subprocess.call( + ["gpg", "--verify", self.name + ".asc"], cwd=self.target_dir + ) + if returncode != 0: + raise Exception(f"Could not validate signature of file {self.name}") + + @classmethod + def sha256_checksum(cls, filename, folder, block_size=65536): + sha256 = hashlib.sha256() + with open(os.path.join(folder, filename), "rb") as f: + for block in iter(lambda: f.read(block_size), b""): + sha256.update(block) + return sha256.hexdigest() + + +class ReleaseHelper: + def __init__(self): + pass + + def init_gitlab(self): + import gitlab + + if os.environ.get("GITLAB_PRIVATE_TOKEN"): + logger.info("Using GITLAB_PRIVATE_TOKEN") + self.gl = gitlab.Gitlab( + "http://gitlab.com", + private_token=os.environ.get("GITLAB_PRIVATE_TOKEN"), + ) + elif os.environ.get("CI_JOB_TOKEN"): + logger.info("Using CI_JOB_TOKEN") + self.gl = gitlab.Gitlab( + "http://gitlab.com", job_token=os.environ["CI_JOB_TOKEN"] + ) + else: + raise Exception( + "Can't authenticate against Gitlab ( export GITLAB_PRIVATE_TOKEN )" + ) + + if os.environ.get("CI_PROJECT_ROOT_NAMESPACE"): + project_root_namespace = os.environ.get("CI_PROJECT_ROOT_NAMESPACE") + logger.info(f"Using project_root_namespace: {project_root_namespace}") + else: + raise Exception( + "no CI_PROJECT_ROOT_NAMESPACE given ( export CI_PROJECT_ROOT_NAMESPACE=k9ert )" + ) + + if os.environ.get("CI_PROJECT_ID"): + self.project_id = os.environ.get("CI_PROJECT_ID") + self.github_project = f"{project_root_namespace}/specter-desktop" + else: + self.project_id = 15721074 # cryptoadvance/specter-desktop + self.project_id = 15541285 + self.github_project = f"{project_root_namespace}/specter-desktop" + + logger.info(f"Using project_id: {self.project_id}") + logger.info(f"Using github_project: {self.github_project}") + + self.project = self.gl.projects.get(self.project_id) + + if os.environ.get("CI_PIPELINE_ID"): + pipeline_id = os.environ.get("CI_PIPELINE_ID") + else: + pipeline_id = 387387482 # cryptoadavance v1.7.0-pre1 + pipeline_id = 389780348 # k9ert v0.0.1-pre1 + + logger.info(f"Using pipeline_id: {pipeline_id}") + + if os.environ.get("CI_COMMIT_TAG"): + self.tag = os.environ.get("CI_COMMIT_TAG") + else: + raise Exception("no tag given ( export CI_COMMIT_TAG=v0.0.0.0-pre13 )") + + logger.info(f"Using tag: {self.tag}") + + self.pipeline = self.project.pipelines.get(pipeline_id) + + self.target_dir = "signing_dir" + + Path(self.target_dir).mkdir(parents=True, exist_ok=True) + + def download_and_unpack_all_artifacts(self): + if os.path.isdir(self.target_dir): + logger.info(f"First purging {self.target_dir}") + shutil.rmtree(self.target_dir) + for job in self.pipeline.jobs.list(): + if job.name in [ + "release_electron_linux_windows", + "release_binary_windows", + "release_pip", + ]: + zipfn = f"/tmp/_artifacts_{job.name}.zip" + job_obj = self.project.jobs.get(job.id, lazy=True) + + if not os.path.isfile(zipfn): + logger.info(f"Downloading artifacts for {job.name}") + with open(zipfn, "wb") as f: + job_obj.artifacts(streamed=True, action=f.write) + else: + logger.info(f"Skipping Download artifacts for {job.name}") + + logger.info(f"Unzipping {zipfn} in target-folder") + with zipfile.ZipFile(zipfn, "r") as zip: + for zip_info in zip.infolist(): + if zip_info.filename[-1] == "/": + continue + zip_info.filename = os.path.basename(zip_info.filename) + logger.info(f" Extracting {zip_info.filename}") + zip.extract(zip_info, self.target_dir) + + def download_and_unpack_new_artifacts_from_github(self): + from utils import github + + gc = github.GithubConnection(self.github_project) + release = gc.fetch_existing_release(self.tag) + assets = gc.list_assets(release) + for asset in assets: + if not asset.name.startswith("SHA256"): + continue + if asset.name == "SHA256SUMS" or asset.name == "SHA256SUMS.asc": + continue + if asset.name.endswith(".asc"): + continue + shasumfile = Sha256sumFile(asset.name) + if not shasumfile.is_in_target_dir(): + shasumfile.download_from_tag(self.tag, gc) + shasumfile.download_hashed_files(gc) + shasumfile.check_hashes() + shasumfile.check_sig() + + def create_sha256sum_file(self): + with open(f"{self.target_dir}/SHA256SUMS", "w") as shafile: + for file in os.listdir(self.target_dir): + if file.startswith("SHA256SUMS-") and not file.endswith(".asc"): + logger.debug(f"Processing {file}") + sha_src = Sha256sumFile(file) + sha_src.read() + for hashed_file in sha_src.hashed_files.keys(): + print(f"{sha_src.hashed_files[hashed_file]} {hashed_file}\n") + shafile.write( + f"{sha_src.hashed_files[hashed_file]} {hashed_file}\n" + ) + returncode = subprocess.call( + ["sha256sum", "-c", "SHA256SUMS"], cwd=self.target_dir + ) + if returncode != 0: + raise Exception( + f"One of the hashes is not matching: {subprocess.run(['sha256sum', '-c', 'SHA256SUMS'], cwd=self.target_dir)}" + ) + + def check_all_hashes(self): + for file in os.listdir(self.target_dir): + if file.startswith("SHA256SUM") and not file.endswith(".asc"): + returncode = subprocess.call( + ["sha256sum", "-c", file], cwd=self.target_dir + ) + if returncode != 0: + raise Exception(f"Could not validate hashes for file {file}") + + def check_all_sigs(self): + for file in os.listdir(self.target_dir): + if file.endswith(".asc"): + returncode = subprocess.call( + ["gpg", "--verify", file], cwd=self.target_dir + ) + if returncode != 0: + raise Exception( + f"Could not validate signature of file {file}: {subprocess.run(['gpg', '--verify', file], cwd=self.target_dir)}" + ) + + def calculate_publish_params(self): + if not "CI_PROJECT_ROOT_NAMESPACE" in os.environ: + logger.error("CI_PROJECT_ROOT_NAMESPACE not found") + exit(2) + else: + self.github_project = ( + f"{os.environ['CI_PROJECT_ROOT_NAMESPACE']}/specter-desktop" + ) + if not "CI_COMMIT_TAG" in os.environ: + logger.error("CI_COMMIT_TAG not found") + exit(2) + else: + tag = os.environ["CI_COMMIT_TAG"] + if not "GH_BIN_UPLOAD_PW" in os.environ: + logger.error("GH_BIN_UPLOAD_PW not found.") + exit(2) + else: + self.password = os.environ["GH_BIN_UPLOAD_PW"] + + def upload_sha256sum_file(self): + artifact = os.path.join("signing_dir", "SHA256SUMS") + self.calculate_publish_params() + + if self.github.artifact_exists( + self.github_project, self.tag, Path(artifact).name + ): + logger.info(f"Github artifact {artifact} existing. Skipping upload.") + exit(0) + else: + logger.info(f"Github artifact {artifact} does not exist. Let's upload!") + self.github.publish_release_from_tag( + self.github_project, + self.tag, + [artifact], + "github.com", + "gitlab_upload_release_binaries", + self.password, + ) + + +def sha256sum(filenames): + sha_file = Sha256sumFile("SHA256SUMS", target_dir=".") + for filename in filenames: + logger.info(f"Adding {filename}") + sha_file.add_file(filename) + sha_file.print() + + +if __name__ == "__main__": + if "sha256sums" in sys.argv: + sha256sum(sys.argv[2:]) + exit(0) + rh = ReleaseHelper() + rh.init_gitlab() + if "download" in sys.argv: + rh.download_and_unpack_all_artifacts() + if "downloadgithub" in sys.argv: + rh.download_and_unpack_new_artifacts_from_github() + if "checkhashes" in sys.argv: + rh.check_all_hashes() + if "checksigs" in sys.argv: + rh.check_all_sigs() + if "create" in sys.argv: + rh.create_sha256sum_file() + if "upload" in sys.argv: + rh.upload_sha256sum_file() diff --git a/utils/sign_artifact.sh b/utils/sign_artifact.sh new file mode 100755 index 0000000000..5b464e56c4 --- /dev/null +++ b/utils/sign_artifact.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +while [[ $# -gt 0 ]] +do +key="$1" +command="main" +case $key in + --artifact) + artifact=$2 + shift + shift + ;; + --debug) + set -x + shift # past argument + ;; + *) # unknown option + POSITIONAL="$1" # save it in an array for later + shift # past argument + ;; +esac +done + +if [[ -z $artifact ]]; then + echo "no --artifact given " + exit 1 +fi + + +# We want a detached signature in cleartext. Extension: .asc (as in bitcoin) +output_file=${artifact}.asc + +if [[ -f /credentials/private.key ]]; then + echo "signing ..." ; + gpg --import --no-tty --batch --yes /credentials/private.key + echo $GPG_PASSPHRASE | gpg --detach-sign --armor --no-tty --batch --yes --passphrase-fd 0 --pinentry-mode loopback $artifact +else + gpg --detach-sign --armor $artifact +fi \ No newline at end of file diff --git a/utils/test-cypress.sh b/utils/test-cypress.sh index 0ea6a84793..ecb8d115c5 100755 --- a/utils/test-cypress.sh +++ b/utils/test-cypress.sh @@ -44,11 +44,11 @@ Subcommands: run [spec-file] will run the tests. open and run take a spec-file optionally. If you add a spec-file, then automatically the corresponding snapshot is untarred before and, - in the case of run, only the spec-file and all subsequent spec_files - are executed. - snapshot will create a snapshot of the spec-file. It will create a tarball - of the btc-dir and the specter-dir and store those files in the - ./cypress/fixtures directory + in the case of run, only the spec-file and all subsequent spec_files + are executed. + snapshot [spec_file] will create a snapshot of the spec-file. It will create a tarball + of the btc-/elm-dir and the specter-dir and store those files in the + ./cypress/fixtures directory generic-options: --debug Run as much stuff in debug as we can diff --git a/utils/trigger_docker_build.sh b/utils/trigger_docker_build.sh new file mode 100755 index 0000000000..83bc9e57dc --- /dev/null +++ b/utils/trigger_docker_build.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +payload="{\"ref\":\"master\", \"inputs\": {\"tag\": \"${CI_COMMIT_TAG}\"}}" +echo $payload +# This Token is controlled by https://github.com/AaronDewes + +curl -X POST -H "Accept: application/vnd.github.v3+json" -H "Authorization: token ${AARON_TOKEN}" \ + https://api.github.com/repos/lncm/docker-specter-desktop/actions/workflows/dispatch.yml/dispatches -d @<(cat <