From d1e871ddb1caaaf76714176bb568cc84413f5077 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Mon, 15 Apr 2024 21:36:32 +0200 Subject: [PATCH 1/3] feat: Add support for .conda format Signed-off-by: Marcel Bargull --- bioconda_utils/artifacts.py | 19 ++++++----- .../bioconda_utils-requirements.txt | 1 + bioconda_utils/cli.py | 34 ++++++++++--------- bioconda_utils/docker_utils.py | 7 ++-- bioconda_utils/pkg_test.py | 17 ++++++---- bioconda_utils/utils.py | 9 +++-- 6 files changed, 50 insertions(+), 37 deletions(-) diff --git a/bioconda_utils/artifacts.py b/bioconda_utils/artifacts.py index c77afb1fb4..59ceaeaa5a 100644 --- a/bioconda_utils/artifacts.py +++ b/bioconda_utils/artifacts.py @@ -74,15 +74,16 @@ def upload_pr_artifacts(config, repo, git_sha, dryrun=False, mulled_upload_targe platform_patterns.append("noarch") for platform_pattern in platform_patterns: - pattern = f"{tmpdir}/*/packages/{platform_pattern}/*.tar.bz2" - logger.info(f"Checking for packages at {pattern}.") - for pkg in glob.glob(pattern): - if dryrun: - logger.info(f"Would upload {pkg} to anaconda.org.") - else: - logger.info(f"Uploading {pkg} to anaconda.org.") - # upload the package - success.append(anaconda_upload(pkg, label=label)) + for ext in (".tar.bz2", ".conda"): + pattern = f"{tmpdir}/*/packages/{platform_pattern}/*{ext}" + logger.info(f"Checking for packages at {pattern}.") + for pkg in glob.glob(pattern): + if dryrun: + logger.info(f"Would upload {pkg} to anaconda.org.") + else: + logger.info(f"Uploading {pkg} to anaconda.org.") + # upload the package + success.append(anaconda_upload(pkg, label=label)) if mulled_upload_target: quay_login = os.environ['QUAY_LOGIN'] diff --git a/bioconda_utils/bioconda_utils-requirements.txt b/bioconda_utils/bioconda_utils-requirements.txt index e026e092fa..2b8dcbf655 100644 --- a/bioconda_utils/bioconda_utils-requirements.txt +++ b/bioconda_utils/bioconda_utils-requirements.txt @@ -31,6 +31,7 @@ anaconda-client=1.12.* # anaconda_upload galaxy-tool-util=24.* # mulled test and container build involucro=1.1.* # mulled test and container build skopeo=1.15.* # docker upload +findutils # find/xargs copy built packages from container git=2.* # well - git # hosters - special regex not supported by RE diff --git a/bioconda_utils/cli.py b/bioconda_utils/cli.py index 7a5cfada6b..8f75948ce0 100644 --- a/bioconda_utils/cli.py +++ b/bioconda_utils/cli.py @@ -238,23 +238,25 @@ def duplicates(config, check_fields += ['build'] def remove_package(spec): - fn = '{}-{}-{}.tar.bz2'.format(*spec) - name, version = spec[:2] - subcmd = [ - 'remove', '-f', - '{channel}/{name}/{version}/{fn}'.format( - name=name, version=version, fn=fn, channel=our_channel - ) - ] - if dryrun: - logger.info(" ".join([utils.bin_for('anaconda')] + subcmd)) - else: - token = os.environ.get('ANACONDA_TOKEN') - if token is None: - token = [] + for ext in (".tar.bz2", ".conda"): + name, version = spec[:2] + dist = '{}-{}-{}'.format(*spec) + fn = f"{dist}{ext}" + subcmd = [ + 'remove', '-f', + '{channel}/{name}/{version}/{fn}'.format( + name=name, version=version, fn=fn, channel=our_channel + ) + ] + if dryrun: + logger.info(" ".join([utils.bin_for('anaconda')] + subcmd)) else: - token = ['-t', token] - logger.info(utils.run([utils.bin_for('anaconda')] + token + subcmd, mask=[token]).stdout) + token = os.environ.get('ANACONDA_TOKEN') + if token is None: + token = [] + else: + token = ['-t', token] + logger.info(utils.run([utils.bin_for('anaconda')] + token + subcmd, mask=[token]).stdout) # packages in our channel repodata = utils.RepoData() diff --git a/bioconda_utils/docker_utils.py b/bioconda_utils/docker_utils.py index cfeecbd7bf..3479444b18 100644 --- a/bioconda_utils/docker_utils.py +++ b/bioconda_utils/docker_utils.py @@ -83,7 +83,7 @@ # filled in here. # BUILD_SCRIPT_TEMPLATE = \ -""" +r""" #!/bin/bash set -eo pipefail @@ -111,9 +111,10 @@ conda-build -c file://{self.container_staging} {self.conda_build_args} {self.container_recipe}/meta.yaml 2>&1 # copy all built packages to the staging area -cp /opt/conda/conda-bld/*/*.tar.bz2 {self.container_staging}/{arch} +find /opt/conda/conda-bld -type f \( -name '*.tar.bz2' -o -name '*.conda' \) -print0 | + xargs -0 -- cp -t '{self.container_staging}/{arch}' -- #While technically better, this is slower and more prone to breaking -#cp `conda-build {self.conda_build_args} {self.container_recipe}/meta.yaml --output | grep tar.bz2` {self.container_staging}/{arch} +#cp `conda-build {self.conda_build_args} {self.container_recipe}/meta.yaml --output | grep -e '\.tar\.bz2$' -e '\.conda$')` {self.container_staging}/{arch} conda index {self.container_staging} # Ensure permissions are correct on the host. HOST_USER={self.user_info[uid]} diff --git a/bioconda_utils/pkg_test.py b/bioconda_utils/pkg_test.py index c85628c26a..950cede3b6 100644 --- a/bioconda_utils/pkg_test.py +++ b/bioconda_utils/pkg_test.py @@ -70,12 +70,17 @@ def get_image_name(path): ---------- path : str - Path to .tar.by2 package build by conda-build + Path to .tar.bz2 or .conda package build by conda-build """ - assert path.endswith('.tar.bz2') - - pkg = os.path.basename(path).replace('.tar.bz2', '') + if path.endswith(".tar.bz2"): + ext = ".tar.bz2" + elif path.endswith(".conda"): + ext = ".conda" + else: + raise ValueError() + + pkg = os.path.basename(path).removesuffix(ext) toks = pkg.split('-') build_string = toks[-1] version = toks[-2] @@ -100,7 +105,7 @@ def test_package( Parameters ---------- path : str - Path to a .tar.bz2 package built by conda-build + Path to a .tar.bz2 or .conda package built by conda-build name_override : str Passed as the --name-override argument to mulled-build @@ -125,7 +130,7 @@ def test_package( If True, enable live logging during the build process """ - assert path.endswith('.tar.bz2'), "Unrecognized path {0}".format(path) + assert path.endswith((".tar.bz2", ".conda")), "Unrecognized path {0}".format(path) # assert os.path.exists(path), '{0} does not exist'.format(path) conda_bld_dir = os.path.abspath(os.path.dirname(os.path.dirname(path))) diff --git a/bioconda_utils/utils.py b/bioconda_utils/utils.py index 4a8bf67891..242fc7ea6e 100644 --- a/bioconda_utils/utils.py +++ b/bioconda_utils/utils.py @@ -1492,13 +1492,16 @@ def _load_channel_dataframe(self): def to_dataframe(json_data, meta_data): channel, platform = meta_data repo = json.loads(json_data) - df = pd.DataFrame.from_dict(repo['packages'], 'index', - columns=self._load_columns) + subdir = repo["info"]["subdir"] + packages = repo["packages"] + packages.update(repo.get("packages.conda", {})) + + df = pd.DataFrame.from_dict(packages, 'index', columns=self._load_columns) # Ensure that version is always a string. df['version'] = df['version'].astype(str) df['channel'] = channel df['platform'] = platform - df['subdir'] = repo['info']['subdir'] + df['subdir'] = subdir return df if urls: From 770d401be3758fc7d7bda70349217baaa0c24781 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Mon, 15 Apr 2024 21:40:57 +0200 Subject: [PATCH 2/3] fix: Ignore Zstandard-packed repodata artifact Signed-off-by: Marcel Bargull --- bioconda_utils/artifacts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bioconda_utils/artifacts.py b/bioconda_utils/artifacts.py index 59ceaeaa5a..24add7eb26 100644 --- a/bioconda_utils/artifacts.py +++ b/bioconda_utils/artifacts.py @@ -214,7 +214,7 @@ def get_circleci_artifacts(check_run, platform): else: for artifact in json_job["items"]: artifact_url = artifact["url"] - if artifact_url.endswith(".html") or artifact_url.endswith(".json") or artifact_url.endswith(".json.bz2"): + if artifact_url.endswith((".html", ".json", ".json.bz2", ".json.zst")): continue else: yield artifact_url From 43be58fcb8bbecb62357aa0ea1ca519ceb140635 Mon Sep 17 00:00:00 2001 From: Marcel Bargull Date: Mon, 15 Apr 2024 21:43:02 +0200 Subject: [PATCH 3/3] fix: Ignore src_cache for container-built packages Signed-off-by: Marcel Bargull --- bioconda_utils/docker_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bioconda_utils/docker_utils.py b/bioconda_utils/docker_utils.py index 3479444b18..af56a316f8 100644 --- a/bioconda_utils/docker_utils.py +++ b/bioconda_utils/docker_utils.py @@ -111,7 +111,9 @@ conda-build -c file://{self.container_staging} {self.conda_build_args} {self.container_recipe}/meta.yaml 2>&1 # copy all built packages to the staging area -find /opt/conda/conda-bld -type f \( -name '*.tar.bz2' -o -name '*.conda' \) -print0 | +find /opt/conda/conda-bld \ + -name src_cache -prune -o \ + -type f \( -name '*.tar.bz2' -o -name '*.conda' \) -print0 | xargs -0 -- cp -t '{self.container_staging}/{arch}' -- #While technically better, this is slower and more prone to breaking #cp `conda-build {self.conda_build_args} {self.container_recipe}/meta.yaml --output | grep -e '\.tar\.bz2$' -e '\.conda$')` {self.container_staging}/{arch}