diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..9ebfc48 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,33 @@ +repos: + - repo: https://github.com/Quantco/pre-commit-mirrors-black + rev: 23.1.0 + hooks: + - id: black-conda + args: + - --safe + - --target-version=py38 + - repo: https://github.com/Quantco/pre-commit-mirrors-flake8 + rev: 6.0.0 + hooks: + - id: flake8-conda + - repo: https://github.com/Quantco/pre-commit-mirrors-isort + rev: 5.12.0 + hooks: + - id: isort-conda + additional_dependencies: [-c, conda-forge, toml=0.10.2] + #- repo: https://github.com/Quantco/pre-commit-mirrors-mypy + # rev: "0.991" + # hooks: + # - id: mypy-conda + # additional_dependencies: [-c, conda-forge, types-setuptools] + - repo: https://github.com/Quantco/pre-commit-mirrors-pyupgrade + rev: 3.3.1 + hooks: + - id: pyupgrade-conda + args: + - --py38 + - repo: https://github.com/Quantco/pre-commit-mirrors-prettier + rev: 2.7.1 + hooks: + - id: prettier-conda + files: "\\.md$" diff --git a/AUTHORS.md b/AUTHORS.md index 8bb4097..2023c73 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -1,19 +1,19 @@ All of the people who have made at least one contribution to conda-mirror. Authors are sorted by number of commits. -* Eric Dill -* Magnus Holm -* Ilan Schnell -* Peter Parente -* Anthony Scopatz -* John Kirkham -* Michael Peters -* magnuhho -* Uwe L. Korn -* Diogo Pereira -* Jens Nie -* Willi Rath -* David Kent -* Johannes Haberl -* Marius van Niekerk -* Bruno Oliveira +- Eric Dill +- Magnus Holm +- Ilan Schnell +- Peter Parente +- Anthony Scopatz +- John Kirkham +- Michael Peters +- magnuhho +- Uwe L. Korn +- Diogo Pereira +- Jens Nie +- Willi Rath +- David Kent +- Johannes Haberl +- Marius van Niekerk +- Bruno Oliveira diff --git a/CHANGELOG.md b/CHANGELOG.md index 7af2958..d51b21b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,71 +6,68 @@ **Implemented enhancements:** -* tba +- tba **Contributors:** -* tba +- tba ## [0.9.0](https://github.com/regro/conda-mirror/tree/0.9.0) (2023-01-13) **Implemented enhancements:** -* Address the lack of support of `.conda` packages +- Address the lack of support of `.conda` packages [\#10](https://github.com/conda-incubator/conda-mirror/issues/10) -* CI: Convert Travis to GitHub workflow -* Support ability to include dependencies of whitelist +- CI: Convert Travis to GitHub workflow +- Support ability to include dependencies of whitelist [\#42](https://github.com/conda-incubator/conda-mirror/issues/42) -* Added ability to include dependencies of required packages ('--include-depends`) +- Added ability to include dependencies of required packages ('--include-depends`) [\#22](https://github.com/conda-incubator/conda-mirror/issues/22) -* Support use of conda package version specifiers +- Support use of conda package version specifiers [\#37](https://github.com/conda-incubator/conda-mirror/issues/37) -* Added tqdm based progress bars. +- Added tqdm based progress bars. [\#29](https://github.com/conda-incubator/conda-mirror/issues/29) -* Improve download speed, especially for smaller packages. +- Improve download speed, especially for smaller packages. [\#28](https://github.com/conda-incubator/conda-mirror/issues/28) -* Add optional targets for reference.json and update.tar to conda-diff-tar +- Add optional targets for reference.json and update.tar to conda-diff-tar [\#19](https://github.com/conda-incubator/conda-mirror/issues/19) -* Switch to using yaml.safe_load to avoid Deprecation warnings. +- Switch to using yaml.safe_load to avoid Deprecation warnings. **Contributors:** -* Christopher Barber -* Faustin Carter -* Alessandro Silenzi -* Simon Li -* Uwe L. Korn - +- Christopher Barber +- Faustin Carter +- Alessandro Silenzi +- Simon Li +- Uwe L. Korn ## [0.8.2](https://github.com/regro/conda-mirror/tree/0.8.2) (2020-02-27) **Implemented enhancements:** -* New conda-diff-tar command, a tools which allows creating +- New conda-diff-tar command, a tools which allows creating differential tarballs of a (usually mirrored) conda repository. **Contributors:** -* Ilan Schnell -* Anthony Scopatz -* John Kirkham -* Uwe L. Korn - - +- Ilan Schnell +- Anthony Scopatz +- John Kirkham +- Uwe L. Korn ## [0.8.1](https://github.com/regro/conda-mirror/tree/0.8.1) (2019-20-22) **Implemented enhancements:** -* Added proxy support (`--proxy`) for running conda-mirror HTTP requests through a +- Added proxy support (`--proxy`) for running conda-mirror HTTP requests through a proxy server. -* Added Certificate authority checking via `--ssl-verify` option. -* New capability to run in insecure mode `-k, --insecure`. -* May now pass in the number of retries for HTTP requests with `--max-retries`. +- Added Certificate authority checking via `--ssl-verify` option. +- New capability to run in insecure mode `-k, --insecure`. +- May now pass in the number of retries for HTTP requests with `--max-retries`. **Contributors:** -* Anthony Scopatz +- Anthony Scopatz ## [0.8.0](https://github.com/maxpoint/conda-mirror/tree/0.8.0) @@ -96,6 +93,7 @@ - ENH: First pass at dry-run [\#59](https://github.com/maxpoint/conda-mirror/pull/59) ([ericdill](https://github.com/ericdill)) ## [0.7.3](https://github.com/maxpoint/conda-mirror/tree/0.7.3) (2017-08-07) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.7.2...0.7.3) **Fixed bugs:** @@ -109,10 +107,11 @@ **Merged pull requests:** - Handle EOFError when reading tar files. [\#58](https://github.com/maxpoint/conda-mirror/pull/58) ([dmkent](https://github.com/dmkent)) -- Add release script that edits the output of github\_changelog\_generator [\#55](https://github.com/maxpoint/conda-mirror/pull/55) ([ericdill](https://github.com/ericdill)) +- Add release script that edits the output of github_changelog_generator [\#55](https://github.com/maxpoint/conda-mirror/pull/55) ([ericdill](https://github.com/ericdill)) - DOC: Rerun changelog generator [\#54](https://github.com/maxpoint/conda-mirror/pull/54) ([ericdill](https://github.com/ericdill)) ## [0.7.2](https://github.com/maxpoint/conda-mirror/tree/0.7.2) (2017-05-23) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.7.1...0.7.2) **Merged pull requests:** @@ -122,6 +121,7 @@ - Raise don't exit when required args aren't present [\#51](https://github.com/maxpoint/conda-mirror/pull/51) ([ericdill](https://github.com/ericdill)) ## [0.7.1](https://github.com/maxpoint/conda-mirror/tree/0.7.1) (2017-05-18) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.7.0...0.7.1) **Merged pull requests:** @@ -129,6 +129,7 @@ - Capture information on downloaded and removed packages [\#50](https://github.com/maxpoint/conda-mirror/pull/50) ([ericdill](https://github.com/ericdill)) ## [0.7.0](https://github.com/maxpoint/conda-mirror/tree/0.7.0) (2017-05-11) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.6.6...0.7.0) **Closed issues:** @@ -141,6 +142,7 @@ - TST: Fix broken test [\#46](https://github.com/maxpoint/conda-mirror/pull/46) ([ericdill](https://github.com/ericdill)) ## [0.6.6](https://github.com/maxpoint/conda-mirror/tree/0.6.6) (2017-03-28) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.6.5...0.6.6) **Implemented enhancements:** @@ -162,6 +164,7 @@ - Package standard docs [\#37](https://github.com/maxpoint/conda-mirror/pull/37) ([jakirkham](https://github.com/jakirkham)) ## [0.6.5](https://github.com/maxpoint/conda-mirror/tree/0.6.5) (2017-02-23) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.6.4...0.6.5) **Closed issues:** @@ -173,6 +176,7 @@ - Return after removing the package [\#32](https://github.com/maxpoint/conda-mirror/pull/32) ([ericdill](https://github.com/ericdill)) ## [0.6.4](https://github.com/maxpoint/conda-mirror/tree/0.6.4) (2017-02-23) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.6.3...0.6.4) **Closed issues:** @@ -184,6 +188,7 @@ - Fix Type Error [\#30](https://github.com/maxpoint/conda-mirror/pull/30) ([ericdill](https://github.com/ericdill)) ## [0.6.3](https://github.com/maxpoint/conda-mirror/tree/0.6.3) (2017-02-15) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.6.2...0.6.3) **Closed issues:** @@ -196,6 +201,7 @@ - Updates to README, Travis, test coverage [\#27](https://github.com/maxpoint/conda-mirror/pull/27) ([opiethehokie](https://github.com/opiethehokie)) ## [0.6.2](https://github.com/maxpoint/conda-mirror/tree/0.6.2) (2017-02-07) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.6.1...0.6.2) **Merged pull requests:** @@ -203,6 +209,7 @@ - Handle fully qualified channels [\#25](https://github.com/maxpoint/conda-mirror/pull/25) ([ericdill](https://github.com/ericdill)) ## [0.6.1](https://github.com/maxpoint/conda-mirror/tree/0.6.1) (2017-01-18) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.6.0...0.6.1) **Closed issues:** @@ -215,13 +222,15 @@ - DOC/MNT: Remove dead code. Update readme. Update changelog [\#22](https://github.com/maxpoint/conda-mirror/pull/22) ([ericdill](https://github.com/ericdill)) ## [0.6.0](https://github.com/maxpoint/conda-mirror/tree/0.6.0) (2017-01-10) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.5.1...0.6.0) **Merged pull requests:** -- Update conda\_mirror.py [\#21](https://github.com/maxpoint/conda-mirror/pull/21) ([jneines](https://github.com/jneines)) +- Update conda_mirror.py [\#21](https://github.com/maxpoint/conda-mirror/pull/21) ([jneines](https://github.com/jneines)) ## [0.5.1](https://github.com/maxpoint/conda-mirror/tree/0.5.1) (2017-01-03) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.5.0...0.5.1) **Merged pull requests:** @@ -229,11 +238,12 @@ - download repodata to temp dir first, then move it [\#20](https://github.com/maxpoint/conda-mirror/pull/20) ([ericdill](https://github.com/ericdill)) ## [0.5.0](https://github.com/maxpoint/conda-mirror/tree/0.5.0) (2016-12-19) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.4.6...0.5.0) **Closed issues:** -- Remove conda\_build as a dependency [\#13](https://github.com/maxpoint/conda-mirror/issues/13) +- Remove conda_build as a dependency [\#13](https://github.com/maxpoint/conda-mirror/issues/13) **Merged pull requests:** @@ -241,18 +251,23 @@ - Updated badges to point to new org. [\#17](https://github.com/maxpoint/conda-mirror/pull/17) ([mariusvniekerk](https://github.com/mariusvniekerk)) ## [0.4.6](https://github.com/maxpoint/conda-mirror/tree/0.4.6) (2016-12-15) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.4.5...0.4.6) ## [0.4.5](https://github.com/maxpoint/conda-mirror/tree/0.4.5) (2016-12-15) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.4.4...0.4.5) ## [0.4.4](https://github.com/maxpoint/conda-mirror/tree/0.4.4) (2016-12-15) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.4.3...0.4.4) ## [0.4.3](https://github.com/maxpoint/conda-mirror/tree/0.4.3) (2016-12-15) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.4.2...0.4.3) ## [0.4.2](https://github.com/maxpoint/conda-mirror/tree/0.4.2) (2016-12-13) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.4.1...0.4.2) **Merged pull requests:** @@ -260,9 +275,11 @@ - Remove packages that fail their assertion [\#16](https://github.com/maxpoint/conda-mirror/pull/16) ([ericdill](https://github.com/ericdill)) ## [0.4.1](https://github.com/maxpoint/conda-mirror/tree/0.4.1) (2016-12-13) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.4.0...0.4.1) ## [0.4.0](https://github.com/maxpoint/conda-mirror/tree/0.4.0) (2016-12-12) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.3.0...0.4.0) **Merged pull requests:** @@ -270,6 +287,7 @@ - ENH: Rework conda mirror to not use conda-index [\#14](https://github.com/maxpoint/conda-mirror/pull/14) ([ericdill](https://github.com/ericdill)) ## [0.3.0](https://github.com/maxpoint/conda-mirror/tree/0.3.0) (2016-12-07) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.2.3...0.3.0) **Merged pull requests:** @@ -277,6 +295,7 @@ - ENH: Copy index from anaconda [\#12](https://github.com/maxpoint/conda-mirror/pull/12) ([ericdill](https://github.com/ericdill)) ## [0.2.3](https://github.com/maxpoint/conda-mirror/tree/0.2.3) (2016-11-09) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.2.2...0.2.3) **Merged pull requests:** @@ -284,6 +303,7 @@ - Disable tqdm by default [\#11](https://github.com/maxpoint/conda-mirror/pull/11) ([ericdill](https://github.com/ericdill)) ## [0.2.2](https://github.com/maxpoint/conda-mirror/tree/0.2.2) (2016-11-09) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.2.1...0.2.2) **Merged pull requests:** @@ -291,6 +311,7 @@ - Handle when the license is a literal value of None [\#10](https://github.com/maxpoint/conda-mirror/pull/10) ([ericdill](https://github.com/ericdill)) ## [0.2.1](https://github.com/maxpoint/conda-mirror/tree/0.2.1) (2016-11-08) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.2.0...0.2.1) **Merged pull requests:** @@ -298,6 +319,7 @@ - Remove bad packages instead of failing [\#9](https://github.com/maxpoint/conda-mirror/pull/9) ([ericdill](https://github.com/ericdill)) ## [0.2.0](https://github.com/maxpoint/conda-mirror/tree/0.2.0) (2016-11-04) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.1.0...0.2.0) **Merged pull requests:** @@ -305,6 +327,7 @@ - Call conda mirror directly [\#8](https://github.com/maxpoint/conda-mirror/pull/8) ([ericdill](https://github.com/ericdill)) ## [0.1.0](https://github.com/maxpoint/conda-mirror/tree/0.1.0) (2016-11-02) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.0.8...0.1.0) **Merged pull requests:** @@ -312,6 +335,7 @@ - ENH: Add license checks [\#7](https://github.com/maxpoint/conda-mirror/pull/7) ([ericdill](https://github.com/ericdill)) ## [0.0.8](https://github.com/maxpoint/conda-mirror/tree/0.0.8) (2016-10-24) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.0.7...0.0.8) **Merged pull requests:** @@ -319,20 +343,23 @@ - WIP: Intermittently run conda index when mirroring lots of packages [\#6](https://github.com/maxpoint/conda-mirror/pull/6) ([ericdill](https://github.com/ericdill)) ## [0.0.7](https://github.com/maxpoint/conda-mirror/tree/0.0.7) (2016-10-24) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.0.6...0.0.7) **Merged pull requests:** -- WIP: entry\_points... [\#5](https://github.com/maxpoint/conda-mirror/pull/5) ([ericdill](https://github.com/ericdill)) +- WIP: entry_points... [\#5](https://github.com/maxpoint/conda-mirror/pull/5) ([ericdill](https://github.com/ericdill)) ## [0.0.6](https://github.com/maxpoint/conda-mirror/tree/0.0.6) (2016-10-24) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.0.4...0.0.6) **Merged pull requests:** -- WIP: add setup.py directive to make conda\_mirror.py a script [\#4](https://github.com/maxpoint/conda-mirror/pull/4) ([ericdill](https://github.com/ericdill)) +- WIP: add setup.py directive to make conda_mirror.py a script [\#4](https://github.com/maxpoint/conda-mirror/pull/4) ([ericdill](https://github.com/ericdill)) ## [0.0.4](https://github.com/maxpoint/conda-mirror/tree/0.0.4) (2016-10-24) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.0.3...0.0.4) **Merged pull requests:** @@ -340,17 +367,18 @@ - WIP: Update travis yaml [\#3](https://github.com/maxpoint/conda-mirror/pull/3) ([ericdill](https://github.com/ericdill)) ## [0.0.3](https://github.com/maxpoint/conda-mirror/tree/0.0.3) (2016-10-24) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.0.2...0.0.3) ## [0.0.2](https://github.com/maxpoint/conda-mirror/tree/0.0.2) (2016-10-24) + [Full Changelog](https://github.com/maxpoint/conda-mirror/compare/0.0.1...0.0.2) ## [0.0.1](https://github.com/maxpoint/conda-mirror/tree/0.0.1) (2016-10-20) + **Merged pull requests:** - test the cli too [\#2](https://github.com/maxpoint/conda-mirror/pull/2) ([ericdill](https://github.com/ericdill)) - Remove unused validator [\#1](https://github.com/maxpoint/conda-mirror/pull/1) ([ericdill](https://github.com/ericdill)) - - -\* *This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)* +\* _This Change Log was automatically generated by [github_changelog_generator](https://github.com/skywinder/Github-Changelog-Generator)_ diff --git a/README.md b/README.md index c9261b8..d2e88f9 100644 --- a/README.md +++ b/README.md @@ -101,12 +101,12 @@ example-conf.yaml: ```yaml blacklist: - - license: "*agpl*" - - license: None - - license: "" + - license: "*agpl*" + - license: None + - license: "" whitelist: - - name: system + - name: system ``` `blacklist` removes package(s) that match the condition(s) listed from the @@ -152,42 +152,46 @@ See implementation details in the `conda_mirror:match` function for more information. #### Common usage patterns + ##### Mirror **only** one specific package + If you wanted to match exactly the botocore package listed above with your config, then you could use the following configuration to first blacklist **all** packages and then include just the botocore packages: ```yaml blacklist: - - name: "*" + - name: "*" whitelist: - - name: botocore - version: 1.4.10 - build: py34_0 + - name: botocore + version: 1.4.10 + build: py34_0 ``` you can use standard conda package version specifiers to filter a range of versions: ```yaml blacklist: - - name: "*" + - name: "*" whitelist: - - name: botocore - version: ">=1.4.10,<1.5" + - name: botocore + version: ">=1.4.10,<1.5" ``` ##### Mirror everything but agpl licenses + ```yaml blacklist: - - license: "*agpl*" + - license: "*agpl*" ``` ##### Mirror only python 3 packages + ```yaml blacklist: - - name: "*" + - name: "*" whitelist: - - build: "*py3*" + - build: "*py3*" ``` ##### Mirror specified packages and their dependencies @@ -198,7 +202,7 @@ dependencies (and likewise for dependencies of those packages). ```yaml blacklist: - - name: "*" + - name: "*" whitelist: - name: botocore version: ">=1.4.10" @@ -259,7 +263,6 @@ TOTAL 239 20 92% After a new contributor makes a pull-request that is approved, we will reach out and invite you to be a maintainer of the project. - ## Releasing To release you need three things diff --git a/conda_mirror/_version.py b/conda_mirror/_version.py index a6454e4..b4524dc 100644 --- a/conda_mirror/_version.py +++ b/conda_mirror/_version.py @@ -84,7 +84,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env= stderr=(subprocess.PIPE if hide_stderr else None), ) break - except EnvironmentError: + except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue @@ -94,7 +94,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env= return None, None else: if verbose: - print("unable to find command, tried %s" % (commands,)) + print(f"unable to find command, tried {commands}") return None, None stdout = p.communicate()[0].strip() if sys.version_info[0] >= 3: @@ -147,7 +147,7 @@ def git_get_keywords(versionfile_abs): # _version.py. keywords = {} try: - f = open(versionfile_abs, "r") + f = open(versionfile_abs) for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) @@ -162,7 +162,7 @@ def git_get_keywords(versionfile_abs): if mo: keywords["date"] = mo.group(1) f.close() - except EnvironmentError: + except OSError: pass return keywords @@ -186,11 +186,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -199,7 +199,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) + tags = {r for r in refs if re.search(r"\d", r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -302,7 +302,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + pieces["error"] = "tag '{}' doesn't start with prefix '{}'".format( full_tag, tag_prefix, ) diff --git a/conda_mirror/conda_mirror.py b/conda_mirror/conda_mirror.py index 73ec033..320a311 100644 --- a/conda_mirror/conda_mirror.py +++ b/conda_mirror/conda_mirror.py @@ -8,19 +8,18 @@ import multiprocessing import os import pdb +import random import re import shutil import sys import tarfile import tempfile import time -import random from pprint import pformat from typing import Any, Callable, Dict, Set, Union import requests import yaml - from tqdm import tqdm try: @@ -200,7 +199,7 @@ def _restore_required_dependencies( # TODO - support platform-specific + noarch - already_required = set(all_packages.get(r, {}).get("name") for r in required) + already_required = {all_packages.get(r, {}).get("name") for r in required} final_excluded: Set[str] = set(excluded) @@ -454,7 +453,7 @@ def _parse_and_format_args(): config_dict = {} if args.config: logger.info("Loading config from %s", args.config) - with open(args.config, "r") as f: + with open(args.config) as f: config_dict = yaml.safe_load(f) logger.info("config: %s", config_dict) @@ -500,7 +499,7 @@ def pdb_hook(exctype, value, traceback): if len(url) > 1: url = ":".join(url) else: - url = "{}:{}".format(scheme, url[0]) + url = f"{scheme}:{url[0]}" proxies = {scheme: url} return { "upstream_channel": args.upstream_channel, @@ -544,7 +543,7 @@ def _remove_package(pkg_path, reason): reason : str The reason why the package is being removed """ - msg = "Removing: %s. Reason: %s" % (pkg_path, reason) + msg = f"Removing: {pkg_path}. Reason: {reason}" if logger: logger.warning(msg) else: @@ -771,9 +770,7 @@ def _download_backoff_retry( break except Exception: if c < max_retries: - logger.debug( - "downloading failed, retrying {0}/{1}".format(c, max_retries) - ) + logger.debug(f"downloading failed, retrying {c}/{max_retries}") time.sleep(delay * random.randint(0, two_c - 1)) else: raise @@ -906,7 +903,7 @@ def _validate_or_remove_package(args): return _remove_package(package_path, reason=reason) # validate the integrity of the package, the size of the package and # its hashes - log_msg = "Validating {:4d} of {:4d}: {}.".format(num + 1, num_packages, package) + log_msg = f"Validating {num + 1:4d} of {num_packages:4d}: {package}." if logger: logger.info(log_msg) else: diff --git a/conda_mirror/diff_tar.py b/conda_mirror/diff_tar.py index 5965299..088bb76 100644 --- a/conda_mirror/diff_tar.py +++ b/conda_mirror/diff_tar.py @@ -4,14 +4,13 @@ tarball can be used to update a copy of the mirror on a remote (air-gapped) system, without having to copy the entire conda repository. """ +import hashlib +import json import os import sys -import json -import hashlib import tarfile from os.path import abspath, isdir, join, relpath - DEFAULT_REFERENCE_PATH = "./reference.json" DEFAULT_UPDATE_PATH = "./update.tar" diff --git a/conda_mirror/versionspec.py b/conda_mirror/versionspec.py index 58b10ef..37c3cad 100644 --- a/conda_mirror/versionspec.py +++ b/conda_mirror/versionspec.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # Copyright (C) 2012 Anaconda, Inc # SPDX-License-Identifier: BSD-3-Clause # @@ -9,11 +8,10 @@ # not part of the officially supported public API nor is it available in a package # that can be installed safely outside of the base environment. -from logging import getLogger import operator as op import re - from itertools import zip_longest +from logging import getLogger log = getLogger(__name__) @@ -22,7 +20,7 @@ version_cache = {} -class excepts(object): +class excepts: def __init__(self, exc, func, handler=lambda exc: None): self.exc = exc self.func = func @@ -48,12 +46,10 @@ def __call__(cls, arg): try: return cls._cache_[arg] except KeyError: - val = cls._cache_[arg] = super(SingleStrArgCachingType, cls).__call__( - arg - ) + val = cls._cache_[arg] = super().__call__(arg) return val else: - return super(SingleStrArgCachingType, cls).__call__(arg) + return super().__call__(arg) class VersionOrder(metaclass=SingleStrArgCachingType): @@ -252,7 +248,7 @@ def __str__(self): return self.norm_version def __repr__(self): - return '%s("%s")' % (self.__class__.__name__, self) + return f'{self.__class__.__name__}("{self}")' def _eq(self, t1, t2): for v1, v2 in zip_longest(t1, t2, fillvalue=[]): @@ -450,7 +446,7 @@ def compatible_release_operator(x, y): OPERATOR_START = frozenset(("=", "<", ">", "!", "~")) -class BaseSpec(object): +class BaseSpec: def __init__(self, spec_str, matcher, is_exact): self.spec_str = spec_str self._is_exact = is_exact @@ -480,7 +476,7 @@ def __str__(self): return self.spec def __repr__(self): - return "%s('%s')" % (self.__class__.__name__, self.spec) + return f"{self.__class__.__name__}('{self.spec}')" @property def raw_value(self): @@ -519,7 +515,7 @@ class VersionSpec( def __init__(self, vspec): vspec_str, matcher, is_exact = self.get_matcher(vspec) - super(VersionSpec, self).__init__(vspec_str, matcher, is_exact) + super().__init__(vspec_str, matcher, is_exact) def get_matcher(self, vspec): if isinstance(vspec, str) and regex_split_re.match(vspec): @@ -632,7 +628,7 @@ class BuildNumberMatch( def __init__(self, vspec): vspec_str, matcher, is_exact = self.get_matcher(vspec) - super(BuildNumberMatch, self).__init__(vspec_str, matcher, is_exact) + super().__init__(vspec_str, matcher, is_exact) def get_matcher(self, vspec): try: diff --git a/diff-tar.md b/diff-tar.md index a8b899c..9eef9b7 100644 --- a/diff-tar.md +++ b/diff-tar.md @@ -1,13 +1,12 @@ -Create differential tarballs -============================ +# Create differential tarballs This tools allows you to create differential tarballs of a (usually -mirrored) conda repository. The resulting tarball can be used to update +mirrored) conda repository. The resulting tarball can be used to update a copy of the mirror on a remote (air-gapped) system, without having to copy the entire conda repository. -Usage: ------- +## Usage: + Running `conda-diff-tar --help` will show the following output: ``` @@ -38,20 +37,17 @@ optional arguments: --version print version and exit ``` -Example workflow: ------------------ - - 1. we assume that the remote and local repository are in sync - 2. create a `reference.json` file of the local repository with the `--reference` flag - 3. update the local repository using `conda-mirror` or some other tools - 4. create the "differential" tarball with the `--create` flag - 5. move the differential tarball to the remote machine, and unpack it - 6. now that the remote repository is up-to-date, we should create a new - `reference.json` on the local machine. That is, repeat step 2 +## Example workflow: +1. we assume that the remote and local repository are in sync +2. create a `reference.json` file of the local repository with the `--reference` flag +3. update the local repository using `conda-mirror` or some other tools +4. create the "differential" tarball with the `--create` flag +5. move the differential tarball to the remote machine, and unpack it +6. now that the remote repository is up-to-date, we should create a new + `reference.json` on the local machine. That is, repeat step 2 -Notes: ------- +## Notes: The file `reference.json` (or whatever you named it) is a collection of all `repodata.json` files (`linux-64`, `win-32`, `noarch`, etc.) in the local repository. @@ -59,7 +55,7 @@ It is created in order to compare a future state of the repository to the state of the repository when `reference.json` was created. The differential tarball contains files which either have been updated (such -as `repodata.json`) or new files (new conda packages). It is meant to be +as `repodata.json`) or new files (new conda packages). It is meant to be unpacked on top of the existing mirror on the remote machine by: cd @@ -67,8 +63,7 @@ unpacked on top of the existing mirror on the remote machine by: # or y using tar's -C option from any directory tar xf update.tar -C -Example: --------- +## Example: In this example we assume that a conda mirror is located in `./repo`. Create `reference.json`: @@ -76,7 +71,7 @@ Create `reference.json`: conda-diff-tar --reference ./repo Show the files in respect to the latest reference point file (which would be -included in the differential tarball). Since we just created the reference +included in the differential tarball). Since we just created the reference file, we don't expect any output: conda-diff-tar --show ./repo diff --git a/news/TEMPLATE.md b/news/TEMPLATE.md index 22efbfe..aaedebf 100644 --- a/news/TEMPLATE.md +++ b/news/TEMPLATE.md @@ -1,15 +1,15 @@ **Implemented enhancements:** -* +- **Fixed bugs:** -* +- **Closed issues:** -* +- **Merged pull requests:** -* +- diff --git a/pyproject.toml b/pyproject.toml index 993698f..78d9e0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,28 @@ [tool.black] -line-length = 88 -include = '\.pyi?$' \ No newline at end of file +target-version = ['py38'] +exclude = ''' +/( + \.eggs + | \.git + | \.venv + | build + | dist +)/ +''' + +[tool.isort] +profile = "black" +multi_line_output = 3 +include_trailing_comma = true +ensure_newline_before_comments = true +line_length = 88 +known_first_party = "spox" +skip_glob = '\.eggs/*,\.git/*,\.venv/*,build/*,dist/*' +default_section = 'THIRDPARTY' + +[tool.mypy] +python_version = "3.8" +ignore_missing_imports = true +no_implicit_optional = true +check_untyped_defs = true + diff --git a/run_tests.py b/run_tests.py index 7633fb7..a68c174 100755 --- a/run_tests.py +++ b/run_tests.py @@ -1,5 +1,6 @@ #!/usr/bin/env python import sys + import pytest if __name__ == "__main__": @@ -7,7 +8,7 @@ args = ["-v"] # show the message output for skipped and expected failure tests args.append("-rxs") - print("sys.argv={}".format(sys.argv)) + print(f"sys.argv={sys.argv}") args.extend(sys.argv[1:]) # call pytest and exit with the return code from pytest so that # travis will fail correctly if tests fail diff --git a/setup.py b/setup.py index 188f602..91aa825 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ from setuptools import find_packages, setup + import versioneer try: diff --git a/test/test_conda_mirror.py b/test/test_conda_mirror.py index 72bb0e0..0399737 100644 --- a/test/test_conda_mirror.py +++ b/test/test_conda_mirror.py @@ -4,13 +4,11 @@ import json import os import sys - from os.path import join -from conda_mirror import conda_mirror - import pytest +from conda_mirror import conda_mirror anaconda_channel = "https://repo.continuum.io/pkgs/free" @@ -27,7 +25,7 @@ def test_match(repodata): """Unit test for internal _match function.""" repodata_info, repodata_packages = repodata["conda-forge"] matched = conda_mirror._match(repodata_packages, {"name": "jupyter"}) - assert set([v["name"] for v in matched.values()]) == set(["jupyter"]) + assert {v["name"] for v in matched.values()} == {"jupyter"} matched = conda_mirror._match(repodata_packages, {"name": "*"}) assert len(matched) == len(repodata_packages) @@ -43,10 +41,8 @@ def test_match(repodata): def test_restore_required_dependencies(repodata): """Unit tests for internal _restore_required_dependencies function.""" - from conda_mirror.conda_mirror import ( - _match, - _restore_required_dependencies as restore, - ) + from conda_mirror.conda_mirror import _match + from conda_mirror.conda_mirror import _restore_required_dependencies as restore _, all_packages = repodata["conda-forge"] excluded = set(all_packages) @@ -61,7 +57,7 @@ def test_restore_required_dependencies(repodata): required = set(list(conda_packages)[:1]) # just take the first match excluded2 = restore(all_packages, excluded - required, required) reincluded = excluded - excluded2 - reincluded_names = set(all_packages.get(r).get("name") for r in reincluded) + reincluded_names = {all_packages.get(r).get("name") for r in reincluded} assert len(excluded) > len(reincluded) > 0 assert "yaml" in reincluded_names @@ -145,7 +141,7 @@ def test_cli(tmpdir, channel, platform, repodata, num_threads): assert f in os.listdir(os.path.join(f2.strpath, platform)) # make sure that the repodata contains less than upstream since we prune it - with open(os.path.join(f2.strpath, platform, "repodata.json"), "r") as f: + with open(os.path.join(f2.strpath, platform, "repodata.json")) as f: disk_repodata = json.load(f) disk_info = disk_repodata.get("info", {}) assert len(disk_info) == len(info) @@ -163,7 +159,7 @@ def _write_bad_package(channel_dir, platform_name, pkg_name): if not os.path.exists(target_dir): os.makedirs(target_dir) with bz2.BZ2File(os.path.join(target_dir, pkg_name), "wb") as f: - f.write("This is a fake package".encode()) + f.write(b"This is a fake package") def test_main(tmpdir, repodata): diff --git a/test/test_diff_tar.py b/test/test_diff_tar.py index 637cd14..c774f81 100644 --- a/test/test_diff_tar.py +++ b/test/test_diff_tar.py @@ -1,16 +1,15 @@ -import os -import sys import json +import os +import pathlib import shutil +import sys import tempfile from os.path import isfile, join -import pathlib import pytest import conda_mirror.diff_tar as dt - EMPTY_MD5 = "d41d8cd98f00b204e9800998ecf8427e" @@ -152,8 +151,8 @@ def test_cli_reference_outfile(tmpdir): assert isfile(dt.DEFAULT_REFERENCE_PATH) run_with_args(["--reference", "--outfile", target_path, dt.mirror_dir]) assert isfile(target_path) - with open(dt.DEFAULT_REFERENCE_PATH, "r") as ref1: - with open(target_path, "r") as ref2: + with open(dt.DEFAULT_REFERENCE_PATH) as ref1: + with open(target_path) as ref2: assert ref1.readlines() == ref2.readlines() diff --git a/versioneer.py b/versioneer.py index a5d4360..335e72d 100644 --- a/versioneer.py +++ b/versioneer.py @@ -275,7 +275,6 @@ """ -from __future__ import print_function try: import configparser @@ -344,7 +343,7 @@ def get_config_from_root(root): # the top of versioneer.py for instructions on writing your setup.cfg . setup_cfg = os.path.join(root, "setup.cfg") parser = configparser.SafeConfigParser() - with open(setup_cfg, "r") as f: + with open(setup_cfg) as f: parser.readfp(f) VCS = parser.get("versioneer", "VCS") # mandatory @@ -404,7 +403,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env= stderr=(subprocess.PIPE if hide_stderr else None), ) break - except EnvironmentError: + except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue @@ -414,7 +413,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env= return None, None else: if verbose: - print("unable to find command, tried %s" % (commands,)) + print(f"unable to find command, tried {commands}") return None, None stdout = p.communicate()[0].strip() if sys.version_info[0] >= 3: @@ -429,7 +428,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env= LONG_VERSION_PY[ "git" -] = ''' +] = r''' # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -961,7 +960,7 @@ def git_get_keywords(versionfile_abs): # _version.py. keywords = {} try: - f = open(versionfile_abs, "r") + f = open(versionfile_abs) for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) @@ -976,7 +975,7 @@ def git_get_keywords(versionfile_abs): if mo: keywords["date"] = mo.group(1) f.close() - except EnvironmentError: + except OSError: pass return keywords @@ -1000,11 +999,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -1013,7 +1012,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r"\d", r)]) + tags = {r for r in refs if re.search(r"\d", r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -1116,7 +1115,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + pieces["error"] = "tag '{}' doesn't start with prefix '{}'".format( full_tag, tag_prefix, ) @@ -1166,13 +1165,13 @@ def do_vcs_install(manifest_in, versionfile_source, ipy): files.append(versioneer_file) present = False try: - f = open(".gitattributes", "r") + f = open(".gitattributes") for line in f.readlines(): if line.strip().startswith(versionfile_source): if "export-subst" in line.strip().split()[1:]: present = True f.close() - except EnvironmentError: + except OSError: pass if not present: f = open(".gitattributes", "a+") @@ -1236,7 +1235,7 @@ def versions_from_file(filename): try: with open(filename) as f: contents = f.read() - except EnvironmentError: + except OSError: raise NotThisMethod("unable to read _version.py") mo = re.search( r"version_json = '''\n(.*)''' # END VERSION_JSON", contents, re.M | re.S @@ -1257,7 +1256,7 @@ def write_to_version_file(filename, versions): with open(filename, "w") as f: f.write(SHORT_VERSION_PY % contents) - print("set %s to '%s'" % (filename, versions["version"])) + print("set {} to '{}'".format(filename, versions["version"])) def plus_or_dot(pieces): @@ -1482,7 +1481,7 @@ def get_versions(verbose=False): try: ver = versions_from_file(versionfile_abs) if verbose: - print("got version from file %s %s" % (versionfile_abs, ver)) + print(f"got version from file {versionfile_abs} {ver}") return ver except NotThisMethod: pass @@ -1755,11 +1754,7 @@ def do_setup(): root = get_root() try: cfg = get_config_from_root(root) - except ( - EnvironmentError, - configparser.NoSectionError, - configparser.NoOptionError, - ) as e: + except (OSError, configparser.NoSectionError, configparser.NoOptionError) as e: if isinstance(e, (EnvironmentError, configparser.NoSectionError)): print("Adding sample versioneer config to setup.cfg", file=sys.stderr) with open(os.path.join(root, "setup.cfg"), "a") as f: @@ -1784,9 +1779,9 @@ def do_setup(): ipy = os.path.join(os.path.dirname(cfg.versionfile_source), "__init__.py") if os.path.exists(ipy): try: - with open(ipy, "r") as f: + with open(ipy) as f: old = f.read() - except EnvironmentError: + except OSError: old = "" if INIT_PY_SNIPPET not in old: print(" appending to %s" % ipy) @@ -1805,12 +1800,12 @@ def do_setup(): manifest_in = os.path.join(root, "MANIFEST.in") simple_includes = set() try: - with open(manifest_in, "r") as f: + with open(manifest_in) as f: for line in f: if line.startswith("include "): for include in line.split()[1:]: simple_includes.add(include) - except EnvironmentError: + except OSError: pass # That doesn't cover everything MANIFEST.in can do # (http://docs.python.org/2/distutils/sourcedist.html#commands), so @@ -1844,7 +1839,7 @@ def scan_setup_py(): found = set() setters = False errors = 0 - with open("setup.py", "r") as f: + with open("setup.py") as f: for line in f.readlines(): if "import versioneer" in line: found.add("import")