diff --git a/conda_mirror/conda_mirror.py b/conda_mirror/conda_mirror.py index bb26541..b5bfc9b 100644 --- a/conda_mirror/conda_mirror.py +++ b/conda_mirror/conda_mirror.py @@ -14,6 +14,7 @@ import time import random from pprint import pformat +from typing import List, Union import requests import yaml @@ -149,8 +150,11 @@ def _make_arg_parser(): ) ap.add_argument( "--platform", + action="append", + default=[], help=( - f"The OS platform(s) to mirror. e.g.: {', '.join(DEFAULT_PLATFORMS)}" + f"The OS platform(s) to mirror. e.g.: {', '.join(DEFAULT_PLATFORMS)} " + "May be specified multiple times." ), ) ap.add_argument( @@ -348,6 +352,7 @@ def pdb_hook(exctype, value, traceback): else: url = "{}:{}".format(scheme, url[0]) proxies = {scheme: url} + return { "upstream_channel": args.upstream_channel, "target_directory": args.target_directory, @@ -748,7 +753,7 @@ def main( upstream_channel, target_directory, temp_directory, - platform, + platform: Union[str, List[str]], blacklist=None, whitelist=None, num_threads=1, @@ -777,8 +782,8 @@ def main( The path on disk to an existing and writable directory to temporarily store the packages before moving them to the target_directory to apply checks - platform : str - The platform that you wish to mirror for. Common options are + platform : Union[str,List[str]] + The platforms that you wish to mirror. Common options are 'linux-64', 'osx-64', 'win-64, 'win-32' and 'noarch'. Any platform is valid as long as the url resolves. blacklist : iterable of tuples, optional @@ -869,6 +874,55 @@ def main( "blacklisted": set(), "to-mirror": set(), } + + _platforms = [platform] if isinstance(platform, str) else platform + + for platform in tqdm( + _platforms, + desc="platforms", + leave=False, + disable=not show_progress or len(_platforms) < 2, + ): + _download_platform( + summary, + upstream_channel, + target_directory, + temp_directory, + platform, + blacklist=blacklist, + whitelist=whitelist, + num_threads=num_threads, + dry_run=dry_run, + no_validate_target=no_validate_target, + minimum_free_space=minimum_free_space, + proxies=proxies, + ssl_verify=ssl_verify, + chunk_size=chunk_size, + max_retries=max_retries, + show_progress=show_progress, + ) + + return summary + + +def _download_platform( + summary, + upstream_channel, + target_directory, + temp_directory, + platform: str, + blacklist=None, + whitelist=None, + num_threads=1, + dry_run=False, + no_validate_target=False, + minimum_free_space=0, + proxies=None, + ssl_verify=None, + chunk_size: int = DEFAULT_CHUNK_SIZE, + max_retries=100, + show_progress=True, +): # Implementation: if not os.path.exists(os.path.join(target_directory, platform)): os.makedirs(os.path.join(target_directory, platform)) @@ -944,7 +998,7 @@ def main( summary["to-mirror"].update(to_mirror) if dry_run: logger.info("Dry run complete. Exiting") - return summary + return # 6. for each download: # a. download to temp file @@ -959,9 +1013,9 @@ def main( logger.info("downloading to the tempdir %s", download_dir) for package_name in tqdm( sorted(to_mirror), - desc="progress", + desc=platform, unit="package", - leave=True, + leave=False, disable=not show_progress, ): url = download_url.format( @@ -1049,8 +1103,6 @@ def main( noarch_repodata = {"info": {}, "packages": {}} _write_repodata(noarch_path, noarch_repodata) - return summary - def _write_repodata(package_dir, repodata_dict): data = json.dumps(repodata_dict, indent=2, sort_keys=True)