Merge multi-platform download support (conda-incubator#32)

conda_mirror/conda_mirror.py

@@ -14,6 +14,7 @@
 import time
 import random
 from pprint import pformat
+from typing import List, Union
 
 import requests
 import yaml
@@ -149,8 +150,11 @@ def _make_arg_parser():
     )
     ap.add_argument(
         "--platform",
+        action="append",
+        default=[],
         help=(
-            f"The OS platform(s) to mirror. e.g.: {', '.join(DEFAULT_PLATFORMS)}"
+            f"The OS platform(s) to mirror. e.g.: {', '.join(DEFAULT_PLATFORMS)} "
+            "May be specified multiple times."
         ),
     )
     ap.add_argument(
@@ -348,6 +352,7 @@ def pdb_hook(exctype, value, traceback):
         else:
             url = "{}:{}".format(scheme, url[0])
         proxies = {scheme: url}
+
     return {
         "upstream_channel": args.upstream_channel,
         "target_directory": args.target_directory,
@@ -748,7 +753,7 @@ def main(
     upstream_channel,
     target_directory,
     temp_directory,
-    platform,
+    platform: Union[str, List[str]],
     blacklist=None,
     whitelist=None,
     num_threads=1,
@@ -777,8 +782,8 @@ def main(
         The path on disk to an existing and writable directory to temporarily
         store the packages before moving them to the target_directory to
         apply checks
-    platform : str
-        The platform that you wish to mirror for. Common options are
+    platform : Union[str,List[str]]
+        The platforms that you wish to mirror. Common options are
         'linux-64', 'osx-64', 'win-64, 'win-32' and 'noarch'. Any platform is valid as
         long as the url resolves.
     blacklist : iterable of tuples, optional
@@ -869,6 +874,55 @@ def main(
         "blacklisted": set(),
         "to-mirror": set(),
     }
+
+    _platforms = [platform] if isinstance(platform, str) else platform
+
+    for platform in tqdm(
+        _platforms,
+        desc="platforms",
+        leave=False,
+        disable=not show_progress or len(_platforms) < 2,
+    ):
+        _download_platform(
+            summary,
+            upstream_channel,
+            target_directory,
+            temp_directory,
+            platform,
+            blacklist=blacklist,
+            whitelist=whitelist,
+            num_threads=num_threads,
+            dry_run=dry_run,
+            no_validate_target=no_validate_target,
+            minimum_free_space=minimum_free_space,
+            proxies=proxies,
+            ssl_verify=ssl_verify,
+            chunk_size=chunk_size,
+            max_retries=max_retries,
+            show_progress=show_progress,
+        )
+
+    return summary
+
+
+def _download_platform(
+    summary,
+    upstream_channel,
+    target_directory,
+    temp_directory,
+    platform: str,
+    blacklist=None,
+    whitelist=None,
+    num_threads=1,
+    dry_run=False,
+    no_validate_target=False,
+    minimum_free_space=0,
+    proxies=None,
+    ssl_verify=None,
+    chunk_size: int = DEFAULT_CHUNK_SIZE,
+    max_retries=100,
+    show_progress=True,
+):
     # Implementation:
     if not os.path.exists(os.path.join(target_directory, platform)):
         os.makedirs(os.path.join(target_directory, platform))
@@ -944,7 +998,7 @@ def main(
     summary["to-mirror"].update(to_mirror)
     if dry_run:
         logger.info("Dry run complete. Exiting")
-        return summary
+        return
 
     # 6. for each download:
     # a. download to temp file
@@ -959,9 +1013,9 @@ def main(
         logger.info("downloading to the tempdir %s", download_dir)
         for package_name in tqdm(
             sorted(to_mirror),
-            desc="progress",
+            desc=platform,
             unit="package",
-            leave=True,
+            leave=False,
             disable=not show_progress,
         ):
             url = download_url.format(
@@ -1049,8 +1103,6 @@ def main(
         noarch_repodata = {"info": {}, "packages": {}}
         _write_repodata(noarch_path, noarch_repodata)
 
-    return summary
-
 
 def _write_repodata(package_dir, repodata_dict):
     data = json.dumps(repodata_dict, indent=2, sort_keys=True)