From a2be9131247379765ee51902b413b8e87db5f0d6 Mon Sep 17 00:00:00 2001 From: martinbrose <13284268+martinbrose@users.noreply.github.com> Date: Tue, 22 Aug 2023 18:19:23 +0100 Subject: [PATCH 01/16] Implemented CLI upload functionality --- .../commands/huggingface_cli.py | 2 + src/huggingface_hub/commands/upload.py | 165 ++++++++++++++++++ 2 files changed, 167 insertions(+) create mode 100644 src/huggingface_hub/commands/upload.py diff --git a/src/huggingface_hub/commands/huggingface_cli.py b/src/huggingface_hub/commands/huggingface_cli.py index d5d4bbc79b..9e7c4afb6a 100644 --- a/src/huggingface_hub/commands/huggingface_cli.py +++ b/src/huggingface_hub/commands/huggingface_cli.py @@ -20,6 +20,7 @@ from huggingface_hub.commands.lfs import LfsCommands from huggingface_hub.commands.scan_cache import ScanCacheCommand from huggingface_hub.commands.user import UserCommands +from huggingface_hub.commands.upload import UploadCommand def main(): @@ -32,6 +33,7 @@ def main(): LfsCommands.register_subcommand(commands_parser) ScanCacheCommand.register_subcommand(commands_parser) DeleteCacheCommand.register_subcommand(commands_parser) + UploadCommand.register_subcommand(commands_parser) # Let's go args = parser.parse_args() diff --git a/src/huggingface_hub/commands/upload.py b/src/huggingface_hub/commands/upload.py new file mode 100644 index 0000000000..d8ace50b98 --- /dev/null +++ b/src/huggingface_hub/commands/upload.py @@ -0,0 +1,165 @@ +"""Contains command to upload a repo or file with the CLI. +Usage: + huggingface-cli upload repo_id + huggingface-cli upload repo_id [path] [path-in-repo] +""" +import os +from argparse import _SubParsersAction + +from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.constants import ( + REPO_TYPES, +) +from huggingface_hub.hf_api import HfApi + +from ..utils import HfFolder + + +class UploadCommand(BaseHuggingfaceCLICommand): + def __init__(self, args): + self.args = args + self._api = HfApi() + + @staticmethod + def register_subcommand(parser: _SubParsersAction): + upload_parser = parser.add_parser( + "upload", + help="Upload a repo or a repo file to huggingface.co", + ) + + upload_parser.add_argument( + "repo_id", + type=str, + help="The ID of the repo to upload.", + ) + upload_parser.add_argument( + "path", + nargs="?", + help="Local path. (optional)", + ) + upload_parser.add_argument( + "path_in_repo", + nargs="?", + help="Path in repo. (optional)", + ) + upload_parser.add_argument( + "--token", + type=str, + help="Token generated from https://huggingface.co/settings/tokens", + ) + upload_parser.add_argument( + "--type", + type=str, + help=( + "The type of the repo to upload. Can be one of:" + f" {', '.join([item for item in REPO_TYPES if isinstance(item, str)])}" + ), + ) + upload_parser.add_argument( + "--revision", + type=str, + help="The revision of the repo to upload.", + ) + upload_parser.add_argument( + "--allow-patterns", + nargs="+", + type=str, + help="Glob patterns to match files to upload.", + ) + upload_parser.add_argument( + "--ignore-patterns", + nargs="+", + type=str, + help="Glob patterns to exclude from files to upload.", + ) + upload_parser.add_argument( + "--delete-patterns", + nargs="+", + type=str, + help="Glob patterns for file to be deleted from the repo while committing.", + ) + upload_parser.add_argument( + "--commit-message", + type=str, + help="The summary / title / first line of the generated commit.", + ) + upload_parser.add_argument( + "--commit-description", + type=str, + help="The description of the generated commit.", + ) + upload_parser.add_argument( + "--create-pr", + action="store_true", + help="Whether to create a PR.", + ) + upload_parser.add_argument( + "--every", + action="store_true", + help="Run a CommitScheduler instead of a single commit.", + ) + + upload_parser.set_defaults(func=UploadCommand) + + def run(self): + if self.args.token: + self.token = self.args.token + HfFolder.save_token(self.args.token) + else: + self.token = HfFolder.get_token() + + if self.token is None: + raise ValueError("Not logged in or token is not provided. Consider running `huggingface-cli login`.") + + if self.args.type not in REPO_TYPES: + raise ValueError( + f"Invalid repo --type: {self.args.type}. " + f"Can be one of: {', '.join([item for item in REPO_TYPES if isinstance(item, str)])}." + ) + + self.path = "." if self.args.path is None else self.args.path + + self.path_in_repo = ( + self.args.path_in_repo if self.args.path_in_repo + else (os.path.relpath(self.path).replace("\\", "/") if self.path != "." else "/") + ) + + # File or Folder based uploading + if os.path.isfile(self.path): + if self.args.allow_patterns or self.args.ignore_patterns: + raise ValueError("--allow-patterns / --ignore-patterns cannot be used with a file path.") + + self._api.upload_file( + path_or_fileobj=self.path, + path_in_repo=self.path_in_repo, + repo_id=self.args.repo_id, + token=self.token, + repo_type=self.args.type, + revision=self.args.revision, + commit_message=self.args.commit_message, + commit_description=self.args.commit_description, + create_pr=self.args.create_pr, + run_as_future=self.args.every, + ) + print(f"Successfully uploaded selected file to repo {self.args.repo_id}") + + elif os.path.isdir(self.path): + self._api.upload_folder( + folder_path=self.path, + path_in_repo=self.path_in_repo, + repo_id=self.args.repo_id, + token=self.token, + repo_type=self.args.type, + revision=self.args.revision, + commit_message=self.args.commit_message, + commit_description=self.args.commit_description, + create_pr=self.args.create_pr, + allow_patterns=self.args.allow_patterns, + ignore_patterns=self.args.ignore_patterns, + delete_patterns=self.args.delete_patterns, + run_as_future=self.args.every, + ) + print(f"Successfully uploaded selected folder to repo {self.args.repo_id}") + + else: + raise ValueError(f"Provided PATH: {self.args.path} does not exist.") \ No newline at end of file From 8a827bba1813b5af28301a68cd66a16e2bebd8b6 Mon Sep 17 00:00:00 2001 From: martinbrose <13284268+martinbrose@users.noreply.github.com> Date: Tue, 22 Aug 2023 20:45:50 +0100 Subject: [PATCH 02/16] Addressed code quality --- src/huggingface_hub/commands/huggingface_cli.py | 2 +- src/huggingface_hub/commands/upload.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/huggingface_hub/commands/huggingface_cli.py b/src/huggingface_hub/commands/huggingface_cli.py index 9e7c4afb6a..59bdf58c19 100644 --- a/src/huggingface_hub/commands/huggingface_cli.py +++ b/src/huggingface_hub/commands/huggingface_cli.py @@ -19,8 +19,8 @@ from huggingface_hub.commands.env import EnvironmentCommand from huggingface_hub.commands.lfs import LfsCommands from huggingface_hub.commands.scan_cache import ScanCacheCommand -from huggingface_hub.commands.user import UserCommands from huggingface_hub.commands.upload import UploadCommand +from huggingface_hub.commands.user import UserCommands def main(): diff --git a/src/huggingface_hub/commands/upload.py b/src/huggingface_hub/commands/upload.py index d8ace50b98..09d4ba44b6 100644 --- a/src/huggingface_hub/commands/upload.py +++ b/src/huggingface_hub/commands/upload.py @@ -103,11 +103,11 @@ def register_subcommand(parser: _SubParsersAction): def run(self): if self.args.token: - self.token = self.args.token + self.token = self.args.token HfFolder.save_token(self.args.token) else: self.token = HfFolder.get_token() - + if self.token is None: raise ValueError("Not logged in or token is not provided. Consider running `huggingface-cli login`.") @@ -120,7 +120,8 @@ def run(self): self.path = "." if self.args.path is None else self.args.path self.path_in_repo = ( - self.args.path_in_repo if self.args.path_in_repo + self.args.path_in_repo + if self.args.path_in_repo else (os.path.relpath(self.path).replace("\\", "/") if self.path != "." else "/") ) @@ -162,4 +163,4 @@ def run(self): print(f"Successfully uploaded selected folder to repo {self.args.repo_id}") else: - raise ValueError(f"Provided PATH: {self.args.path} does not exist.") \ No newline at end of file + raise ValueError(f"Provided PATH: {self.args.path} does not exist.") From f256debfa43312d97c358c5302deea692045589e Mon Sep 17 00:00:00 2001 From: martinbrose <13284268+martinbrose@users.noreply.github.com> Date: Thu, 31 Aug 2023 21:34:10 +0100 Subject: [PATCH 03/16] Refactoring CLI --- src/huggingface_hub/commands/upload.py | 141 ++++++++++++++----------- 1 file changed, 80 insertions(+), 61 deletions(-) diff --git a/src/huggingface_hub/commands/upload.py b/src/huggingface_hub/commands/upload.py index 09d4ba44b6..fa3abc5c8c 100644 --- a/src/huggingface_hub/commands/upload.py +++ b/src/huggingface_hub/commands/upload.py @@ -1,36 +1,44 @@ +# coding=utf-8 +# Copyright 2023-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Contains command to upload a repo or file with the CLI. + Usage: huggingface-cli upload repo_id huggingface-cli upload repo_id [path] [path-in-repo] """ import os +import warnings from argparse import _SubParsersAction +from typing import List, Optional +from huggingface_hub import HfApi from huggingface_hub.commands import BaseHuggingfaceCLICommand -from huggingface_hub.constants import ( - REPO_TYPES, -) -from huggingface_hub.hf_api import HfApi - -from ..utils import HfFolder +from huggingface_hub.utils import disable_progress_bars, enable_progress_bars class UploadCommand(BaseHuggingfaceCLICommand): - def __init__(self, args): - self.args = args - self._api = HfApi() - @staticmethod def register_subcommand(parser: _SubParsersAction): upload_parser = parser.add_parser( "upload", help="Upload a repo or a repo file to huggingface.co", ) - upload_parser.add_argument( "repo_id", type=str, - help="The ID of the repo to upload.", + help="The ID of the repo to upload to (e.g. `username/repo-name`).", ) upload_parser.add_argument( "path", @@ -42,18 +50,10 @@ def register_subcommand(parser: _SubParsersAction): nargs="?", help="Path in repo. (optional)", ) - upload_parser.add_argument( - "--token", - type=str, - help="Token generated from https://huggingface.co/settings/tokens", - ) upload_parser.add_argument( "--type", type=str, - help=( - "The type of the repo to upload. Can be one of:" - f" {', '.join([item for item in REPO_TYPES if isinstance(item, str)])}" - ), + help="The type of the repo to upload (e.g. `dataset`).", ) upload_parser.add_argument( "--revision", @@ -98,69 +98,88 @@ def register_subcommand(parser: _SubParsersAction): action="store_true", help="Run a CommitScheduler instead of a single commit.", ) - + upload_parser.add_argument( + "--token", + type=str, + help="A User Access Token generated from https://huggingface.co/settings/tokens", + ) + upload_parser.add_argument( + "--quiet", + action="store_true", + help="If True, progress bars are disabled and only the path to the uploaded files is printed.", + ) upload_parser.set_defaults(func=UploadCommand) + def __init__(self, args): + self.api = HfApi(token=args.token) + self.repo_id: str = args.repo_id + self.path: str = args.path + self.path_in_repo: str = args.path_in_repo + self.type: Optional[str] = args.type + self.revision: Optional[str] = args.revision + self.allow_patterns: List[str] = args.allow_patterns + self.ignore_patterns: List[str] = args.ignore_patterns + self.delete_patterns: List[str] = args.delete_patterns + self.commit_message: Optional[str] = args.commit_message + self.commit_description: Optional[str] = args.commit_description + self.create_pr: Optional[bool] = args.create_pr + self.every: Optional[bool] = args.every + self.token: Optional[str] = args.token + self.quiet: bool = args.quiet + def run(self): - if self.args.token: - self.token = self.args.token - HfFolder.save_token(self.args.token) + if self.quiet: + disable_progress_bars() + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + print(self._upload()) # Print path to uploaded files + enable_progress_bars() else: - self.token = HfFolder.get_token() - - if self.token is None: - raise ValueError("Not logged in or token is not provided. Consider running `huggingface-cli login`.") - - if self.args.type not in REPO_TYPES: - raise ValueError( - f"Invalid repo --type: {self.args.type}. " - f"Can be one of: {', '.join([item for item in REPO_TYPES if isinstance(item, str)])}." - ) + print(self._upload()) # Print path to uploaded files - self.path = "." if self.args.path is None else self.args.path + def _upload(self) -> str: + self.path = "." if self.path is None else self.path self.path_in_repo = ( - self.args.path_in_repo - if self.args.path_in_repo + self.path_in_repo + if self.path_in_repo else (os.path.relpath(self.path).replace("\\", "/") if self.path != "." else "/") ) # File or Folder based uploading if os.path.isfile(self.path): - if self.args.allow_patterns or self.args.ignore_patterns: + if self.allow_patterns or self.ignore_patterns: raise ValueError("--allow-patterns / --ignore-patterns cannot be used with a file path.") - self._api.upload_file( + return self.api.upload_file( path_or_fileobj=self.path, path_in_repo=self.path_in_repo, - repo_id=self.args.repo_id, + repo_id=self.repo_id, token=self.token, - repo_type=self.args.type, - revision=self.args.revision, - commit_message=self.args.commit_message, - commit_description=self.args.commit_description, - create_pr=self.args.create_pr, - run_as_future=self.args.every, + repo_type=self.type, + revision=self.revision, + commit_message=self.commit_message, + commit_description=self.commit_description, + create_pr=self.create_pr, + run_as_future=self.every, ) - print(f"Successfully uploaded selected file to repo {self.args.repo_id}") elif os.path.isdir(self.path): - self._api.upload_folder( + return self.api.upload_folder( folder_path=self.path, path_in_repo=self.path_in_repo, - repo_id=self.args.repo_id, + repo_id=self.repo_id, token=self.token, - repo_type=self.args.type, - revision=self.args.revision, - commit_message=self.args.commit_message, - commit_description=self.args.commit_description, - create_pr=self.args.create_pr, - allow_patterns=self.args.allow_patterns, - ignore_patterns=self.args.ignore_patterns, - delete_patterns=self.args.delete_patterns, - run_as_future=self.args.every, + repo_type=self.type, + revision=self.revision, + commit_message=self.commit_message, + commit_description=self.commit_description, + create_pr=self.create_pr, + allow_patterns=self.allow_patterns, + ignore_patterns=self.ignore_patterns, + delete_patterns=self.delete_patterns, + run_as_future=self.every, ) - print(f"Successfully uploaded selected folder to repo {self.args.repo_id}") else: - raise ValueError(f"Provided PATH: {self.args.path} does not exist.") + raise ValueError(f"Provided PATH: {self.path} does not exist.") From a18eaf20587dac135d99f1fbb45ef21ece463738 Mon Sep 17 00:00:00 2001 From: martinbrose <13284268+martinbrose@users.noreply.github.com> Date: Sat, 2 Sep 2023 10:36:53 +0100 Subject: [PATCH 04/16] Add CLI upload test cases --- tests/test_cli.py | 82 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index bfe64de76c..926d3e2b0a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -3,6 +3,11 @@ from huggingface_hub.commands.delete_cache import DeleteCacheCommand from huggingface_hub.commands.scan_cache import ScanCacheCommand +from huggingface_hub.commands.upload import UploadCommand + +from .testing_utils import ( + DUMMY_MODEL_ID, +) class TestCLI(unittest.TestCase): @@ -50,3 +55,80 @@ def test_delete_cache_with_dir(self) -> None: args = self.parser.parse_args(["delete-cache", "--dir", "something"]) self.assertEqual(args.dir, "something") self.assertEqual(args.func, DeleteCacheCommand) + + +class TestUploadCommand(unittest.TestCase): + def setUp(self) -> None: + """ + Set up CLI as in `src/huggingface_hub/commands/huggingface_cli.py`. + """ + self.parser = ArgumentParser("huggingface-cli", usage="huggingface-cli []") + commands_parser = self.parser.add_subparsers() + UploadCommand.register_subcommand(commands_parser) + + def test_upload_basic(self) -> None: + """Test `huggingface-cli upload my-file to dummy-repo`.""" + args = self.parser.parse_args(["upload", DUMMY_MODEL_ID, "my-file"]) + self.assertEqual(args.repo_id, DUMMY_MODEL_ID) + self.assertEqual(args.path, "my-file") + self.assertEqual(args.path_in_repo, None) + self.assertEqual(args.type, None) + self.assertEqual(args.revision, None) + self.assertEqual(args.allow_patterns, None) + self.assertEqual(args.ignore_patterns, None) + self.assertEqual(args.delete_patterns, None) + self.assertEqual(args.commit_message, None) + self.assertEqual(args.commit_description, None) + self.assertEqual(args.create_pr, False) + self.assertEqual(args.every, False) + self.assertEqual(args.token, None) + self.assertEqual(args.quiet, False) + self.assertEqual(args.func, UploadCommand) + + def test_upload_with_all_options(self) -> None: + """Test `huggingface-cli upload my-file to dummy-repo with all options selected`.""" + args = self.parser.parse_args( + [ + "upload", + DUMMY_MODEL_ID, + "my-file", + "/", + "--type", + "model", + "--revision", + "v1.0.0", + "--allow-patterns", + "*.json", + "*.yaml", + "--ignore-patterns", + "*.log", + "*.txt", + "--delete-patterns", + "*.config", + "*.secret", + "--commit-message", + "My commit message", + "--commit-description", + "My commit description", + "--create-pr", + "--every", + "--token", + "my-token", + "--quiet", + ] + ) + self.assertEqual(args.repo_id, DUMMY_MODEL_ID) + self.assertEqual(args.path, "my-file") + self.assertEqual(args.path_in_repo, "/") + self.assertEqual(args.type, "model") + self.assertEqual(args.revision, "v1.0.0") + self.assertEqual(args.allow_patterns, ["*.json", "*.yaml"]) + self.assertEqual(args.ignore_patterns, ["*.log", "*.txt"]) + self.assertEqual(args.delete_patterns, ["*.config", "*.secret"]) + self.assertEqual(args.commit_message, "My commit message") + self.assertEqual(args.commit_description, "My commit description") + self.assertEqual(args.create_pr, True) + self.assertEqual(args.every, True) + self.assertEqual(args.token, "my-token") + self.assertEqual(args.quiet, True) + self.assertEqual(args.func, UploadCommand) From 1e85adefb0cba43b59d2c61a0ff016478c74be9e Mon Sep 17 00:00:00 2001 From: martinbrose <13284268+martinbrose@users.noreply.github.com> Date: Sat, 2 Sep 2023 16:34:50 +0100 Subject: [PATCH 05/16] Update argument names and function typing --- src/huggingface_hub/commands/upload.py | 42 +++++++++++++------------- tests/test_cli.py | 24 +++++++-------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/huggingface_hub/commands/upload.py b/src/huggingface_hub/commands/upload.py index fa3abc5c8c..fabeba0b96 100644 --- a/src/huggingface_hub/commands/upload.py +++ b/src/huggingface_hub/commands/upload.py @@ -20,7 +20,7 @@ """ import os import warnings -from argparse import _SubParsersAction +from argparse import Namespace, _SubParsersAction from typing import List, Optional from huggingface_hub import HfApi @@ -51,9 +51,9 @@ def register_subcommand(parser: _SubParsersAction): help="Path in repo. (optional)", ) upload_parser.add_argument( - "--type", + "--repo-type", type=str, - help="The type of the repo to upload (e.g. `dataset`).", + help="The type of the repo to upload to (e.g. `dataset`).", ) upload_parser.add_argument( "--revision", @@ -61,19 +61,19 @@ def register_subcommand(parser: _SubParsersAction): help="The revision of the repo to upload.", ) upload_parser.add_argument( - "--allow-patterns", + "--include", nargs="+", type=str, help="Glob patterns to match files to upload.", ) upload_parser.add_argument( - "--ignore-patterns", + "--exclude", nargs="+", type=str, help="Glob patterns to exclude from files to upload.", ) upload_parser.add_argument( - "--delete-patterns", + "--delete", nargs="+", type=str, help="Glob patterns for file to be deleted from the repo while committing.", @@ -110,24 +110,24 @@ def register_subcommand(parser: _SubParsersAction): ) upload_parser.set_defaults(func=UploadCommand) - def __init__(self, args): + def __init__(self, args: Namespace) -> None: self.api = HfApi(token=args.token) self.repo_id: str = args.repo_id self.path: str = args.path self.path_in_repo: str = args.path_in_repo - self.type: Optional[str] = args.type + self.repo_type: Optional[str] = args.repo_type self.revision: Optional[str] = args.revision - self.allow_patterns: List[str] = args.allow_patterns - self.ignore_patterns: List[str] = args.ignore_patterns - self.delete_patterns: List[str] = args.delete_patterns + self.include: List[str] = args.include + self.exclude: List[str] = args.exclude + self.delete: List[str] = args.delete self.commit_message: Optional[str] = args.commit_message self.commit_description: Optional[str] = args.commit_description - self.create_pr: Optional[bool] = args.create_pr - self.every: Optional[bool] = args.every + self.create_pr: bool = args.create_pr + self.every: bool = args.every self.token: Optional[str] = args.token self.quiet: bool = args.quiet - def run(self): + def run(self) -> None: if self.quiet: disable_progress_bars() with warnings.catch_warnings(): @@ -148,15 +148,15 @@ def _upload(self) -> str: # File or Folder based uploading if os.path.isfile(self.path): - if self.allow_patterns or self.ignore_patterns: - raise ValueError("--allow-patterns / --ignore-patterns cannot be used with a file path.") + if self.include or self.exclude or self.delete: + raise ValueError("--include / --exclude / --delete cannot be used with a file path.") return self.api.upload_file( path_or_fileobj=self.path, path_in_repo=self.path_in_repo, repo_id=self.repo_id, token=self.token, - repo_type=self.type, + repo_type=self.repo_type, revision=self.revision, commit_message=self.commit_message, commit_description=self.commit_description, @@ -170,14 +170,14 @@ def _upload(self) -> str: path_in_repo=self.path_in_repo, repo_id=self.repo_id, token=self.token, - repo_type=self.type, + repo_type=self.repo_type, revision=self.revision, commit_message=self.commit_message, commit_description=self.commit_description, create_pr=self.create_pr, - allow_patterns=self.allow_patterns, - ignore_patterns=self.ignore_patterns, - delete_patterns=self.delete_patterns, + allow_patterns=self.include, + ignore_patterns=self.exclude, + delete_patterns=self.delete, run_as_future=self.every, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 926d3e2b0a..8ae9cbdb20 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -72,11 +72,11 @@ def test_upload_basic(self) -> None: self.assertEqual(args.repo_id, DUMMY_MODEL_ID) self.assertEqual(args.path, "my-file") self.assertEqual(args.path_in_repo, None) - self.assertEqual(args.type, None) + self.assertEqual(args.repo_type, None) self.assertEqual(args.revision, None) - self.assertEqual(args.allow_patterns, None) - self.assertEqual(args.ignore_patterns, None) - self.assertEqual(args.delete_patterns, None) + self.assertEqual(args.include, None) + self.assertEqual(args.exclude, None) + self.assertEqual(args.delete, None) self.assertEqual(args.commit_message, None) self.assertEqual(args.commit_description, None) self.assertEqual(args.create_pr, False) @@ -93,17 +93,17 @@ def test_upload_with_all_options(self) -> None: DUMMY_MODEL_ID, "my-file", "/", - "--type", + "--repo-type", "model", "--revision", "v1.0.0", - "--allow-patterns", + "--include", "*.json", "*.yaml", - "--ignore-patterns", + "--exclude", "*.log", "*.txt", - "--delete-patterns", + "--delete", "*.config", "*.secret", "--commit-message", @@ -120,11 +120,11 @@ def test_upload_with_all_options(self) -> None: self.assertEqual(args.repo_id, DUMMY_MODEL_ID) self.assertEqual(args.path, "my-file") self.assertEqual(args.path_in_repo, "/") - self.assertEqual(args.type, "model") + self.assertEqual(args.repo_type, "model") self.assertEqual(args.revision, "v1.0.0") - self.assertEqual(args.allow_patterns, ["*.json", "*.yaml"]) - self.assertEqual(args.ignore_patterns, ["*.log", "*.txt"]) - self.assertEqual(args.delete_patterns, ["*.config", "*.secret"]) + self.assertEqual(args.include, ["*.json", "*.yaml"]) + self.assertEqual(args.exclude, ["*.log", "*.txt"]) + self.assertEqual(args.delete, ["*.config", "*.secret"]) self.assertEqual(args.commit_message, "My commit message") self.assertEqual(args.commit_description, "My commit description") self.assertEqual(args.create_pr, True) From 52ae4c68426a97584812b5e6bb2d8bd9c3f2d70d Mon Sep 17 00:00:00 2001 From: martinbrose <13284268+martinbrose@users.noreply.github.com> Date: Sat, 2 Sep 2023 16:44:03 +0100 Subject: [PATCH 06/16] Update repo_type argument --- src/huggingface_hub/commands/upload.py | 5 +++-- tests/test_cli.py | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/huggingface_hub/commands/upload.py b/src/huggingface_hub/commands/upload.py index fabeba0b96..1eeee88659 100644 --- a/src/huggingface_hub/commands/upload.py +++ b/src/huggingface_hub/commands/upload.py @@ -52,8 +52,9 @@ def register_subcommand(parser: _SubParsersAction): ) upload_parser.add_argument( "--repo-type", - type=str, - help="The type of the repo to upload to (e.g. `dataset`).", + choices=["model", "dataset", "space"], + default="model", + help="Type of the repo to upload to (e.g. `dataset`).", ) upload_parser.add_argument( "--revision", diff --git a/tests/test_cli.py b/tests/test_cli.py index 8ae9cbdb20..b47fab7995 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -72,7 +72,7 @@ def test_upload_basic(self) -> None: self.assertEqual(args.repo_id, DUMMY_MODEL_ID) self.assertEqual(args.path, "my-file") self.assertEqual(args.path_in_repo, None) - self.assertEqual(args.repo_type, None) + self.assertEqual(args.repo_type, "model") self.assertEqual(args.revision, None) self.assertEqual(args.include, None) self.assertEqual(args.exclude, None) @@ -94,7 +94,7 @@ def test_upload_with_all_options(self) -> None: "my-file", "/", "--repo-type", - "model", + "dataset", "--revision", "v1.0.0", "--include", @@ -120,7 +120,7 @@ def test_upload_with_all_options(self) -> None: self.assertEqual(args.repo_id, DUMMY_MODEL_ID) self.assertEqual(args.path, "my-file") self.assertEqual(args.path_in_repo, "/") - self.assertEqual(args.repo_type, "model") + self.assertEqual(args.repo_type, "dataset") self.assertEqual(args.revision, "v1.0.0") self.assertEqual(args.include, ["*.json", "*.yaml"]) self.assertEqual(args.exclude, ["*.log", "*.txt"]) From acc570b60211cc39b57d504867894f16527b808d Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Tue, 5 Sep 2023 18:18:55 +0200 Subject: [PATCH 07/16] add support for every + add tests + add doc --- docs/source/en/guides/upload.md | 53 ++++++ src/huggingface_hub/commands/upload.py | 223 ++++++++++++++++-------- tests/test_cli.py | 228 ++++++++++++++++++------- 3 files changed, 367 insertions(+), 137 deletions(-) diff --git a/docs/source/en/guides/upload.md b/docs/source/en/guides/upload.md index 9de33e6572..ff00a2ea95 100644 --- a/docs/source/en/guides/upload.md +++ b/docs/source/en/guides/upload.md @@ -106,6 +106,59 @@ but before that, all previous logs on the repo on deleted. All of this in a sing ... ) ``` +## Upload from the CLI + +You can also upload files to the Hub directly from your terminal using the `huggingface-cli upload` command. Internally +it uses the same [`upload_file`] and [`upload_folder`] helpers described above. + +You can either upload a single file or an entire folder: + +```bash +# Usage: huggingface-cli upload [repo_id] [local_path] [path_in_repo] +>>> huggingface-cli upload Wauplin/my-cool-model ./models/model.safetensors model.safetensors +https://huggingface.co/Wauplin/my-cool-model/blob/main/model.safetensors + +>>> huggingface-cli upload Wauplin/my-cool-model ./models . +https://huggingface.co/Wauplin/my-cool-model/tree/main +``` + +`local_path` and `path_in_repo` are optional and can be implicitly inferred. By default, `local_path` will be set to +the current directory and `path_in_repo` will be set to the relative path between the current directory and `local_path`. +If the implicit paths cannot be inferred, an error is raised. + +```bash +# Upload file (implicit path_in_repo) +huggingface-cli upload my-cool-model model.safetensors + +# Upload directory (implicit path_in_repo) +huggingface-cli upload my-cool-model ./models + +# Upload directory (implicit local_path, implicit path_in_repo) +huggingface-cli upload my-cool-model +``` + +By default, the token saved locally (using `huggingface-cli login`) will be used. If you want to authenticate explicitly, +use the `--token` option: + +```bash +huggingface-cli upload my-cool-model --token=hf_**** +``` + +When uploading a folder, you can use the `--include` and `--exclude` arguments to filter the files to upload. You can +also use `--delete` to delete existing files on the Hub. + +```bash +# Sync local Space with Hub (upload new files except from logs/, delete removed files) +huggingface-cli upload Wauplin/space-example --repo-type=space --exclude="/logs/*" --delete="*" --commit-message="Sync local Space with Hub" +``` + +Finally, you can also schedule a job that will upload your files regularly (see [scheduled uploads](#scheduled-uploads)). + +```bash +# Upload new logs every 10 minutes +huggingface-cli upload training-model logs/ --every=10 +``` + ## Advanced features In most cases, you won't need more than [`upload_file`] and [`upload_folder`] to upload your files to the Hub. diff --git a/src/huggingface_hub/commands/upload.py b/src/huggingface_hub/commands/upload.py index 1eeee88659..17b24df003 100644 --- a/src/huggingface_hub/commands/upload.py +++ b/src/huggingface_hub/commands/upload.py @@ -15,40 +15,61 @@ """Contains command to upload a repo or file with the CLI. Usage: - huggingface-cli upload repo_id - huggingface-cli upload repo_id [path] [path-in-repo] + # Upload file (implicit path in repo) + huggingface-cli upload my-cool-model ./my-cool-model.safetensors + + # Upload file (explicit path in repo) + huggingface-cli upload my-cool-model ./my-cool-model.safetensors model.safetensors + + # Upload directory (implicit paths) + huggingface-cli upload my-cool-model + + # Upload directory (explicit local path, explicit path in repo) + huggingface-cli upload my-cool-model ./models/my-cool-model . + + # Upload filtered directory (example: tensorboard logs except for the last run) + huggingface-cli upload my-cool-model ./model/training /logs --include "*.tfevents.*" --exclude "*20230905*" + + # Upload private dataset + huggingface-cli upload Wauplin/my-cool-dataset ./data . --repo-type=dataset --private + + # Upload with token + huggingface-cli upload Wauplin/my-cool-model --token=hf_**** + + # Sync local Space with Hub (upload new files, delete removed files) + huggingface-cli upload Wauplin/space-example --repo-type=space --exclude="/logs/*" --delete="*" --commit-message="Sync local Space with Hub" + + # Schedule commits every 30 minutes + huggingface-cli upload Wauplin/my-cool-model --every=30 """ import os +import time import warnings from argparse import Namespace, _SubParsersAction +from pathlib import Path from typing import List, Optional -from huggingface_hub import HfApi +from huggingface_hub import logging +from huggingface_hub._commit_scheduler import CommitScheduler from huggingface_hub.commands import BaseHuggingfaceCLICommand +from huggingface_hub.hf_api import create_repo, upload_file, upload_folder from huggingface_hub.utils import disable_progress_bars, enable_progress_bars class UploadCommand(BaseHuggingfaceCLICommand): @staticmethod def register_subcommand(parser: _SubParsersAction): - upload_parser = parser.add_parser( - "upload", - help="Upload a repo or a repo file to huggingface.co", - ) + upload_parser = parser.add_parser("upload", help="Upload a file or a folder to a repo on the Hub") upload_parser.add_argument( - "repo_id", - type=str, - help="The ID of the repo to upload to (e.g. `username/repo-name`).", + "repo_id", type=str, help="The ID of the repo to upload to (e.g. `username/repo-name`)." ) upload_parser.add_argument( - "path", - nargs="?", - help="Local path. (optional)", + "local_path", nargs="?", help="Local path to the file or folder to upload. Defaults to current directory." ) upload_parser.add_argument( "path_in_repo", nargs="?", - help="Path in repo. (optional)", + help="Path of the file or folder in the repo. Defaults to the relative path of the file or folder.", ) upload_parser.add_argument( "--repo-type", @@ -59,128 +80,182 @@ def register_subcommand(parser: _SubParsersAction): upload_parser.add_argument( "--revision", type=str, - help="The revision of the repo to upload.", + help="An optional Git revision id which can be a branch name, a tag, or a commit hash.", ) upload_parser.add_argument( - "--include", - nargs="+", - type=str, - help="Glob patterns to match files to upload.", + "--private", + action="store_true", + help=( + "Whether to create a private repo if repo doesn't exist on the Hub. Ignored if the repo already" + " exists." + ), ) + upload_parser.add_argument("--include", nargs="*", type=str, help="Glob patterns to match files to upload.") upload_parser.add_argument( - "--exclude", - nargs="+", - type=str, - help="Glob patterns to exclude from files to upload.", + "--exclude", nargs="*", type=str, help="Glob patterns to exclude from files to upload." ) upload_parser.add_argument( "--delete", - nargs="+", + nargs="*", type=str, help="Glob patterns for file to be deleted from the repo while committing.", ) upload_parser.add_argument( - "--commit-message", - type=str, - help="The summary / title / first line of the generated commit.", - ) - upload_parser.add_argument( - "--commit-description", - type=str, - help="The description of the generated commit.", + "--commit-message", type=str, help="The summary / title / first line of the generated commit." ) + upload_parser.add_argument("--commit-description", type=str, help="The description of the generated commit.") upload_parser.add_argument( - "--create-pr", - action="store_true", - help="Whether to create a PR.", + "--create-pr", action="store_true", help="Whether to upload content as a new Pull Request." ) upload_parser.add_argument( "--every", - action="store_true", - help="Run a CommitScheduler instead of a single commit.", + type=float, + help="If set, a background job is scheduled to create commits every `every` minutes.", ) upload_parser.add_argument( - "--token", - type=str, - help="A User Access Token generated from https://huggingface.co/settings/tokens", + "--token", type=str, help="A User Access Token generated from https://huggingface.co/settings/tokens" ) upload_parser.add_argument( "--quiet", action="store_true", help="If True, progress bars are disabled and only the path to the uploaded files is printed.", ) + upload_parser.add_argument("--verbose", action="store_true", help="If True, more logs are printed.") upload_parser.set_defaults(func=UploadCommand) def __init__(self, args: Namespace) -> None: - self.api = HfApi(token=args.token) self.repo_id: str = args.repo_id - self.path: str = args.path - self.path_in_repo: str = args.path_in_repo self.repo_type: Optional[str] = args.repo_type self.revision: Optional[str] = args.revision - self.include: List[str] = args.include - self.exclude: List[str] = args.exclude - self.delete: List[str] = args.delete + self.private: bool = args.private + + self.include: Optional[List[str]] = args.include + self.exclude: Optional[List[str]] = args.exclude + self.delete: Optional[List[str]] = args.delete + self.commit_message: Optional[str] = args.commit_message self.commit_description: Optional[str] = args.commit_description self.create_pr: bool = args.create_pr - self.every: bool = args.every self.token: Optional[str] = args.token - self.quiet: bool = args.quiet + + # Quiet/verbose mode + self.quiet: bool = args.quiet # disable warnings and progress bars + self.verbose: bool = args.verbose # set verbosity to INFO + if self.quiet and self.verbose: + raise ValueError("Cannot set both `--quiet` and `--verbose`.") + + # Possibly implicit `path` and `path_in_repo` + self.local_path: str = args.local_path if args.local_path is not None else "." + self.path_in_repo: str + if args.path_in_repo is not None: + self.path_in_repo = args.path_in_repo + else: # Implicit path_in_repo => relative to current directory + try: + self.path_in_repo = Path(self.local_path).relative_to(".").as_posix() + except ValueError as e: + raise ValueError( + "Cannot determine `path_in_repo` implicitly. Please set `--path-in-repo=...` and retry." + ) from e + + if args.every is not None and args.every <= 0: + raise ValueError(f"`every` must be a positive value (got '{args.every}')") + self.every: Optional[float] = args.every def run(self) -> None: if self.quiet: disable_progress_bars() with warnings.catch_warnings(): warnings.simplefilter("ignore") - print(self._upload()) # Print path to uploaded files + print(self._upload()) enable_progress_bars() else: - print(self._upload()) # Print path to uploaded files + if self.verbose: + logging.set_verbosity_info() + print(self._upload()) def _upload(self) -> str: - self.path = "." if self.path is None else self.path + if os.path.isfile(self.local_path): + if self.include is not None and len(self.include) > 0: + warnings.warn("Ignoring `--include` since a single file is uploaded.") + if self.exclude is not None and len(self.exclude) > 0: + warnings.warn("Ignoring `--exclude` since a single file is uploaded.") + if self.delete is not None and len(self.delete) > 0: + warnings.warn("Ignoring `--delete` since a single file is uploaded.") - self.path_in_repo = ( - self.path_in_repo - if self.path_in_repo - else (os.path.relpath(self.path).replace("\\", "/") if self.path != "." else "/") - ) + # Schedule commits if `every` is set + if self.every is not None: + if os.path.isfile(self.local_path): + # If file => watch entire folder + use allow_patterns + folder_path = os.path.dirname(self.local_path) + path_in_repo = ( + self.path_in_repo[: -len(self.local_path)] # remove filename from path_in_repo + if self.path_in_repo.endswith(self.local_path) + else self.path_in_repo + ) + allow_patterns = [self.local_path] + ignore_patterns = [] + else: + folder_path = self.local_path + path_in_repo = self.path_in_repo + allow_patterns = self.include or [] + ignore_patterns = self.exclude or [] + if self.delete is not None and len(self.delete) > 0: + warnings.warn("Ignoring `--delete` when uploading with scheduled commits.") - # File or Folder based uploading - if os.path.isfile(self.path): - if self.include or self.exclude or self.delete: - raise ValueError("--include / --exclude / --delete cannot be used with a file path.") - - return self.api.upload_file( - path_or_fileobj=self.path, - path_in_repo=self.path_in_repo, + scheduler = CommitScheduler( + folder_path=folder_path, repo_id=self.repo_id, + repo_type=self.repo_type, + revision=self.revision, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + path_in_repo=path_in_repo, + private=self.private, + every=self.every, token=self.token, + ) + print(f"Scheduling commits every {self.every} minutes to {scheduler.repo_id}.") + try: # Block main thread until KeyboardInterrupt + while True: + time.sleep(100) + except KeyboardInterrupt: + scheduler.stop() + return "Stopped scheduled commits." + + # Otherwise, create repo and proceed with the upload + if not os.path.isfile(self.local_path) and not os.path.isdir(self.local_path): + raise FileNotFoundError(f"No such file or directory: '{self.local_path}'.") + repo_id = create_repo( + repo_id=self.repo_id, repo_type=self.repo_type, exist_ok=True, private=self.private, token=self.token + ).repo_id + + # File-based upload + if os.path.isfile(self.local_path): + return upload_file( + path_or_fileobj=self.local_path, + path_in_repo=self.path_in_repo, + repo_id=repo_id, repo_type=self.repo_type, revision=self.revision, + token=self.token, commit_message=self.commit_message, commit_description=self.commit_description, create_pr=self.create_pr, - run_as_future=self.every, ) - elif os.path.isdir(self.path): - return self.api.upload_folder( - folder_path=self.path, + # Folder-based upload + else: + return upload_folder( + folder_path=self.local_path, path_in_repo=self.path_in_repo, - repo_id=self.repo_id, - token=self.token, + repo_id=repo_id, repo_type=self.repo_type, revision=self.revision, + token=self.token, commit_message=self.commit_message, commit_description=self.commit_description, create_pr=self.create_pr, allow_patterns=self.include, ignore_patterns=self.exclude, delete_patterns=self.delete, - run_as_future=self.every, ) - - else: - raise ValueError(f"Provided PATH: {self.path} does not exist.") diff --git a/tests/test_cli.py b/tests/test_cli.py index b47fab7995..8fda6af546 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,13 +1,14 @@ import unittest from argparse import ArgumentParser +from pathlib import Path +from unittest.mock import Mock, patch from huggingface_hub.commands.delete_cache import DeleteCacheCommand from huggingface_hub.commands.scan_cache import ScanCacheCommand from huggingface_hub.commands.upload import UploadCommand +from huggingface_hub.utils import SoftTemporaryDirectory -from .testing_utils import ( - DUMMY_MODEL_ID, -) +from .testing_utils import DUMMY_MODEL_ID class TestCLI(unittest.TestCase): @@ -68,67 +69,168 @@ def setUp(self) -> None: def test_upload_basic(self) -> None: """Test `huggingface-cli upload my-file to dummy-repo`.""" - args = self.parser.parse_args(["upload", DUMMY_MODEL_ID, "my-file"]) - self.assertEqual(args.repo_id, DUMMY_MODEL_ID) - self.assertEqual(args.path, "my-file") - self.assertEqual(args.path_in_repo, None) - self.assertEqual(args.repo_type, "model") - self.assertEqual(args.revision, None) - self.assertEqual(args.include, None) - self.assertEqual(args.exclude, None) - self.assertEqual(args.delete, None) - self.assertEqual(args.commit_message, None) - self.assertEqual(args.commit_description, None) - self.assertEqual(args.create_pr, False) - self.assertEqual(args.every, False) - self.assertEqual(args.token, None) - self.assertEqual(args.quiet, False) - self.assertEqual(args.func, UploadCommand) + cmd = UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, "my-file"])) + self.assertEqual(cmd.repo_id, DUMMY_MODEL_ID) + self.assertEqual(cmd.local_path, "my-file") + self.assertEqual(cmd.path_in_repo, "my-file") # implicit + self.assertEqual(cmd.repo_type, "model") + self.assertEqual(cmd.revision, None) + self.assertEqual(cmd.include, None) + self.assertEqual(cmd.exclude, None) + self.assertEqual(cmd.delete, None) + self.assertEqual(cmd.commit_message, None) + self.assertEqual(cmd.commit_description, None) + self.assertEqual(cmd.create_pr, False) + self.assertEqual(cmd.every, None) + self.assertEqual(cmd.token, None) + self.assertEqual(cmd.quiet, False) def test_upload_with_all_options(self) -> None: """Test `huggingface-cli upload my-file to dummy-repo with all options selected`.""" - args = self.parser.parse_args( - [ - "upload", - DUMMY_MODEL_ID, - "my-file", - "/", - "--repo-type", - "dataset", - "--revision", - "v1.0.0", - "--include", - "*.json", - "*.yaml", - "--exclude", - "*.log", - "*.txt", - "--delete", - "*.config", - "*.secret", - "--commit-message", - "My commit message", - "--commit-description", - "My commit description", - "--create-pr", - "--every", - "--token", - "my-token", - "--quiet", - ] + cmd = UploadCommand( + self.parser.parse_args( + [ + "upload", + DUMMY_MODEL_ID, + "my-file", + "/", + "--repo-type", + "dataset", + "--revision", + "v1.0.0", + "--include", + "*.json", + "*.yaml", + "--exclude", + "*.log", + "*.txt", + "--delete", + "*.config", + "*.secret", + "--commit-message", + "My commit message", + "--commit-description", + "My commit description", + "--create-pr", + "--every", + "5", + "--token", + "my-token", + "--quiet", + ] + ) ) - self.assertEqual(args.repo_id, DUMMY_MODEL_ID) - self.assertEqual(args.path, "my-file") - self.assertEqual(args.path_in_repo, "/") - self.assertEqual(args.repo_type, "dataset") - self.assertEqual(args.revision, "v1.0.0") - self.assertEqual(args.include, ["*.json", "*.yaml"]) - self.assertEqual(args.exclude, ["*.log", "*.txt"]) - self.assertEqual(args.delete, ["*.config", "*.secret"]) - self.assertEqual(args.commit_message, "My commit message") - self.assertEqual(args.commit_description, "My commit description") - self.assertEqual(args.create_pr, True) - self.assertEqual(args.every, True) - self.assertEqual(args.token, "my-token") - self.assertEqual(args.quiet, True) - self.assertEqual(args.func, UploadCommand) + self.assertEqual(cmd.repo_id, DUMMY_MODEL_ID) + self.assertEqual(cmd.local_path, "my-file") + self.assertEqual(cmd.path_in_repo, "/") + self.assertEqual(cmd.repo_type, "dataset") + self.assertEqual(cmd.revision, "v1.0.0") + self.assertEqual(cmd.include, ["*.json", "*.yaml"]) + self.assertEqual(cmd.exclude, ["*.log", "*.txt"]) + self.assertEqual(cmd.delete, ["*.config", "*.secret"]) + self.assertEqual(cmd.commit_message, "My commit message") + self.assertEqual(cmd.commit_description, "My commit description") + self.assertEqual(cmd.create_pr, True) + self.assertEqual(cmd.every, 5) + self.assertEqual(cmd.token, "my-token") + self.assertEqual(cmd.quiet, True) + + def test_upload_implicit_paths(self) -> None: + cmd = UploadCommand(self.parser.parse_args(["upload", "my-repo"])) + self.assertEqual(cmd.local_path, ".") + self.assertEqual(cmd.path_in_repo, ".") + + def test_upload_explicit_local_path_implicit_path_in_repo(self) -> None: + cmd = UploadCommand(self.parser.parse_args(["upload", "my-repo", "./path/to/folder"])) + self.assertEqual(cmd.local_path, "./path/to/folder") + self.assertEqual(cmd.path_in_repo, "path/to/folder") + + def test_upload_explicit_paths(self) -> None: + cmd = UploadCommand(self.parser.parse_args(["upload", "my-repo", "./path/to/folder", "data/"])) + self.assertEqual(cmd.local_path, "./path/to/folder") + self.assertEqual(cmd.path_in_repo, "data/") + + def test_cannot_upload_verbose_and_quiet(self) -> None: + with self.assertRaises(ValueError): + UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, "my-file", "--quiet", "--verbose"])) + + def test_every_must_be_positive(self) -> None: + with self.assertRaises(ValueError): + UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, "--every", "0"])) + + with self.assertRaises(ValueError): + UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, "--every", "-10"])) + + def test_every_as_int(self) -> None: + cmd = UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, "--every", "10"])) + self.assertEqual(cmd.every, 10) + + def test_every_as_float(self) -> None: + cmd = UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, "--every", "0.5"])) + self.assertEqual(cmd.every, 0.5) + + @patch("huggingface_hub.commands.upload.upload_folder") + @patch("huggingface_hub.commands.upload.create_repo") + def test_upload_folder_mock(self, create_mock: Mock, upload_mock: Mock) -> None: + with SoftTemporaryDirectory() as cache_dir: + cmd = UploadCommand( + self.parser.parse_args( + ["upload", "my-model", cache_dir, ".", "--private", "--include", "*.json", "--delete", "*.json"] + ) + ) + cmd.run() + + create_mock.assert_called_once_with( + repo_id="my-model", repo_type="model", exist_ok=True, private=True, token=None + ) + upload_mock.assert_called_once_with( + folder_path=cache_dir, + path_in_repo=".", + repo_id=create_mock.return_value.repo_id, + repo_type="model", + revision=None, + token=None, + commit_message=None, + commit_description=None, + create_pr=False, + allow_patterns=["*.json"], + ignore_patterns=None, + delete_patterns=["*.json"], + ) + + @patch("huggingface_hub.commands.upload.upload_file") + @patch("huggingface_hub.commands.upload.create_repo") + def test_upload_file_mock(self, create_mock: Mock, upload_mock: Mock) -> None: + with SoftTemporaryDirectory() as cache_dir: + file_path = Path(cache_dir) / "file.txt" + file_path.write_text("content") + cmd = UploadCommand( + self.parser.parse_args( + ["upload", "my-dataset", str(file_path), "logs/file.txt", "--repo-type", "dataset", "--create-pr"] + ) + ) + cmd.run() + + create_mock.assert_called_once_with( + repo_id="my-dataset", repo_type="dataset", exist_ok=True, private=False, token=None + ) + upload_mock.assert_called_once_with( + path_or_fileobj=str(file_path), + path_in_repo="logs/file.txt", + repo_id=create_mock.return_value.repo_id, + repo_type="dataset", + revision=None, + token=None, + commit_message=None, + commit_description=None, + create_pr=True, + ) + + @patch("huggingface_hub.commands.upload.create_repo") + def test_upload_missing_path(self, create_mock: Mock) -> None: + cmd = UploadCommand(self.parser.parse_args(["upload", "my-model", "/path/to/missing_file", "logs/file.txt"])) + with self.assertRaises(FileNotFoundError): + cmd.run() # File/folder does not exist locally + + # Repo creation happens before the check + create_mock.assert_not_called() From e84b1918159bfb482d94a41576a3bd29ee02ff79 Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Tue, 5 Sep 2023 18:30:16 +0200 Subject: [PATCH 08/16] remove --verbose and keep only --quiet as option --- src/huggingface_hub/commands/upload.py | 9 +-------- tests/test_cli.py | 4 ---- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/huggingface_hub/commands/upload.py b/src/huggingface_hub/commands/upload.py index 17b24df003..3e8d3d9fef 100644 --- a/src/huggingface_hub/commands/upload.py +++ b/src/huggingface_hub/commands/upload.py @@ -120,7 +120,6 @@ def register_subcommand(parser: _SubParsersAction): action="store_true", help="If True, progress bars are disabled and only the path to the uploaded files is printed.", ) - upload_parser.add_argument("--verbose", action="store_true", help="If True, more logs are printed.") upload_parser.set_defaults(func=UploadCommand) def __init__(self, args: Namespace) -> None: @@ -137,12 +136,7 @@ def __init__(self, args: Namespace) -> None: self.commit_description: Optional[str] = args.commit_description self.create_pr: bool = args.create_pr self.token: Optional[str] = args.token - - # Quiet/verbose mode self.quiet: bool = args.quiet # disable warnings and progress bars - self.verbose: bool = args.verbose # set verbosity to INFO - if self.quiet and self.verbose: - raise ValueError("Cannot set both `--quiet` and `--verbose`.") # Possibly implicit `path` and `path_in_repo` self.local_path: str = args.local_path if args.local_path is not None else "." @@ -169,8 +163,7 @@ def run(self) -> None: print(self._upload()) enable_progress_bars() else: - if self.verbose: - logging.set_verbosity_info() + logging.set_verbosity_info() print(self._upload()) def _upload(self) -> str: diff --git a/tests/test_cli.py b/tests/test_cli.py index 8fda6af546..600153a50c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -150,10 +150,6 @@ def test_upload_explicit_paths(self) -> None: self.assertEqual(cmd.local_path, "./path/to/folder") self.assertEqual(cmd.path_in_repo, "data/") - def test_cannot_upload_verbose_and_quiet(self) -> None: - with self.assertRaises(ValueError): - UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, "my-file", "--quiet", "--verbose"])) - def test_every_must_be_positive(self) -> None: with self.assertRaises(ValueError): UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, "--every", "0"])) From 62ab37848b32e6ae2cfeb8e1dc21fcc82c470d73 Mon Sep 17 00:00:00 2001 From: Lucain Date: Wed, 6 Sep 2023 09:56:05 +0200 Subject: [PATCH 09/16] Apply suggestions from code review Co-authored-by: Steven Liu <59462357+stevhliu@users.noreply.github.com> --- docs/source/en/guides/upload.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/en/guides/upload.md b/docs/source/en/guides/upload.md index bdccd15af1..62981909cd 100644 --- a/docs/source/en/guides/upload.md +++ b/docs/source/en/guides/upload.md @@ -108,7 +108,7 @@ but before that, all previous logs on the repo on deleted. All of this in a sing ## Upload from the CLI -You can also upload files to the Hub directly from your terminal using the `huggingface-cli upload` command. Internally +You can use the `huggingface-cli upload` command from the terminal to directly upload files to the Hub. Internally it uses the same [`upload_file`] and [`upload_folder`] helpers described above. You can either upload a single file or an entire folder: @@ -122,8 +122,8 @@ https://huggingface.co/Wauplin/my-cool-model/blob/main/model.safetensors https://huggingface.co/Wauplin/my-cool-model/tree/main ``` -`local_path` and `path_in_repo` are optional and can be implicitly inferred. By default, `local_path` will be set to -the current directory and `path_in_repo` will be set to the relative path between the current directory and `local_path`. +`local_path` and `path_in_repo` are optional and can be implicitly inferred. By default, `local_path` is set to +the current directory, and `path_in_repo` is set to the relative path between the current directory and `local_path`. If the implicit paths cannot be inferred, an error is raised. ```bash From 9631b80e71226a08c64b6368697d049213d00ef3 Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Wed, 6 Sep 2023 11:15:25 +0200 Subject: [PATCH 10/16] Update implicit path strategy given feedback --- src/huggingface_hub/commands/upload.py | 43 ++++++++++----- tests/test_cli.py | 74 +++++++++++++++++++++----- 2 files changed, 91 insertions(+), 26 deletions(-) diff --git a/src/huggingface_hub/commands/upload.py b/src/huggingface_hub/commands/upload.py index 3e8d3d9fef..0c9152d5de 100644 --- a/src/huggingface_hub/commands/upload.py +++ b/src/huggingface_hub/commands/upload.py @@ -46,7 +46,6 @@ import time import warnings from argparse import Namespace, _SubParsersAction -from pathlib import Path from typing import List, Optional from huggingface_hub import logging @@ -138,23 +137,39 @@ def __init__(self, args: Namespace) -> None: self.token: Optional[str] = args.token self.quiet: bool = args.quiet # disable warnings and progress bars - # Possibly implicit `path` and `path_in_repo` - self.local_path: str = args.local_path if args.local_path is not None else "." - self.path_in_repo: str - if args.path_in_repo is not None: - self.path_in_repo = args.path_in_repo - else: # Implicit path_in_repo => relative to current directory - try: - self.path_in_repo = Path(self.local_path).relative_to(".").as_posix() - except ValueError as e: - raise ValueError( - "Cannot determine `path_in_repo` implicitly. Please set `--path-in-repo=...` and retry." - ) from e - + # Check `--every` is valid if args.every is not None and args.every <= 0: raise ValueError(f"`every` must be a positive value (got '{args.every}')") self.every: Optional[float] = args.every + # Resolve `local_path` and `path_in_repo` + self.local_path: str + self.path_in_repo: str + if args.local_path is None and os.path.isfile(args.repo_id): + # Implicit case 1: user provided only a repo_id which happen to be a local file as well => upload it with same name + self.local_path = args.repo_id + self.path_in_repo = args.repo_id + elif args.local_path is None and os.path.isdir(args.repo_id): + # Implicit case 2: user provided only a repo_id which happen to be a local folder as well => upload it at root + self.local_path = args.repo_id + self.path_in_repo = "." + elif args.local_path is None: + # Implicit case 3: user provided only a repo_id that does not match a local file or folder => upload the current directory at root + self.local_path = "." + self.path_in_repo = "." + elif args.path_in_repo is None and os.path.isfile(args.local_path): + # Explicit local path to file, no path in repo => upload it at root with same name + self.local_path = args.local_path + self.path_in_repo = os.path.basename(args.local_path) + elif args.path_in_repo is None: + # Explicit local path to folder, no path in repo => upload at root + self.local_path = args.local_path + self.path_in_repo = "." + else: + # Finally, if both paths are explicit + self.local_path = args.local_path + self.path_in_repo = args.path_in_repo + def run(self) -> None: if self.quiet: disable_progress_bars() diff --git a/tests/test_cli.py b/tests/test_cli.py index 56ac73d931..74d4f8079a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,7 +1,10 @@ +import os import unittest import warnings from argparse import ArgumentParser, Namespace +from contextlib import contextmanager from pathlib import Path +from typing import Generator from unittest.mock import Mock, patch from huggingface_hub.commands.delete_cache import DeleteCacheCommand @@ -10,7 +13,7 @@ from huggingface_hub.commands.upload import UploadCommand from huggingface_hub.utils import SoftTemporaryDirectory, capture_output -from .testing_utils import DUMMY_MODEL_ID +from .testing_utils import DUMMY_MODEL_ID, xfail_on_windows class TestCacheCommand(unittest.TestCase): @@ -68,11 +71,11 @@ def setUp(self) -> None: UploadCommand.register_subcommand(commands_parser) def test_upload_basic(self) -> None: - """Test `huggingface-cli upload my-file to dummy-repo`.""" - cmd = UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, "my-file"])) + """Test `huggingface-cli upload my-folder to dummy-repo`.""" + cmd = UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, "my-folder"])) self.assertEqual(cmd.repo_id, DUMMY_MODEL_ID) - self.assertEqual(cmd.local_path, "my-file") - self.assertEqual(cmd.path_in_repo, "my-file") # implicit + self.assertEqual(cmd.local_path, "my-folder") + self.assertEqual(cmd.path_in_repo, ".") # implicit self.assertEqual(cmd.repo_type, "model") self.assertEqual(cmd.revision, None) self.assertEqual(cmd.include, None) @@ -93,7 +96,7 @@ def test_upload_with_all_options(self) -> None: "upload", DUMMY_MODEL_ID, "my-file", - "/", + "data/", "--repo-type", "dataset", "--revision", @@ -122,7 +125,7 @@ def test_upload_with_all_options(self) -> None: ) self.assertEqual(cmd.repo_id, DUMMY_MODEL_ID) self.assertEqual(cmd.local_path, "my-file") - self.assertEqual(cmd.path_in_repo, "/") + self.assertEqual(cmd.path_in_repo, "data/") self.assertEqual(cmd.repo_type, "dataset") self.assertEqual(cmd.revision, "v1.0.0") self.assertEqual(cmd.include, ["*.json", "*.yaml"]) @@ -135,15 +138,52 @@ def test_upload_with_all_options(self) -> None: self.assertEqual(cmd.token, "my-token") self.assertEqual(cmd.quiet, True) - def test_upload_implicit_paths(self) -> None: - cmd = UploadCommand(self.parser.parse_args(["upload", "my-repo"])) + def test_upload_implicit_local_path_when_folder_exists(self) -> None: + with tmp_current_directory() as cache_dir: + folder_path = Path(cache_dir) / "my-cool-model" + folder_path.mkdir() + cmd = UploadCommand(self.parser.parse_args(["upload", "my-cool-model"])) + + # A folder with the same name as the repo exists => upload it at the root of the repo + self.assertEqual(cmd.local_path, "my-cool-model") + self.assertEqual(cmd.path_in_repo, ".") + + def test_upload_implicit_local_path_when_file_exists(self) -> None: + with tmp_current_directory() as cache_dir: + folder_path = Path(cache_dir) / "my-cool-model" + folder_path.touch() + cmd = UploadCommand(self.parser.parse_args(["upload", "my-cool-model"])) + + # A file with the same name as the repo exists => upload it at the root of the repo + self.assertEqual(cmd.local_path, "my-cool-model") + self.assertEqual(cmd.path_in_repo, "my-cool-model") + + def test_upload_implicit_local_path_otherwise(self) -> None: + with tmp_current_directory(): + cmd = UploadCommand(self.parser.parse_args(["upload", "my-cool-model"])) + + # No folder or file has the same name as the repo => upload entire folder ("./") at the root of the repo self.assertEqual(cmd.local_path, ".") self.assertEqual(cmd.path_in_repo, ".") - def test_upload_explicit_local_path_implicit_path_in_repo(self) -> None: - cmd = UploadCommand(self.parser.parse_args(["upload", "my-repo", "./path/to/folder"])) + @xfail_on_windows(reason="No implicit path on Windows") + def test_upload_explicit_local_path_to_folder_implicit_path_in_repo(self) -> None: + with tmp_current_directory() as cache_dir: + folder_path = Path(cache_dir) / "path" / "to" / "folder" + folder_path.mkdir(parents=True, exist_ok=True) + cmd = UploadCommand(self.parser.parse_args(["upload", "my-repo", "./path/to/folder"])) self.assertEqual(cmd.local_path, "./path/to/folder") - self.assertEqual(cmd.path_in_repo, "path/to/folder") + self.assertEqual(cmd.path_in_repo, ".") # Always upload the folder at the root of the repo + + @xfail_on_windows(reason="No implicit path on Windows") + def test_upload_explicit_local_path_to_file_implicit_path_in_repo(self) -> None: + with tmp_current_directory() as cache_dir: + file_path = Path(cache_dir) / "path" / "to" / "file.txt" + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.touch() + cmd = UploadCommand(self.parser.parse_args(["upload", "my-repo", "./path/to/file.txt"])) + self.assertEqual(cmd.local_path, "./path/to/file.txt") + self.assertEqual(cmd.path_in_repo, "file.txt") # If a file, upload it at the root of the repo and keep name def test_upload_explicit_paths(self) -> None: cmd = UploadCommand(self.parser.parse_args(["upload", "my-repo", "./path/to/folder", "data/"])) @@ -452,3 +492,13 @@ def test_download_with_ignored_patterns(self, mock: Mock) -> None: # Taken from https://docs.pytest.org/en/latest/how-to/capture-warnings.html#additional-use-cases-of-warnings-in-tests warnings.simplefilter("error") DownloadCommand(args).run() + + +@contextmanager +def tmp_current_directory() -> Generator[str, None, None]: + """Change current directory to a tmp dir and revert back when exiting.""" + with SoftTemporaryDirectory() as tmp_dir: + cwd = os.getcwd() + os.chdir(tmp_dir) + yield tmp_dir + os.chdir(cwd) From d760b045f6205952207dc502131bf873dcdaa15f Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Wed, 6 Sep 2023 11:32:56 +0200 Subject: [PATCH 11/16] explain better implicit paths in docs --- docs/source/en/guides/upload.md | 13 +++++++------ src/huggingface_hub/commands/upload.py | 9 +++++---- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/source/en/guides/upload.md b/docs/source/en/guides/upload.md index 62981909cd..6350e3f24f 100644 --- a/docs/source/en/guides/upload.md +++ b/docs/source/en/guides/upload.md @@ -122,18 +122,19 @@ https://huggingface.co/Wauplin/my-cool-model/blob/main/model.safetensors https://huggingface.co/Wauplin/my-cool-model/tree/main ``` -`local_path` and `path_in_repo` are optional and can be implicitly inferred. By default, `local_path` is set to -the current directory, and `path_in_repo` is set to the relative path between the current directory and `local_path`. -If the implicit paths cannot be inferred, an error is raised. +`local_path` and `path_in_repo` are optional and can be implicitly inferred. If `local_path` is not set, the tool will +check if a local folder or file has the same name as the `repo_id`. If that's the case, its content will be uploaded. +Otherwise, `local_path` will default to the current directory. In any case, if `path_in_repo` is not set, files are +uploaded at the root of the repo. ```bash -# Upload file (implicit path_in_repo) +# Upload file at root huggingface-cli upload my-cool-model model.safetensors -# Upload directory (implicit path_in_repo) +# Upload directory at root huggingface-cli upload my-cool-model ./models -# Upload directory (implicit local_path, implicit path_in_repo) +# Upload `my-cool-model/` directory if it exist, `./` otherwise huggingface-cli upload my-cool-model ``` diff --git a/src/huggingface_hub/commands/upload.py b/src/huggingface_hub/commands/upload.py index 0c9152d5de..5d1ee22da2 100644 --- a/src/huggingface_hub/commands/upload.py +++ b/src/huggingface_hub/commands/upload.py @@ -15,16 +15,17 @@ """Contains command to upload a repo or file with the CLI. Usage: - # Upload file (implicit path in repo) + # Upload file (implicit) huggingface-cli upload my-cool-model ./my-cool-model.safetensors - # Upload file (explicit path in repo) + # Upload file (explicit) huggingface-cli upload my-cool-model ./my-cool-model.safetensors model.safetensors - # Upload directory (implicit paths) + # Upload directory (implicit). + # If `my-cool-model` is a directory, it will be uploaded. Otherwise, the current directory is uploaded. huggingface-cli upload my-cool-model - # Upload directory (explicit local path, explicit path in repo) + # Upload directory (explicit) huggingface-cli upload my-cool-model ./models/my-cool-model . # Upload filtered directory (example: tensorboard logs except for the last run) From 4833f769b076e876d1284bfbd5ac362b193530a5 Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Wed, 6 Sep 2023 11:36:57 +0200 Subject: [PATCH 12/16] fix when implicit local dir + org repo_id --- src/huggingface_hub/commands/upload.py | 11 ++++++----- tests/test_cli.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/huggingface_hub/commands/upload.py b/src/huggingface_hub/commands/upload.py index 5d1ee22da2..aa2b822f6b 100644 --- a/src/huggingface_hub/commands/upload.py +++ b/src/huggingface_hub/commands/upload.py @@ -144,15 +144,16 @@ def __init__(self, args: Namespace) -> None: self.every: Optional[float] = args.every # Resolve `local_path` and `path_in_repo` + repo_name: str = args.repo_id.split("/")[-1] # e.g. "Wauplin/my-cool-model" => "my-cool-model" self.local_path: str self.path_in_repo: str - if args.local_path is None and os.path.isfile(args.repo_id): + if args.local_path is None and os.path.isfile(repo_name): # Implicit case 1: user provided only a repo_id which happen to be a local file as well => upload it with same name - self.local_path = args.repo_id - self.path_in_repo = args.repo_id - elif args.local_path is None and os.path.isdir(args.repo_id): + self.local_path = repo_name + self.path_in_repo = repo_name + elif args.local_path is None and os.path.isdir(repo_name): # Implicit case 2: user provided only a repo_id which happen to be a local folder as well => upload it at root - self.local_path = args.repo_id + self.local_path = repo_name self.path_in_repo = "." elif args.local_path is None: # Implicit case 3: user provided only a repo_id that does not match a local file or folder => upload the current directory at root diff --git a/tests/test_cli.py b/tests/test_cli.py index 74d4f8079a..c80b58f835 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -158,6 +158,16 @@ def test_upload_implicit_local_path_when_file_exists(self) -> None: self.assertEqual(cmd.local_path, "my-cool-model") self.assertEqual(cmd.path_in_repo, "my-cool-model") + def test_upload_implicit_local_path_when_org_repo(self) -> None: + with tmp_current_directory() as cache_dir: + folder_path = Path(cache_dir) / "my-cool-model" + folder_path.mkdir() + cmd = UploadCommand(self.parser.parse_args(["upload", "my-cool-org/my-cool-model"])) + + # A folder with the same name as the repo exists => upload it at the root of the repo + self.assertEqual(cmd.local_path, "my-cool-model") + self.assertEqual(cmd.path_in_repo, ".") + def test_upload_implicit_local_path_otherwise(self) -> None: with tmp_current_directory(): cmd = UploadCommand(self.parser.parse_args(["upload", "my-cool-model"])) From f190632e1fe78aa831864781b0467c33ef0418b7 Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Wed, 6 Sep 2023 12:15:30 +0200 Subject: [PATCH 13/16] raise exception if implicit local path not found --- docs/source/en/guides/upload.md | 6 +++--- src/huggingface_hub/commands/upload.py | 9 ++++----- tests/test_cli.py | 26 ++++++++++++++------------ 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/docs/source/en/guides/upload.md b/docs/source/en/guides/upload.md index 6350e3f24f..fe2f130774 100644 --- a/docs/source/en/guides/upload.md +++ b/docs/source/en/guides/upload.md @@ -124,8 +124,8 @@ https://huggingface.co/Wauplin/my-cool-model/tree/main `local_path` and `path_in_repo` are optional and can be implicitly inferred. If `local_path` is not set, the tool will check if a local folder or file has the same name as the `repo_id`. If that's the case, its content will be uploaded. -Otherwise, `local_path` will default to the current directory. In any case, if `path_in_repo` is not set, files are -uploaded at the root of the repo. +Otherwise, an exception is raised asking the user to explicitly set `local_path`. In any case, if `path_in_repo` is not +set, files are uploaded at the root of the repo. ```bash # Upload file at root @@ -134,7 +134,7 @@ huggingface-cli upload my-cool-model model.safetensors # Upload directory at root huggingface-cli upload my-cool-model ./models -# Upload `my-cool-model/` directory if it exist, `./` otherwise +# Upload `my-cool-model/` directory if it exist, raise otherwise huggingface-cli upload my-cool-model ``` diff --git a/src/huggingface_hub/commands/upload.py b/src/huggingface_hub/commands/upload.py index aa2b822f6b..e468a38965 100644 --- a/src/huggingface_hub/commands/upload.py +++ b/src/huggingface_hub/commands/upload.py @@ -21,8 +21,7 @@ # Upload file (explicit) huggingface-cli upload my-cool-model ./my-cool-model.safetensors model.safetensors - # Upload directory (implicit). - # If `my-cool-model` is a directory, it will be uploaded. Otherwise, the current directory is uploaded. + # Upload directory (implicit). If `my-cool-model/` is a directory it will be uploaded, otherwise an exception is raised. huggingface-cli upload my-cool-model # Upload directory (explicit) @@ -156,9 +155,9 @@ def __init__(self, args: Namespace) -> None: self.local_path = repo_name self.path_in_repo = "." elif args.local_path is None: - # Implicit case 3: user provided only a repo_id that does not match a local file or folder => upload the current directory at root - self.local_path = "." - self.path_in_repo = "." + # Implicit case 3: user provided only a repo_id that does not match a local file or folder + # => the user must explicitly provide a local_path => raise exception + raise ValueError(f"'{repo_name}' is not a local file or folder. Please set `local_path` explicitly.") elif args.path_in_repo is None and os.path.isfile(args.local_path): # Explicit local path to file, no path in repo => upload it at root with same name self.local_path = args.local_path diff --git a/tests/test_cli.py b/tests/test_cli.py index c80b58f835..d246dc4f27 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -169,12 +169,10 @@ def test_upload_implicit_local_path_when_org_repo(self) -> None: self.assertEqual(cmd.path_in_repo, ".") def test_upload_implicit_local_path_otherwise(self) -> None: - with tmp_current_directory(): - cmd = UploadCommand(self.parser.parse_args(["upload", "my-cool-model"])) - - # No folder or file has the same name as the repo => upload entire folder ("./") at the root of the repo - self.assertEqual(cmd.local_path, ".") - self.assertEqual(cmd.path_in_repo, ".") + # No folder or file has the same name as the repo => raise exception + with self.assertRaises(ValueError): + with tmp_current_directory(): + UploadCommand(self.parser.parse_args(["upload", "my-cool-model"])) @xfail_on_windows(reason="No implicit path on Windows") def test_upload_explicit_local_path_to_folder_implicit_path_in_repo(self) -> None: @@ -202,17 +200,17 @@ def test_upload_explicit_paths(self) -> None: def test_every_must_be_positive(self) -> None: with self.assertRaises(ValueError): - UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, "--every", "0"])) + UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, ".", "--every", "0"])) with self.assertRaises(ValueError): - UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, "--every", "-10"])) + UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, ".", "--every", "-10"])) def test_every_as_int(self) -> None: - cmd = UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, "--every", "10"])) + cmd = UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, ".", "--every", "10"])) self.assertEqual(cmd.every, 10) def test_every_as_float(self) -> None: - cmd = UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, "--every", "0.5"])) + cmd = UploadCommand(self.parser.parse_args(["upload", DUMMY_MODEL_ID, ".", "--every", "0.5"])) self.assertEqual(cmd.every, 0.5) @patch("huggingface_hub.commands.upload.upload_folder") @@ -510,5 +508,9 @@ def tmp_current_directory() -> Generator[str, None, None]: with SoftTemporaryDirectory() as tmp_dir: cwd = os.getcwd() os.chdir(tmp_dir) - yield tmp_dir - os.chdir(cwd) + try: + yield tmp_dir + except: + raise + finally: + os.chdir(cwd) From ad4c6cc3393dacbcea29eeb1d1b5ff01e6421fd5 Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Wed, 6 Sep 2023 13:22:25 +0200 Subject: [PATCH 14/16] fix tests --- src/huggingface_hub/commands/download.py | 3 +++ src/huggingface_hub/commands/upload.py | 1 + 2 files changed, 4 insertions(+) diff --git a/src/huggingface_hub/commands/download.py b/src/huggingface_hub/commands/download.py index a4060199de..2171857b4e 100644 --- a/src/huggingface_hub/commands/download.py +++ b/src/huggingface_hub/commands/download.py @@ -39,6 +39,7 @@ from argparse import Namespace, _SubParsersAction from typing import List, Literal, Optional, Union +from huggingface_hub import logging from huggingface_hub._snapshot_download import snapshot_download from huggingface_hub.commands import BaseHuggingfaceCLICommand from huggingface_hub.file_download import hf_hub_download @@ -151,7 +152,9 @@ def run(self) -> None: print(self._download()) # Print path to downloaded files enable_progress_bars() else: + logging.set_verbosity_info() print(self._download()) # Print path to downloaded files + logging.set_verbosity_warning() def _download(self) -> str: # Warn user if patterns are ignored diff --git a/src/huggingface_hub/commands/upload.py b/src/huggingface_hub/commands/upload.py index e468a38965..2fa6e70445 100644 --- a/src/huggingface_hub/commands/upload.py +++ b/src/huggingface_hub/commands/upload.py @@ -181,6 +181,7 @@ def run(self) -> None: else: logging.set_verbosity_info() print(self._upload()) + logging.set_verbosity_warning() def _upload(self) -> str: if os.path.isfile(self.local_path): From f163f7a945b62e2492aee58eb9d138e3d56eb4ce Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Wed, 6 Sep 2023 13:52:09 +0200 Subject: [PATCH 15/16] xnotfail on windows --- tests/test_cli.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index d246dc4f27..6b83697424 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -174,7 +174,6 @@ def test_upload_implicit_local_path_otherwise(self) -> None: with tmp_current_directory(): UploadCommand(self.parser.parse_args(["upload", "my-cool-model"])) - @xfail_on_windows(reason="No implicit path on Windows") def test_upload_explicit_local_path_to_folder_implicit_path_in_repo(self) -> None: with tmp_current_directory() as cache_dir: folder_path = Path(cache_dir) / "path" / "to" / "folder" @@ -183,7 +182,6 @@ def test_upload_explicit_local_path_to_folder_implicit_path_in_repo(self) -> Non self.assertEqual(cmd.local_path, "./path/to/folder") self.assertEqual(cmd.path_in_repo, ".") # Always upload the folder at the root of the repo - @xfail_on_windows(reason="No implicit path on Windows") def test_upload_explicit_local_path_to_file_implicit_path_in_repo(self) -> None: with tmp_current_directory() as cache_dir: file_path = Path(cache_dir) / "path" / "to" / "file.txt" From 49b43c5ac0df8a6b6748903e331a1ec9fcdbdd80 Mon Sep 17 00:00:00 2001 From: Lucain Pouget Date: Wed, 6 Sep 2023 14:22:14 +0200 Subject: [PATCH 16/16] style --- tests/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 6b83697424..4117eb86f0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -13,7 +13,7 @@ from huggingface_hub.commands.upload import UploadCommand from huggingface_hub.utils import SoftTemporaryDirectory, capture_output -from .testing_utils import DUMMY_MODEL_ID, xfail_on_windows +from .testing_utils import DUMMY_MODEL_ID class TestCacheCommand(unittest.TestCase):