From 4fe7f640e4147e05688b682ec5e67662090cbbdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Thu, 26 Oct 2023 11:46:29 +0300 Subject: [PATCH 01/10] Add expire, content_* args --- CHANGELOG.md | 1 + b2/console_tool.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39dacba07..5c6d8d2f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +* Add `--expires`, `--content-disposition`, `--content-encoding`, `--content-language` options to subcommands `upload-file`, `upload-unbound-stream`, `copy-file-by-id` ### Infrastructure * Fix `docker run` example in README.md diff --git a/b2/console_tool.py b/b2/console_tool.py index 7774ad053..c252a35c6 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -1053,6 +1053,21 @@ def _setup_parser(cls, parser): parser.add_argument('--metadataDirective', default=None, help=argparse.SUPPRESS) parser.add_argument('--contentType') parser.add_argument('--range', type=parse_range) + parser.add_argument( + '--cache-control', help='Add Cache-Control header' # TODO(vbaltrusaitis-reef): better description + ) + parser.add_argument( + '--content-disposition', help='Add Content-Disposition header' # TODO(vbaltrusaitis-reef): better description + ) + parser.add_argument( + '--content-language', help='Add Content-Language header' # TODO(vbaltrusaitis-reef): better description + ) + parser.add_argument( + '--content-encoding', help='Add Content-Encoding header' # TODO(vbaltrusaitis-reef): better description + ) + parser.add_argument( + '--expires', help='Add Expires header' # TODO(vbaltrusaitis-reef): better description + ) info_group = parser.add_mutually_exclusive_group() @@ -1111,6 +1126,11 @@ def run(self, args): file_retention=file_retention, source_file_info=source_file_info, source_content_type=source_content_type, + cache_control=args.cache_control, + expires=args.expires, + content_disposition=args.content_disposition, + content_encoding=args.content_encoding, + content_language=args.content_language, ) self._print_json(file_version) return 0 @@ -2889,6 +2909,18 @@ def _setup_parser(cls, parser): '--sha1', help="SHA-1 of the data being uploaded for verifying file integrity" ) parser.add_argument('--cache-control', default=None) + parser.add_argument( + '--content-disposition', help='Add Content-Disposition header' # TODO(vbaltrusaitis-reef): better description + ) + parser.add_argument( + '--content-language', help='Add Content-Language header' # TODO(vbaltrusaitis-reef): better description + ) + parser.add_argument( + '--content-encoding', help='Add Content-Encoding header' # TODO(vbaltrusaitis-reef): better description + ) + parser.add_argument( + '--expires', help='Add Expires header' # TODO(vbaltrusaitis-reef): better description + ) parser.add_argument( '--info', action='append', @@ -2940,12 +2972,20 @@ def get_execute_kwargs(self, args) -> dict: self.api.get_bucket_by_name(args.bucketName), "cache_control": args.cache_control, + "content_disposition": + args.content_disposition, + "content_encoding": + args.content_encoding, + "content_language": + args.content_language, "content_type": args.contentType, "custom_upload_timestamp": args.custom_upload_timestamp, "encryption": self._get_destination_sse_setting(args), + "expires": + args.expires, "file_info": file_infos, "file_name": From c71a7f5fff36b574baf03ae3c117eac819a98674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Sun, 19 Nov 2023 17:56:20 +0200 Subject: [PATCH 02/10] Add header arg descriptions and tests --- CHANGELOG.md | 2 + b2/console_tool.py | 24 ++++++----- requirements.txt | 2 +- test/integration/test_b2_command_line.py | 50 ++++++++++++++++++++++ test/unit/console_tool/test_upload_file.py | 10 ++++- 5 files changed, 76 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c6d8d2f3..e6774125a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] + +### Added * Add `--expires`, `--content-disposition`, `--content-encoding`, `--content-language` options to subcommands `upload-file`, `upload-unbound-stream`, `copy-file-by-id` ### Infrastructure diff --git a/b2/console_tool.py b/b2/console_tool.py index c252a35c6..d009cf557 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -1054,19 +1054,19 @@ def _setup_parser(cls, parser): parser.add_argument('--contentType') parser.add_argument('--range', type=parse_range) parser.add_argument( - '--cache-control', help='Add Cache-Control header' # TODO(vbaltrusaitis-reef): better description + '--cache-control', help="optional Cache-Control header, value based on RFC 2616 section 14.9, example: 'public, max-age=86400')" ) parser.add_argument( - '--content-disposition', help='Add Content-Disposition header' # TODO(vbaltrusaitis-reef): better description + '--content-disposition', help="optional Content-Disposition header, value based on RFC 2616 section 19.5.1, example: 'attachment; filename=\"fname.ext\"'" ) parser.add_argument( - '--content-language', help='Add Content-Language header' # TODO(vbaltrusaitis-reef): better description + '--content-encoding', help="optional Content-Encoding header, value based on RFC 2616 section 14.11, example: 'gzip'" ) parser.add_argument( - '--content-encoding', help='Add Content-Encoding header' # TODO(vbaltrusaitis-reef): better description + '--content-language', help="optional Content-Language header, value based on RFC 2616 section 14.12, example: 'mi, EN_US'" ) parser.add_argument( - '--expires', help='Add Expires header' # TODO(vbaltrusaitis-reef): better description + '--expires', help="optional Expires header, value based on RFC 2616 section 14.21, example: 'Thu, 01 Dec 2050 16:00:00 GMT'" ) info_group = parser.add_mutually_exclusive_group() @@ -1381,6 +1381,8 @@ def _print_download_info(self, downloaded_file: DownloadedFile): 'Legal hold', self._represent_legal_hold(download_version.legal_hold) ) for label, attr_name in [ + ('CacheControl', 'cache_control'), + ('Expires', 'expires'), ('ContentDisposition', 'content_disposition'), ('ContentLanguage', 'content_language'), ('ContentEncoding', 'content_encoding'), @@ -2908,18 +2910,20 @@ def _setup_parser(cls, parser): parser.add_argument( '--sha1', help="SHA-1 of the data being uploaded for verifying file integrity" ) - parser.add_argument('--cache-control', default=None) parser.add_argument( - '--content-disposition', help='Add Content-Disposition header' # TODO(vbaltrusaitis-reef): better description + '--cache-control', help="optional Cache-Control header, value based on RFC 2616 section 14.9, example: 'public, max-age=86400')" ) parser.add_argument( - '--content-language', help='Add Content-Language header' # TODO(vbaltrusaitis-reef): better description + '--content-disposition', help="optional Content-Disposition header, value based on RFC 2616 section 19.5.1, example: 'attachment; filename=\"fname.ext\"'" ) parser.add_argument( - '--content-encoding', help='Add Content-Encoding header' # TODO(vbaltrusaitis-reef): better description + '--content-encoding', help="optional Content-Encoding header, value based on RFC 2616 section 14.11, example: 'gzip'" ) parser.add_argument( - '--expires', help='Add Expires header' # TODO(vbaltrusaitis-reef): better description + '--content-language', help="optional Content-Language header, value based on RFC 2616 section 14.12, example: 'mi, EN_US'" + ) + parser.add_argument( + '--expires', help="optional Expires header, value based on RFC 2616 section 14.21, example: 'Thu, 01 Dec 2050 16:00:00 GMT'" ) parser.add_argument( '--info', diff --git a/requirements.txt b/requirements.txt index 68b52fc05..c1503a0e4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ argcomplete>=2,<4 arrow>=1.0.2,<2.0.0 -b2sdk>=1.25.0,<2 +b2sdk @ git+https://github.com/Backblaze/b2-sdk-python@c0507d1b54e376ad5206ab253b028963d4cc5bdc docutils>=0.18.1 idna~=3.4; platform_system == 'Java' importlib-metadata~=3.3; python_version < '3.8' diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index 2cb24c8f7..a2978e7f9 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -2690,3 +2690,53 @@ def test_cat(b2_tool, bucket_name, sample_filepath, tmp_path, uploaded_sample_fi ).replace("\r", "") == sample_filepath.read_text() assert b2_tool.should_succeed(['cat', f"b2id://{uploaded_sample_file['fileId']}" ],).replace("\r", "") == sample_filepath.read_text() + +def test_header_arguments(b2_tool, bucket_name, sample_filepath, tmp_path): + args = [ + '--cache-control', 'max-age=3600', + '--content-disposition', 'attachment', + '--content-encoding', 'gzip', + '--content-language', 'en', + '--expires', 'Thu, 01 Dec 2050 16:00:00 GMT', + ] + expected_file_info = { + 'b2-cache-control': 'max-age=3600', + 'b2-content-disposition': 'attachment', + 'b2-content-encoding': 'gzip', + 'b2-content-language': 'en', + 'b2-expires': 'Thu, 01 Dec 2050 16:00:00 GMT', + } + def assert_expected(file_info, expected=expected_file_info): + for key, val in expected.items(): + assert file_info[key] == val + + file_version = b2_tool.should_succeed_json([ + 'upload-file', + '--quiet', + '--noProgress', + bucket_name, + str(sample_filepath), + 'sample_file', + *args, + ]) + assert_expected(file_version['fileInfo']) + + copied_version = b2_tool.should_succeed_json([ + 'copy-file-by-id', + '--quiet', + *args, + '--contentType', 'text/plain', + file_version['fileId'], + bucket_name, + 'copied_file' + ]) + assert_expected(copied_version['fileInfo']) + + download_output = b2_tool.should_succeed([ + 'download-file-by-id', file_version['fileId'], tmp_path / 'downloaded_file' + ]) + assert re.search(r'CacheControl: *max-age=3600', download_output) + assert re.search(r'ContentDisposition: *attachment', download_output) + assert re.search(r'ContentEncoding: *gzip', download_output) + assert re.search(r'ContentLanguage: *en', download_output) + assert re.search(r'Expires: *Thu, 01 Dec 2050 16:00:00 GMT', download_output) diff --git a/test/unit/console_tool/test_upload_file.py b/test/unit/console_tool/test_upload_file.py index c9da4d322..bc5803538 100644 --- a/test/unit/console_tool/test_upload_file.py +++ b/test/unit/console_tool/test_upload_file.py @@ -13,7 +13,7 @@ import b2 -def test_upload_file__file_info_src_last_modified_millis(b2_cli, bucket, tmpdir): +def test_upload_file__file_info_src_last_modified_millis_and_headers(b2_cli, bucket, tmpdir): """Test upload_file supports manually specifying file info src_last_modified_millis""" filename = 'file1.txt' content = 'hello world' @@ -24,6 +24,11 @@ def test_upload_file__file_info_src_last_modified_millis(b2_cli, bucket, tmpdir) "action": "upload", "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", "fileInfo": { + "b2-cache-control": "max-age=3600", + "b2-expires": "Thu, 01 Dec 2050 16:00:00 GMT", + "b2-content-language": "en", + "b2-content-disposition": "attachment", + "b2-content-encoding": "gzip", "src_last_modified_millis": "1" }, "fileName": filename, @@ -32,6 +37,9 @@ def test_upload_file__file_info_src_last_modified_millis(b2_cli, bucket, tmpdir) b2_cli.run( [ 'upload-file', '--noProgress', '--info=src_last_modified_millis=1', 'my-bucket', + '--cache-control', 'max-age=3600', '--expires', 'Thu, 01 Dec 2050 16:00:00 GMT', + '--content-language', 'en', '--content-disposition', 'attachment', + '--content-encoding', 'gzip', str(local_file1), 'file1.txt' ], expected_json_in_stdout=expected_json, From 329883eeccdddf06fa74d8c317b421cc609e558d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Sun, 19 Nov 2023 17:59:53 +0200 Subject: [PATCH 03/10] Run formatter --- b2/console_tool.py | 40 ++++++++++++++----- test/integration/test_b2_command_line.py | 45 ++++++++++++---------- test/unit/console_tool/test_upload_file.py | 21 +++++----- 3 files changed, 65 insertions(+), 41 deletions(-) diff --git a/b2/console_tool.py b/b2/console_tool.py index d009cf557..9bfabd9fd 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -1054,19 +1054,29 @@ def _setup_parser(cls, parser): parser.add_argument('--contentType') parser.add_argument('--range', type=parse_range) parser.add_argument( - '--cache-control', help="optional Cache-Control header, value based on RFC 2616 section 14.9, example: 'public, max-age=86400')" + '--cache-control', + help= + "optional Cache-Control header, value based on RFC 2616 section 14.9, example: 'public, max-age=86400')" ) parser.add_argument( - '--content-disposition', help="optional Content-Disposition header, value based on RFC 2616 section 19.5.1, example: 'attachment; filename=\"fname.ext\"'" + '--content-disposition', + help= + "optional Content-Disposition header, value based on RFC 2616 section 19.5.1, example: 'attachment; filename=\"fname.ext\"'" ) parser.add_argument( - '--content-encoding', help="optional Content-Encoding header, value based on RFC 2616 section 14.11, example: 'gzip'" + '--content-encoding', + help= + "optional Content-Encoding header, value based on RFC 2616 section 14.11, example: 'gzip'" ) parser.add_argument( - '--content-language', help="optional Content-Language header, value based on RFC 2616 section 14.12, example: 'mi, EN_US'" + '--content-language', + help= + "optional Content-Language header, value based on RFC 2616 section 14.12, example: 'mi, EN_US'" ) parser.add_argument( - '--expires', help="optional Expires header, value based on RFC 2616 section 14.21, example: 'Thu, 01 Dec 2050 16:00:00 GMT'" + '--expires', + help= + "optional Expires header, value based on RFC 2616 section 14.21, example: 'Thu, 01 Dec 2050 16:00:00 GMT'" ) info_group = parser.add_mutually_exclusive_group() @@ -2911,19 +2921,29 @@ def _setup_parser(cls, parser): '--sha1', help="SHA-1 of the data being uploaded for verifying file integrity" ) parser.add_argument( - '--cache-control', help="optional Cache-Control header, value based on RFC 2616 section 14.9, example: 'public, max-age=86400')" + '--cache-control', + help= + "optional Cache-Control header, value based on RFC 2616 section 14.9, example: 'public, max-age=86400')" ) parser.add_argument( - '--content-disposition', help="optional Content-Disposition header, value based on RFC 2616 section 19.5.1, example: 'attachment; filename=\"fname.ext\"'" + '--content-disposition', + help= + "optional Content-Disposition header, value based on RFC 2616 section 19.5.1, example: 'attachment; filename=\"fname.ext\"'" ) parser.add_argument( - '--content-encoding', help="optional Content-Encoding header, value based on RFC 2616 section 14.11, example: 'gzip'" + '--content-encoding', + help= + "optional Content-Encoding header, value based on RFC 2616 section 14.11, example: 'gzip'" ) parser.add_argument( - '--content-language', help="optional Content-Language header, value based on RFC 2616 section 14.12, example: 'mi, EN_US'" + '--content-language', + help= + "optional Content-Language header, value based on RFC 2616 section 14.12, example: 'mi, EN_US'" ) parser.add_argument( - '--expires', help="optional Expires header, value based on RFC 2616 section 14.21, example: 'Thu, 01 Dec 2050 16:00:00 GMT'" + '--expires', + help= + "optional Expires header, value based on RFC 2616 section 14.21, example: 'Thu, 01 Dec 2050 16:00:00 GMT'" ) parser.add_argument( '--info', diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index a2978e7f9..536d538d7 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -2691,7 +2691,9 @@ def test_cat(b2_tool, bucket_name, sample_filepath, tmp_path, uploaded_sample_fi assert b2_tool.should_succeed(['cat', f"b2id://{uploaded_sample_file['fileId']}" ],).replace("\r", "") == sample_filepath.read_text() + def test_header_arguments(b2_tool, bucket_name, sample_filepath, tmp_path): + # yapf: disable args = [ '--cache-control', 'max-age=3600', '--content-disposition', 'attachment', @@ -2699,6 +2701,7 @@ def test_header_arguments(b2_tool, bucket_name, sample_filepath, tmp_path): '--content-language', 'en', '--expires', 'Thu, 01 Dec 2050 16:00:00 GMT', ] + # yapf: enable expected_file_info = { 'b2-cache-control': 'max-age=3600', 'b2-content-disposition': 'attachment', @@ -2706,35 +2709,35 @@ def test_header_arguments(b2_tool, bucket_name, sample_filepath, tmp_path): 'b2-content-language': 'en', 'b2-expires': 'Thu, 01 Dec 2050 16:00:00 GMT', } + def assert_expected(file_info, expected=expected_file_info): for key, val in expected.items(): assert file_info[key] == val - file_version = b2_tool.should_succeed_json([ - 'upload-file', - '--quiet', - '--noProgress', - bucket_name, - str(sample_filepath), - 'sample_file', - *args, - ]) + file_version = b2_tool.should_succeed_json( + [ + 'upload-file', + '--quiet', + '--noProgress', + bucket_name, + str(sample_filepath), + 'sample_file', + *args, + ] + ) assert_expected(file_version['fileInfo']) - copied_version = b2_tool.should_succeed_json([ - 'copy-file-by-id', - '--quiet', - *args, - '--contentType', 'text/plain', - file_version['fileId'], - bucket_name, - 'copied_file' - ]) + copied_version = b2_tool.should_succeed_json( + [ + 'copy-file-by-id', '--quiet', *args, '--contentType', 'text/plain', + file_version['fileId'], bucket_name, 'copied_file' + ] + ) assert_expected(copied_version['fileInfo']) - download_output = b2_tool.should_succeed([ - 'download-file-by-id', file_version['fileId'], tmp_path / 'downloaded_file' - ]) + download_output = b2_tool.should_succeed( + ['download-file-by-id', file_version['fileId'], tmp_path / 'downloaded_file'] + ) assert re.search(r'CacheControl: *max-age=3600', download_output) assert re.search(r'ContentDisposition: *attachment', download_output) assert re.search(r'ContentEncoding: *gzip', download_output) diff --git a/test/unit/console_tool/test_upload_file.py b/test/unit/console_tool/test_upload_file.py index bc5803538..095b1a6bc 100644 --- a/test/unit/console_tool/test_upload_file.py +++ b/test/unit/console_tool/test_upload_file.py @@ -23,14 +23,15 @@ def test_upload_file__file_info_src_last_modified_millis_and_headers(b2_cli, buc expected_json = { "action": "upload", "contentSha1": "2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", - "fileInfo": { - "b2-cache-control": "max-age=3600", - "b2-expires": "Thu, 01 Dec 2050 16:00:00 GMT", - "b2-content-language": "en", - "b2-content-disposition": "attachment", - "b2-content-encoding": "gzip", - "src_last_modified_millis": "1" - }, + "fileInfo": + { + "b2-cache-control": "max-age=3600", + "b2-expires": "Thu, 01 Dec 2050 16:00:00 GMT", + "b2-content-language": "en", + "b2-content-disposition": "attachment", + "b2-content-encoding": "gzip", + "src_last_modified_millis": "1" + }, "fileName": filename, "size": len(content), } @@ -38,8 +39,8 @@ def test_upload_file__file_info_src_last_modified_millis_and_headers(b2_cli, buc [ 'upload-file', '--noProgress', '--info=src_last_modified_millis=1', 'my-bucket', '--cache-control', 'max-age=3600', '--expires', 'Thu, 01 Dec 2050 16:00:00 GMT', - '--content-language', 'en', '--content-disposition', 'attachment', - '--content-encoding', 'gzip', + '--content-language', 'en', '--content-disposition', 'attachment', '--content-encoding', + 'gzip', str(local_file1), 'file1.txt' ], expected_json_in_stdout=expected_json, From 97097f9a58778ca4b4acf2f24f654005f4de5b65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Sun, 19 Nov 2023 18:04:17 +0200 Subject: [PATCH 04/10] Fix --content-language example --- b2/console_tool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/b2/console_tool.py b/b2/console_tool.py index 9bfabd9fd..53bb63481 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -1071,7 +1071,7 @@ def _setup_parser(cls, parser): parser.add_argument( '--content-language', help= - "optional Content-Language header, value based on RFC 2616 section 14.12, example: 'mi, EN_US'" + "optional Content-Language header, value based on RFC 2616 section 14.12, example: 'mi, en'" ) parser.add_argument( '--expires', @@ -2938,7 +2938,7 @@ def _setup_parser(cls, parser): parser.add_argument( '--content-language', help= - "optional Content-Language header, value based on RFC 2616 section 14.12, example: 'mi, EN_US'" + "optional Content-Language header, value based on RFC 2616 section 14.12, example: 'mi, en'" ) parser.add_argument( '--expires', From 4c8c12c1f811203e168e81433a49400be8497571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Sun, 19 Nov 2023 23:43:25 +0200 Subject: [PATCH 05/10] Warn when both --info and explicit header args set the same value --- b2/console_tool.py | 135 ++++++++++++----------- test/integration/test_b2_command_line.py | 10 +- 2 files changed, 77 insertions(+), 68 deletions(-) diff --git a/b2/console_tool.py b/b2/console_tool.py index 53bb63481..5e1876f90 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -37,7 +37,7 @@ from concurrent.futures import Executor, Future, ThreadPoolExecutor from contextlib import suppress from enum import Enum -from typing import Any, BinaryIO, Dict, List, Optional, Tuple +from typing import Any, BinaryIO, Callable, Dict, List, Optional, Tuple import argcomplete import b2sdk @@ -393,6 +393,68 @@ def _get_file_retention_setting(cls, args): return FileRetentionSetting(file_retention_mode, args.retainUntil) +class HeaderFlagsMixin(Described): + @classmethod + def _setup_parser(cls, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + '--cache-control', + help= + "optional Cache-Control header, value based on RFC 2616 section 14.9, example: 'public, max-age=86400')" + ) + parser.add_argument( + '--content-disposition', + help= + "optional Content-Disposition header, value based on RFC 2616 section 19.5.1, example: 'attachment; filename=\"fname.ext\"'" + ) + parser.add_argument( + '--content-encoding', + help= + "optional Content-Encoding header, value based on RFC 2616 section 14.11, example: 'gzip'" + ) + parser.add_argument( + '--content-language', + help= + "optional Content-Language header, value based on RFC 2616 section 14.12, example: 'mi, en'" + ) + parser.add_argument( + '--expires', + help= + "optional Expires header, value based on RFC 2616 section 14.21, example: 'Thu, 01 Dec 2050 16:00:00 GMT'" + ) + super()._setup_parser(parser) + + def _file_info_with_header_args(self, args, file_info: dict[str, str] | None) -> dict[str, str] | None: + """Construct an updated file_info dictionary. + Print a warning if any of file_info items will be overwritten by explicit header arguments. + """ + add_file_info = {} + overwritten = [] + if args.cache_control is not None: + add_file_info['b2-cache-control'] = args.cache_control + if args.content_disposition is not None: + add_file_info['b2-content-disposition'] = args.content_disposition + if args.content_encoding is not None: + add_file_info['b2-content-encoding'] = args.content_encoding + if args.content_language is not None: + add_file_info['b2-content-language'] = args.content_language + if args.expires is not None: + add_file_info['b2-expires'] = args.expires + + for key, value in add_file_info.items(): + if file_info is not None and key in file_info and file_info[key] != value: + overwritten.append(key) + + if overwritten: + self._print_stderr( + 'The following file info items will be overwritten by explicit arguments:\n ' + + '\n '.join(f'{key} = {add_file_info[key]}' for key in overwritten) + ) + + if add_file_info: + return {**(file_info or {}), **add_file_info} + return file_info + + class LegalHoldMixin(Described): """ Setting legal holds requires the **writeFileLegalHolds** capability, and only works in bucket @@ -1010,7 +1072,7 @@ def run(self, args): @B2.register_subcommand class CopyFileById( - DestinationSseMixin, SourceSseMixin, FileRetentionSettingMixin, LegalHoldMixin, Command + HeaderFlagsMixin, DestinationSseMixin, SourceSseMixin, FileRetentionSettingMixin, LegalHoldMixin, Command ): """ Copy a file version to the given bucket (server-side, **not** via download+upload). @@ -1053,31 +1115,6 @@ def _setup_parser(cls, parser): parser.add_argument('--metadataDirective', default=None, help=argparse.SUPPRESS) parser.add_argument('--contentType') parser.add_argument('--range', type=parse_range) - parser.add_argument( - '--cache-control', - help= - "optional Cache-Control header, value based on RFC 2616 section 14.9, example: 'public, max-age=86400')" - ) - parser.add_argument( - '--content-disposition', - help= - "optional Content-Disposition header, value based on RFC 2616 section 19.5.1, example: 'attachment; filename=\"fname.ext\"'" - ) - parser.add_argument( - '--content-encoding', - help= - "optional Content-Encoding header, value based on RFC 2616 section 14.11, example: 'gzip'" - ) - parser.add_argument( - '--content-language', - help= - "optional Content-Language header, value based on RFC 2616 section 14.12, example: 'mi, en'" - ) - parser.add_argument( - '--expires', - help= - "optional Expires header, value based on RFC 2616 section 14.21, example: 'Thu, 01 Dec 2050 16:00:00 GMT'" - ) info_group = parser.add_mutually_exclusive_group() @@ -1097,6 +1134,7 @@ def run(self, args): file_infos = self._parse_file_infos(args.info) elif args.noInfo: file_infos = {} + file_infos = self._file_info_with_header_args(args, file_infos) if args.metadataDirective is not None: self._print_stderr( @@ -1136,11 +1174,6 @@ def run(self, args): file_retention=file_retention, source_file_info=source_file_info, source_content_type=source_content_type, - cache_control=args.cache_control, - expires=args.expires, - content_disposition=args.content_disposition, - content_encoding=args.content_encoding, - content_language=args.content_language, ) self._print_json(file_version) return 0 @@ -2892,6 +2925,7 @@ def _setup_parser(cls, parser): class UploadFileMixin( + HeaderFlagsMixin, MinPartSizeMixin, ThreadsMixin, ProgressMixin, @@ -2920,31 +2954,6 @@ def _setup_parser(cls, parser): parser.add_argument( '--sha1', help="SHA-1 of the data being uploaded for verifying file integrity" ) - parser.add_argument( - '--cache-control', - help= - "optional Cache-Control header, value based on RFC 2616 section 14.9, example: 'public, max-age=86400')" - ) - parser.add_argument( - '--content-disposition', - help= - "optional Content-Disposition header, value based on RFC 2616 section 19.5.1, example: 'attachment; filename=\"fname.ext\"'" - ) - parser.add_argument( - '--content-encoding', - help= - "optional Content-Encoding header, value based on RFC 2616 section 14.11, example: 'gzip'" - ) - parser.add_argument( - '--content-language', - help= - "optional Content-Language header, value based on RFC 2616 section 14.12, example: 'mi, en'" - ) - parser.add_argument( - '--expires', - help= - "optional Expires header, value based on RFC 2616 section 14.21, example: 'Thu, 01 Dec 2050 16:00:00 GMT'" - ) parser.add_argument( '--info', action='append', @@ -2991,25 +3000,17 @@ def get_execute_kwargs(self, args) -> dict: else: file_infos[SRC_LAST_MODIFIED_MILLIS] = str(int(mtime * 1000)) + file_infos = self._file_info_with_header_args(args, file_infos) + return { "bucket": self.api.get_bucket_by_name(args.bucketName), - "cache_control": - args.cache_control, - "content_disposition": - args.content_disposition, - "content_encoding": - args.content_encoding, - "content_language": - args.content_language, "content_type": args.contentType, "custom_upload_timestamp": args.custom_upload_timestamp, "encryption": self._get_destination_sse_setting(args), - "expires": - args.expires, "file_info": file_infos, "file_name": diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index 536d538d7..08a26dfab 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -2714,7 +2714,7 @@ def assert_expected(file_info, expected=expected_file_info): for key, val in expected.items(): assert file_info[key] == val - file_version = b2_tool.should_succeed_json( + status, stdout, stderr = b2_tool.execute( [ 'upload-file', '--quiet', @@ -2723,10 +2723,18 @@ def assert_expected(file_info, expected=expected_file_info): str(sample_filepath), 'sample_file', *args, + '--info', 'b2-content-disposition=will-be-overwritten', ] ) + assert status == 0 + file_version = json.loads(stdout) assert_expected(file_version['fileInfo']) + # Since we used both --info and --content-disposition to set b2-content-disposition, + # a warning should be emitted + assert 'will be overwritten' in stderr and 'b2-content-disposition = attachment' in stderr + + copied_version = b2_tool.should_succeed_json( [ 'copy-file-by-id', '--quiet', *args, '--contentType', 'text/plain', From 38006e0d47c685ae44c67d93b7848008b5ea6187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Mon, 20 Nov 2023 09:21:26 +0200 Subject: [PATCH 06/10] Run formatter --- b2/console_tool.py | 8 +++++--- test/integration/test_b2_command_line.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/b2/console_tool.py b/b2/console_tool.py index 5e1876f90..a1d9db12c 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -37,7 +37,7 @@ from concurrent.futures import Executor, Future, ThreadPoolExecutor from contextlib import suppress from enum import Enum -from typing import Any, BinaryIO, Callable, Dict, List, Optional, Tuple +from typing import Any, BinaryIO, Dict, List, Optional, Tuple import argcomplete import b2sdk @@ -423,7 +423,8 @@ def _setup_parser(cls, parser: argparse.ArgumentParser) -> None: ) super()._setup_parser(parser) - def _file_info_with_header_args(self, args, file_info: dict[str, str] | None) -> dict[str, str] | None: + def _file_info_with_header_args(self, args, + file_info: dict[str, str] | None) -> dict[str, str] | None: """Construct an updated file_info dictionary. Print a warning if any of file_info items will be overwritten by explicit header arguments. """ @@ -1072,7 +1073,8 @@ def run(self, args): @B2.register_subcommand class CopyFileById( - HeaderFlagsMixin, DestinationSseMixin, SourceSseMixin, FileRetentionSettingMixin, LegalHoldMixin, Command + HeaderFlagsMixin, DestinationSseMixin, SourceSseMixin, FileRetentionSettingMixin, + LegalHoldMixin, Command ): """ Copy a file version to the given bucket (server-side, **not** via download+upload). diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py index 08a26dfab..e62e359d3 100755 --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -2723,7 +2723,8 @@ def assert_expected(file_info, expected=expected_file_info): str(sample_filepath), 'sample_file', *args, - '--info', 'b2-content-disposition=will-be-overwritten', + '--info', + 'b2-content-disposition=will-be-overwritten', ] ) assert status == 0 @@ -2734,7 +2735,6 @@ def assert_expected(file_info, expected=expected_file_info): # a warning should be emitted assert 'will be overwritten' in stderr and 'b2-content-disposition = attachment' in stderr - copied_version = b2_tool.should_succeed_json( [ 'copy-file-by-id', '--quiet', *args, '--contentType', 'text/plain', From 540757917536646fb48ac56fea94b62e65ab8353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Mon, 20 Nov 2023 09:35:28 +0200 Subject: [PATCH 07/10] Fix type hints: import annotations form future --- b2/console_tool.py | 1 + 1 file changed, 1 insertion(+) diff --git a/b2/console_tool.py b/b2/console_tool.py index a1d9db12c..7ed2ecf26 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -9,6 +9,7 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### +from __future__ import annotations import argparse import base64 From 5a12858462502ce097f606c67f857e1934e52090 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Mon, 20 Nov 2023 09:41:37 +0200 Subject: [PATCH 08/10] Run formatter --- b2/console_tool.py | 60 +++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/b2/console_tool.py b/b2/console_tool.py index 7ed2ecf26..778ef045e 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -38,7 +38,7 @@ from concurrent.futures import Executor, Future, ThreadPoolExecutor from contextlib import suppress from enum import Enum -from typing import Any, BinaryIO, Dict, List, Optional, Tuple +from typing import Any, BinaryIO, List import argcomplete import b2sdk @@ -760,12 +760,12 @@ def _print_json(self, data) -> None: json.dumps(data, indent=4, sort_keys=True, cls=B2CliJsonEncoder), enforce_output=True ) - def _print(self, *args, enforce_output: bool = False, end: Optional[str] = None) -> None: + def _print(self, *args, enforce_output: bool = False, end: str | None = None) -> None: return self._print_standard_descriptor( self.stdout, "stdout", *args, enforce_output=enforce_output, end=end ) - def _print_stderr(self, *args, end: Optional[str] = None) -> None: + def _print_stderr(self, *args, end: str | None = None) -> None: return self._print_standard_descriptor( self.stderr, "stderr", *args, enforce_output=True, end=end ) @@ -776,7 +776,7 @@ def _print_standard_descriptor( descriptor_name: str, *args, enforce_output: bool = False, - end: Optional[str] = None, + end: str | None = None, ) -> None: """ Prints to fd, unless quiet is set. @@ -797,7 +797,7 @@ def _print_helper( descriptor_encoding: str, descriptor_name: str, *args, - end: Optional[str] = None + end: str | None = None ): try: descriptor.write(' '.join(args)) @@ -1181,7 +1181,7 @@ def run(self, args): self._print_json(file_version) return 0 - def _is_ssec(self, encryption: Optional[EncryptionSetting]): + def _is_ssec(self, encryption: EncryptionSetting | None): if encryption is not None and encryption.mode == EncryptionMode.SSE_C: return True return False @@ -1189,12 +1189,12 @@ def _is_ssec(self, encryption: Optional[EncryptionSetting]): def _determine_source_metadata( self, source_file_id: str, - destination_encryption: Optional[EncryptionSetting], - source_encryption: Optional[EncryptionSetting], - target_file_info: Optional[dict], - target_content_type: Optional[str], + destination_encryption: EncryptionSetting | None, + source_encryption: EncryptionSetting | None, + target_file_info: dict | None, + target_content_type: str | None, fetch_if_necessary: bool, - ) -> Tuple[Optional[dict], Optional[str]]: + ) -> tuple[dict | None, str | None]: """Determine if source file metadata is necessary to perform the copy - due to sse_c_key_id""" if not self._is_ssec(source_encryption) and not self._is_ssec( destination_encryption @@ -2065,7 +2065,7 @@ def _print_file_version( self, args, file_version: FileVersion, - folder_name: Optional[str], + folder_name: str | None, ) -> None: self._print(folder_name or file_version.file_name) @@ -2172,7 +2172,7 @@ def _print_file_version( self, args, file_version: FileVersion, - folder_name: Optional[str], + folder_name: str | None, ) -> None: if not args.long: super()._print_file_version(args, file_version, folder_name) @@ -2294,7 +2294,7 @@ class SubmitThread(threading.Thread): def __init__( self, - runner: 'Rm', + runner: Rm, args: argparse.Namespace, messages_queue: queue.Queue, reporter: ProgressReport, @@ -3035,7 +3035,7 @@ def get_execute_kwargs(self, args) -> dict: } @abstractmethod - def execute_operation(self, **kwargs) -> 'b2sdk.file_version.FileVersion': + def execute_operation(self, **kwargs) -> b2sdk.file_version.FileVersion: raise NotImplementedError def upload_file_kwargs_to_unbound_upload(self, **kwargs): @@ -3047,7 +3047,7 @@ def upload_file_kwargs_to_unbound_upload(self, **kwargs): kwargs["read_size"] = kwargs["min_part_size"] or DEFAULT_MIN_PART_SIZE return kwargs - def get_input_stream(self, filename: str) -> 'str | int | io.BinaryIO': + def get_input_stream(self, filename: str) -> str | int | io.BinaryIO: """Get input stream IF filename points to a FIFO or stdin.""" if filename == "-": if os.path.exists('-'): @@ -3062,7 +3062,7 @@ def get_input_stream(self, filename: str) -> 'str | int | io.BinaryIO': raise self.NotAnInputStream() def file_identifier_to_read_stream( - self, file_id: 'str | int | BinaryIO', buffering + self, file_id: str | int | BinaryIO, buffering ) -> BinaryIO: if isinstance(file_id, (str, int)): return open( @@ -3387,7 +3387,7 @@ def run(self, args): return 0 @classmethod - def alter_rule_by_name(cls, bucket: Bucket, name: str) -> Tuple[bool, bool]: + def alter_rule_by_name(cls, bucket: Bucket, name: str) -> tuple[bool, bool]: """ returns False if rule could not be found """ if not bucket.replication or not bucket.replication.rules: return False, False @@ -3424,7 +3424,7 @@ def alter_rule_by_name(cls, bucket: Bucket, name: str) -> Tuple[bool, bool]: @classmethod @abstractmethod - def alter_one_rule(cls, rule: ReplicationRule) -> Optional[ReplicationRule]: + def alter_one_rule(cls, rule: ReplicationRule) -> ReplicationRule | None: """ return None to delete a rule """ pass @@ -3441,7 +3441,7 @@ class ReplicationDelete(ReplicationRuleChanger): """ @classmethod - def alter_one_rule(cls, rule: ReplicationRule) -> Optional[ReplicationRule]: + def alter_one_rule(cls, rule: ReplicationRule) -> ReplicationRule | None: """ return None to delete rule """ return None @@ -3458,7 +3458,7 @@ class ReplicationPause(ReplicationRuleChanger): """ @classmethod - def alter_one_rule(cls, rule: ReplicationRule) -> Optional[ReplicationRule]: + def alter_one_rule(cls, rule: ReplicationRule) -> ReplicationRule | None: """ return None to delete rule """ rule.is_enabled = False return rule @@ -3476,7 +3476,7 @@ class ReplicationUnpause(ReplicationRuleChanger): """ @classmethod - def alter_one_rule(cls, rule: ReplicationRule) -> Optional[ReplicationRule]: + def alter_one_rule(cls, rule: ReplicationRule) -> ReplicationRule | None: """ return None to delete rule """ rule.is_enabled = True return rule @@ -3576,9 +3576,9 @@ def run(self, args): @classmethod def get_results_for_rule( - cls, bucket: Bucket, rule: ReplicationRule, destination_api: Optional[B2Api], + cls, bucket: Bucket, rule: ReplicationRule, destination_api: B2Api | None, scan_destination: bool, quiet: bool - ) -> List[dict]: + ) -> list[dict]: monitor = ReplicationMonitor( bucket=bucket, rule=rule, @@ -3595,7 +3595,7 @@ def get_results_for_rule( ] @classmethod - def filter_results_columns(cls, results: List[dict], columns: List[str]) -> List[dict]: + def filter_results_columns(cls, results: list[dict], columns: list[str]) -> list[dict]: return [{key: result[key] for key in columns} for result in results] @classmethod @@ -3611,10 +3611,10 @@ def to_human_readable(cls, value: Any) -> str: return str(value) - def output_json(self, results: Dict[str, List[dict]]) -> None: + def output_json(self, results: dict[str, list[dict]]) -> None: self._print_json(results) - def output_console(self, results: Dict[str, List[dict]]) -> None: + def output_console(self, results: dict[str, list[dict]]) -> None: for rule_name, rule_results in results.items(): self._print(f'Replication "{rule_name}":') rule_results = [ @@ -3626,7 +3626,7 @@ def output_console(self, results: Dict[str, List[dict]]) -> None: ] self._print(tabulate(rule_results, headers='keys', tablefmt='grid')) - def output_csv(self, results: Dict[str, List[dict]]) -> None: + def output_csv(self, results: dict[str, list[dict]]) -> None: rows = [] @@ -3803,7 +3803,7 @@ def _put_license_text_for_packages(self, stream: io.StringIO): stream.write(str(summary_table)) @classmethod - def _get_licenses_dicts(cls) -> List[Dict]: + def _get_licenses_dicts(cls) -> list[dict]: assert piplicenses, 'In order to run this command, you need to install the `license` extra: pip install b2[license]' pipdeptree_run = subprocess.run( ["pipdeptree", "--json", "-p", "b2"], @@ -3910,7 +3910,7 @@ class ConsoleTool: Uses a ``b2sdk.SqlitedAccountInfo`` object to keep account data between runs. """ - def __init__(self, b2_api: Optional[B2Api], stdout, stderr): + def __init__(self, b2_api: B2Api | None, stdout, stderr): self.api = b2_api self.stdout = stdout self.stderr = stderr From f202994f2f4410d27d1a9f16d2eaf1402a54633c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Mon, 20 Nov 2023 09:43:38 +0200 Subject: [PATCH 09/10] Run formatter (not idempotent?) --- b2/console_tool.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/b2/console_tool.py b/b2/console_tool.py index 778ef045e..4568c54ee 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -3061,9 +3061,7 @@ def get_input_stream(self, filename: str) -> str | int | io.BinaryIO: raise self.NotAnInputStream() - def file_identifier_to_read_stream( - self, file_id: str | int | BinaryIO, buffering - ) -> BinaryIO: + def file_identifier_to_read_stream(self, file_id: str | int | BinaryIO, buffering) -> BinaryIO: if isinstance(file_id, (str, int)): return open( file_id, From a9601271533ab06597b63515c9b04144116130fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vykintas=20Baltru=C5=A1aitis?= Date: Wed, 22 Nov 2023 19:50:48 +0200 Subject: [PATCH 10/10] Use released b2sdk version 1.26.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c1503a0e4..79f6c88b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ argcomplete>=2,<4 arrow>=1.0.2,<2.0.0 -b2sdk @ git+https://github.com/Backblaze/b2-sdk-python@c0507d1b54e376ad5206ab253b028963d4cc5bdc +b2sdk>=1.26.0,<2 docutils>=0.18.1 idna~=3.4; platform_system == 'Java' importlib-metadata~=3.3; python_version < '3.8'