From 319dc3b5eff02b51cafa976f9d0b7a0376535aae Mon Sep 17 00:00:00 2001 From: Maciej Urbanski Date: Fri, 10 Nov 2023 09:14:10 +0100 Subject: [PATCH] refactor ls json streaming --- b2/console_tool.py | 64 ++++++++++++++++++++++++---------- test/integration/helpers.py | 2 +- test/unit/test_console_tool.py | 16 +++++---- 3 files changed, 56 insertions(+), 26 deletions(-) diff --git a/b2/console_tool.py b/b2/console_tool.py index a226fd064..49a53095b 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -685,27 +685,49 @@ def _parse_file_infos(cls, args_info): file_infos[parts[0]] = parts[1] return file_infos - def _print_json(self, data, *, raw: bool = False): - data = data if raw else json.dumps(data, indent=4, sort_keys=True, cls=B2CliJsonEncoder) - self._print(data, enforce_output=True, raw=raw) + def _print_json(self, data) -> None: + return self._print( + json.dumps(data, indent=4, sort_keys=True, cls=B2CliJsonEncoder), enforce_output=True + ) - def _print(self, *args, enforce_output=False, raw: bool = False): - self._print_standard_descriptor( - self.stdout, 'stdout', *args, enforce_output=enforce_output, raw=raw + def _print(self, *args, enforce_output: bool = False, end: Optional[str] = None) -> None: + return self._print_standard_descriptor( + self.stdout, "stdout", *args, enforce_output=enforce_output, end=end ) - def _print_stderr(self, *args, **kwargs): - self._print_standard_descriptor(self.stderr, 'stderr', *args, enforce_output=True) + def _print_stderr(self, *args, end: Optional[str] = None) -> None: + return self._print_standard_descriptor( + self.stderr, "stderr", *args, enforce_output=True, end=end + ) def _print_standard_descriptor( - self, descriptor, descriptor_name, *args, enforce_output=False, raw: bool = False - ): + self, + descriptor, + descriptor_name: str, + *args, + enforce_output: bool = False, + end: Optional[str] = None, + ) -> None: + """ + Prints to fd, unless quiet is set. + + :param descriptor: file descriptor to print to + :param descriptor_name: name of the descriptor, used for error reporting + :param args: object to be printed + :param enforce_output: overrides quiet setting; Should not be used for anything other than data + :param end: end of the line characters; None for default newline + """ if not self.quiet or enforce_output: - self._print_helper(descriptor, descriptor.encoding, descriptor_name, *args, raw=raw) + self._print_helper(descriptor, descriptor.encoding, descriptor_name, *args, end=end) @classmethod def _print_helper( - cls, descriptor, descriptor_encoding, descriptor_name, *args, raw: bool = False + cls, + descriptor, + descriptor_encoding: str, + descriptor_name: str, + *args, + end: Optional[str] = None ): try: descriptor.write(' '.join(args)) @@ -719,8 +741,7 @@ def _print_helper( args = [arg.encode('ascii', 'backslashreplace').decode() for arg in args] sys.stderr.write("Trying to print: %s\n" % args) descriptor.write(' '.join(args)) - if not raw: - descriptor.write('\n') + descriptor.write("\n" if end is None else end) def __str__(self): return f'{self.__class__.__module__}.{self.__class__.__name__}' @@ -1996,11 +2017,17 @@ def _setup_parser(cls, parser): def run(self, args): super().run(args) - self._print_files(args) if args.json: - if self._first_row: # no files were printed - self._print_json('[', raw=True) - self._print_json(']\n', raw=True) + i = -1 + for i, (file_version, _) in enumerate(self._get_ls_generator(args)): + if i: + self._print(',', end='') + else: + self._print('[') + self._print_json(file_version) + self._print(']' if i >= 0 else '[]') + else: + self._print_files(args) return 0 def _print_file_version( @@ -2222,6 +2249,7 @@ def _setup_parser(cls, parser): super()._setup_parser(parser) def run(self, args): + super().run(args) if args.dryRun: self._print_files(args) return 0 diff --git a/test/integration/helpers.py b/test/integration/helpers.py index d449ed0d6..3d6893855 100755 --- a/test/integration/helpers.py +++ b/test/integration/helpers.py @@ -337,7 +337,7 @@ def should_equal(expected, actual): class CommandLine: EXPECTED_STDERR_PATTERNS = [ - re.compile(r'^Using https?://[\w.]+.com$'), # account auth + re.compile(r'^Using https?://[\w.]+$'), # account auth re.compile(r'.*B/s]$', re.DOTALL), # progress bar re.compile(r'^\r?$'), # empty line re.compile( diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index 76730f41c..57aa6a606 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -84,9 +84,7 @@ def _run_command_ignore_output(self, argv): print('ACTUAL STDERR: ', repr(actual_stderr)) print(actual_stderr) - assert re.match( - r'^(|Using https?://[\w.]+.com\n)$', actual_stderr - ), f"stderr: {actual_stderr!r}" + assert re.match(r'^(|Using https?://[\w.]+)$', actual_stderr), f"stderr: {actual_stderr!r}" self.assertEqual(0, actual_status, 'exit status code') def _trim_leading_spaces(self, s): @@ -287,9 +285,11 @@ def test_authorize_with_good_key_using_underscore(self): assert self.account_info.get_account_auth_token() is None # Authorize an account with a good api key. + expected_stderr = """ + Using http://production.example.com + """ self._run_command( - ['authorize_account', self.account_id, self.master_key], '', - "Using http://production.example.com\n", 0 + ['authorize_account', self.account_id, self.master_key], '', expected_stderr, 0 ) # Auth token should be in account info now @@ -339,6 +339,9 @@ def test_authorize_towards_custom_realm(self): # Auth token should be in account info now assert self.account_info.get_account_auth_token() is not None + expected_stderr = """ + Using http://custom2.example.com + """ # realm provided with env var with mock.patch.dict( 'os.environ', { @@ -346,8 +349,7 @@ def test_authorize_towards_custom_realm(self): } ): self._run_command( - ['authorize-account', self.account_id, self.master_key], '', - 'Using http://custom2.example.com\n', 0 + ['authorize-account', self.account_id, self.master_key], '', expected_stderr, 0 ) def test_create_key_and_authorize_with_it(self):