diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c37bae078..48376151a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,7 +111,6 @@ jobs: B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} runs-on: ubuntu-latest - # Running on raw machine. steps: - uses: actions/checkout@v3 with: @@ -120,11 +119,26 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ env.PYTHON_DEFAULT_VERSION }} + - name: setup sudo NOX_PYTHONS + run: echo NOX_PYTHONS=$(sudo python3 --version | cut -d ' ' -f 2) >> "$GITHUB_ENV" - name: Install dependencies - run: python -m pip install --upgrade nox pip setuptools - - name: Run dockerized tests + run: sudo python -m pip install --upgrade nox pip setuptools + - name: Generate Dockerfile + run: nox -vs generate_dockerfile + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build Docker + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: backblazeit/b2:test + platforms: linux/amd64 + - name: Run tests with docker if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }} - run: nox -vs docker_test + run: sudo NOX_PYTHONS=$NOX_PYTHONS B2_TEST_APPLICATION_KEY=${{ env.B2_TEST_APPLICATION_KEY }} B2_TEST_APPLICATION_KEY_ID=${{ env.B2_TEST_APPLICATION_KEY_ID }} nox -vs docker_test -- backblazeit/b2:test test-linux-bundle: needs: cleanup_buckets env: diff --git a/.github/workflows/push_docker.yml b/.github/workflows/push_docker.yml new file mode 100644 index 000000000..d1de5808e --- /dev/null +++ b/.github/workflows/push_docker.yml @@ -0,0 +1,54 @@ +name: Deploy docker + +on: + push: + tags: 'v*' # push events to matching v*, i.e. v1.0, v20.15.10 + +jobs: + deploy-docker: + runs-on: ubuntu-latest + env: + DEBIAN_FRONTEND: noninteractive + DOCKERHUB_USERNAME: secrets.DOCKERHUB_USERNAME + DOCKERHUB_TOKEN: secrets.DOCKERHUB_TOKEN + B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }} + B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }} + PYTHON_DEFAULT_VERSION: 3.11 + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }} + uses: actions/setup-python@v4 + with: + python-version: ${{ env.PYTHON_DEFAULT_VERSION }} + - name: Install dependencies + run: python -m pip install --upgrade nox pip setuptools + - name: Build Dockerfile + run: nox -vs generate_dockerfile + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }} # TODO: skip whole job without marking it as an error + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: install setuptools_scm + run: pip install setuptools_scm + - name: get version + id: package_version + run: echo package_version=`python -m setuptools_scm` >> $GITHUB_OUTPUT + - name: echo + run: echo ${{ steps.package_version.outputs.package_version }} + - name: Build and push + if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }} # TODO: skip whole job without marking it as an error + uses: docker/build-push-action@v3 + with: + context: . + push: true + tags: backblazeit/b2:latest,backblazeit/b2:${{ steps.package_version.outputs.package_version }} + platforms: linux/amd64 + diff --git a/CHANGELOG.md b/CHANGELOG.md index 74bb80f0b..ff9928a68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [3.10.0] - 2023-09-10 ### Added +* docker tests and pushing the official docker image on release * Add ability to upload from an unbound source such as standard input or a named pipe * --bypassGovernance option to delete_file_version * Declare official support of Python 3.12 diff --git a/Dockerfile.template b/Dockerfile.template index 4ad37a413..99febb0d7 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -9,30 +9,10 @@ LABEL vcs-url="${vcs_url}" LABEL vcs-ref="${vcs_ref}" LABEL build-date-iso8601="${build_date}" -WORKDIR ${homedir} +WORKDIR /root COPY ${tar_path}/${tar_name} . RUN ["pip", "install", "${tar_name}[full]"] -ENV PATH=${homedir}/.local/bin:$$PATH - - -FROM base as test - -WORKDIR ${tests_image_dir} -COPY ${tests_path} ./${tests_path} -COPY noxfile.py . - -# Files used by tests. -${files_used_by_tests} - -RUN ["pip", "install", "nox"] -ENTRYPOINT ["nox", "--no-venv", "--no-install", "-s"] - - -FROM base - -RUN ["useradd", "--home", "${homedir}", "--shell", "/usr/sbin/nologin", "--badnames", "${username}"] -USER ${username} ENTRYPOINT ["b2"] CMD ["--help"] diff --git a/README.md b/README.md index 5f61821fe..5f587ddc1 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,16 @@ You can install the `b2` without them: pip install b2 ``` +### Docker + +For a truly platform independent solution, use the official docker image: + +```bash +docker run backblazeit/b2:latest ... +``` + +See examples in [Usage/Docker image](#docker-image) + ### Installing from source If installing from the repository is needed in order to e.g. check if a pre-release version resolves a bug effectively, it can be installed with: @@ -117,26 +127,33 @@ Note that using many threads could in some cases be detrimental to the other use ### Docker image -An official Docker image is provided for these who want to use B2 Command Line Tool in a Docker environment. +#### Authorization + +User can either authorize on each command (`list-buckets` is just a example here) + +```bash +B2_APPLICATION_KEY= B2_APPLICATION_KEY_ID= docker run --rm -e B2_APPLICATION_KEY -e B2_APPLICATION_KEY_ID backblazeit/b2:latest list-buckets +``` + +or authorize once and keep the credentials persisted: + +```bash +docker run --rm -it -v b2:/root backblazeit/b2:latest authorize-account +docker run --rm -v b2:/root backblazeit/b2:latest list-buckets # remember to include `-v` - authorization details are there +``` + +#### Downloading and uploading -An example workflow could be (with passing environment variables): +When uploading a single file, data can be passed to the container via a pipe: ```bash -B2_APPLICATION_KEY= B2_APPLICATION_KEY_ID= docker run --rm -e B2_APPLICATION_KEY -e B2_APPLICATION_KEY_ID b2:latest authorize-account -B2_APPLICATION_KEY= B2_APPLICATION_KEY_ID= docker run --rm -e B2_APPLICATION_KEY -e B2_APPLICATION_KEY_ID b2:latest create-bucket test-bucket allPrivate -B2_APPLICATION_KEY= B2_APPLICATION_KEY_ID= docker run --rm -e B2_APPLICATION_KEY -e B2_APPLICATION_KEY_ID -v :/data b2:latest upload-file test-bucket /data/local-file remote-file -B2_APPLICATION_KEY= B2_APPLICATION_KEY_ID= docker run --rm -e B2_APPLICATION_KEY -e B2_APPLICATION_KEY_ID -v :/data b2:latest ls test-bucket -B2_APPLICATION_KEY= B2_APPLICATION_KEY_ID= docker run --rm -e B2_APPLICATION_KEY -e B2_APPLICATION_KEY_ID -v :/data b2:latest download-file-by-name test-bucket remote-file /data/local-file-2 +cat source_file.txt | docker run --rm -v b2:/root backblazeit/b2:latest upload-unbound-stream bucket_name - target_file_name ``` -or mapping to a directory where account info will be kept: +or by mounting local files in the docker container: ```bash -docker run --rm -it -v :/b2 b2:latest authorize-account -docker run --rm -v :/b2 b2:latest create-bucket test-bucket allPrivate -docker run --rm -v :/b2 -v :/data b2:latest upload-file test-bucket /data/local-file remote-file -docker run --rm -v :/b2 -v :/data b2:latest ls test-bucket -docker run --rm -v :/b2 -v :/data b2:latest download-file-by-name test-bucket remote-file /data/local-file-2 +docker run --rm -v b2:/root -v /home/user/path/to/data:/data backblazeit/b2:latest upload-file bucket_name /data/source_file.txt target_file_name ``` ## Contrib diff --git a/b2/console_tool.py b/b2/console_tool.py index 2c31de244..c2f25d711 100644 --- a/b2/console_tool.py +++ b/b2/console_tool.py @@ -3492,9 +3492,17 @@ class Version(Command): REQUIRES_AUTH = False + @classmethod + def _setup_parser(cls, parser): + parser.add_argument('--short', action='store_true') + super()._setup_parser(parser) + def run(self, args): super().run(args) - self._print('b2 command line tool, version', VERSION) + if args.short: + self._print(VERSION) + else: + self._print('b2 command line tool, version', VERSION) return 0 diff --git a/noxfile.py b/noxfile.py index 4f6c35c69..4b61124e6 100644 --- a/noxfile.py +++ b/noxfile.py @@ -84,6 +84,11 @@ ) +@nox.session(venv_backend='none') +def install(session): + install_myself(session) + + def install_myself(session, extras=None): """Install from the source.""" @@ -182,13 +187,13 @@ def unit(session): session.notify('cover') -@nox.session(python=PYTHON_VERSIONS) -def integration(session): +def run_integration_test(session, pytest_posargs): """Run integration tests.""" install_myself(session, ['license']) session.run('pip', 'install', *REQUIREMENTS_TEST) session.run( 'pytest', + 'test/integration', '-s', '-n', 'auto', @@ -196,11 +201,16 @@ def integration(session): 'INFO', '-W', 'ignore::DeprecationWarning:rst2ansi.visitor:', - *session.posargs, - 'test/integration', + *pytest_posargs, ) +@nox.session(python=PYTHON_VERSIONS) +def integration(session): + """Run integration tests.""" + run_integration_test(session, session.posargs) + + @nox.session(python=PYTHON_VERSIONS) def test(session): """Run all tests.""" @@ -482,27 +492,22 @@ def _read_readme_name_and_description() -> Tuple[str, str]: @nox.session(python=PYTHON_DEFAULT_VERSION) -def docker(session): - """Build the docker image.""" +def generate_dockerfile(session): + """Generate Dockerfile from Dockerfile.template""" build(session) install_myself(session) # This string is like `b2 command line tool, version ` - version = session.run('b2', 'version', silent=True).split(' ')[-1].strip() + version = session.run('b2', 'version', '--short', silent=True).strip() dist_path = 'dist' - tests_image_dir = '/test' - tests_path = 'test/' full_name, description = _read_readme_name_and_description() vcs_ref = session.run("git", "rev-parse", "HEAD", external=True, silent=True).strip() built_distribution = list(pathlib.Path('.').glob(f'{dist_path}/*'))[0] - username = 'b2' template_mapping = dict( - username='b2', - homedir=f'/{username}', - python_version=session.python, + python_version=PYTHON_DEFAULT_VERSION, vendor='Backblaze', name=full_name, description=description, @@ -512,11 +517,8 @@ def docker(session): vcs_url='https://github.com/Backblaze/B2_Command_Line_Tool', vcs_ref=vcs_ref, build_date=datetime.datetime.utcnow().isoformat(), - tests_image_dir=tests_image_dir, - tests_path=tests_path, tar_path=dist_path, tar_name=built_distribution.name, - files_used_by_tests='\n'.join([f'COPY {filename} .' for filename in FILES_USED_IN_TESTS]) ) template_file = DOCKER_TEMPLATE.read_text() @@ -525,23 +527,35 @@ def docker(session): pathlib.Path('./Dockerfile').write_text(dockerfile) +def run_docker_tests(session, image_tag): + """Run unittests against a docker image.""" + run_integration_test( + session, [ + "--sut", + f"docker run -i -v b2:/root -v /tmp:/tmp:rw " + f"--env-file ENVFILE {image_tag}", + "--env-file-cmd-placeholder", + "ENVFILE", + ] + ) + + @nox.session(python=PYTHON_DEFAULT_VERSION) def docker_test(session): - """Run unittests against the docker image.""" - docker(session) - - image_tag = 'b2:test' - - session.run('docker', 'build', '-t', image_tag, '--target', 'test', '.', external=True) - docker_test_run = [ - 'docker', - 'run', - '--rm', - '-e', - 'B2_TEST_APPLICATION_KEY', - '-e', - 'B2_TEST_APPLICATION_KEY_ID', - image_tag, - ] - session.run(*docker_test_run, 'unit', external=True) - session.run(*docker_test_run, 'integration', '--', '--cleanup', external=True) + """Run unittests against a docker image.""" + if session.posargs: + image_tag = session.posargs[0] + else: + raise ValueError('Provide -- {docker_image_tag}') + run_docker_tests(session, image_tag) + + +@nox.session(python=PYTHON_DEFAULT_VERSION) +def build_and_test_docker(session): + """ + For running locally, CI uses a different set of sessions + """ + test_image_tag = 'b2:test' + generate_dockerfile(session) + session.run('docker', 'build', '-t', test_image_tag, '.', external=True) + run_docker_tests(session, test_image_tag) diff --git a/pyproject.toml b/pyproject.toml index 1dc9a6f2c..27a9965f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,8 +1,7 @@ -# Including project:version breaks `version` number generation under gha CD -# [project] -# requires-python = ">=3.7" -# name = "b2" -# version = "0.0.0" # this is wrong, but setuptools>61 insists its here +[project] +requires-python = ">=3.7" +name = "b2" +dynamic = ["version"] [tool.ruff] target-version = "py37" # to be replaced by project:requires-python when we will have that section in here diff --git a/test/integration/conftest.py b/test/integration/conftest.py old mode 100644 new mode 100755 index c36e62a0b..ff336bdc8 --- a/test/integration/conftest.py +++ b/test/integration/conftest.py @@ -10,8 +10,10 @@ import contextlib import os +import pathlib import subprocess import sys +import tempfile from os import environ, path from tempfile import TemporaryDirectory @@ -22,6 +24,8 @@ from .helpers import Api, CommandLine, bucket_name_part GENERAL_BUCKET_NAME_PREFIX = 'clitst' +TEMPDIR = tempfile.gettempdir() +ROOT_PATH = pathlib.Path(__file__).parent.parent.parent @pytest.hookimpl @@ -29,6 +33,15 @@ def pytest_addoption(parser): parser.addoption( '--sut', default='%s -m b2' % sys.executable, help='Path to the System Under Test' ) + parser.addoption( + '--env-file-cmd-placeholder', + default=None, + help=( + 'If specified, all occurrences of this string in `--sut` will be substituted with a' + 'path to a tmp file containing env vars to be used when running commands in tests. Useful' + 'for docker.' + ) + ) parser.addoption('--cleanup', action='store_true', help='Perform full cleanup at exit') @@ -143,6 +156,7 @@ def global_b2_tool( application_key, realm, this_run_bucket_name_prefix, + request.config.getoption('--env-file-cmd-placeholder'), ) tool.reauthorize(check_key_capabilities=True) # reauthorize for the first time (with check) return tool @@ -155,6 +169,20 @@ def b2_tool(global_b2_tool): return global_b2_tool +@pytest.fixture(autouse=True, scope='session') +def sample_file(): + """Copy the README.md file to /tmp so that docker tests can access it""" + tmp_readme = pathlib.Path(f'{TEMPDIR}/README.md') + if not tmp_readme.exists(): + tmp_readme.write_text((ROOT_PATH / 'README.md').read_text()) + return str(tmp_readme) + + +@pytest.fixture(scope='session') +def is_running_on_docker(pytestconfig): + return pytestconfig.getoption('--sut').startswith('docker') + + SECRET_FIXTURES = {'application_key', 'application_key_id'} @@ -190,9 +218,10 @@ def b2_in_path(tmp_path_factory): @pytest.fixture(scope="module") -def env(b2_in_path, homedir, monkey_patch): +def env(b2_in_path, homedir, monkey_patch, is_running_on_docker): """Get ENV for running b2 command from shell level.""" - monkey_patch.setenv('PATH', b2_in_path) + if not is_running_on_docker: + monkey_patch.setenv('PATH', b2_in_path) monkey_patch.setenv('HOME', str(homedir)) monkey_patch.setenv('SHELL', "/bin/bash") # fix for running under github actions yield os.environ diff --git a/test/integration/helpers.py b/test/integration/helpers.py old mode 100644 new mode 100755 index 61a55e22d..5a5e6d298 --- a/test/integration/helpers.py +++ b/test/integration/helpers.py @@ -7,10 +7,10 @@ # License https://www.backblaze.com/using_b2_code.html # ###################################################################### - import json import logging import os +import pathlib import platform import random import re @@ -24,7 +24,7 @@ from datetime import datetime from os import environ, linesep, path from pathlib import Path -from tempfile import gettempdir, mkdtemp +from tempfile import gettempdir, mkdtemp, mktemp from typing import List, Optional, Union from unittest.mock import MagicMock @@ -298,57 +298,6 @@ def read_from(self, f): self.string = str(e) -def run_command( - cmd: str, - args: Optional[List[Union[str, Path, int]]] = None, - additional_env: Optional[dict] = None, -): - """ - :param cmd: a command to run - :param args: command's arguments - :param additional_env: environment variables to pass to the command, overwriting parent process ones - :return: (status, stdout, stderr) - """ - # We'll run the b2 command-line by running the b2 module from - # the current directory or provided as parameter - environ['PYTHONPATH'] = '.' - environ['PYTHONIOENCODING'] = 'utf-8' - command = cmd.split(' ') - args: List[str] = [str(arg) for arg in args] if args else [] - command.extend(args) - - print('Running:', ' '.join(command)) - - stdout = StringReader() - stderr = StringReader() - - env = environ.copy() - env.update(additional_env or {}) - - p = subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=platform.system() != 'Windows', - env=env, - ) - p.stdin.close() - reader1 = threading.Thread(target=stdout.read_from, args=[p.stdout]) - reader1.start() - reader2 = threading.Thread(target=stderr.read_from, args=[p.stderr]) - reader2.start() - p.wait() - reader1.join() - reader2.join() - - stdout_decoded = remove_warnings(stdout.get_string().decode('utf-8', errors='replace')) - stderr_decoded = remove_warnings(stderr.get_string().decode('utf-8', errors='replace')) - - print_output(p.returncode, stdout_decoded, stderr_decoded) - return p.returncode, stdout_decoded, stderr_decoded - - class EnvVarTestContext: """ Establish config for environment variable test. @@ -401,12 +350,16 @@ class CommandLine: re.compile(r'Trying to print: .*'), ] - def __init__(self, command, account_id, application_key, realm, bucket_name_prefix): + def __init__( + self, command, account_id, application_key, realm, bucket_name_prefix, + env_file_cmd_placeholder + ): self.command = command self.account_id = account_id self.application_key = application_key self.realm = realm self.bucket_name_prefix = bucket_name_prefix + self.env_file_cmd_placeholder = env_file_cmd_placeholder self.env_var_test_context = EnvVarTestContext(SqliteAccountInfo().filename) self.account_info_file_name = SqliteAccountInfo().filename @@ -420,7 +373,7 @@ def run_command(self, args, additional_env: Optional[dict] = None): Runs the command with the given arguments, returns a tuple in form of (succeeded, stdout) """ - status, stdout, stderr = run_command(self.command, args, additional_env) + status, stdout, stderr = self.execute(args, additional_env) return status == 0 and stderr == '', stdout def should_succeed( @@ -432,9 +385,9 @@ def should_succeed( """ Runs the command-line with the given arguments. Raises an exception if there was an error; otherwise, returns the stdout of the command - as as string. + as string. """ - status, stdout, stderr = run_command(self.command, args, additional_env) + status, stdout, stderr = self.execute(args, additional_env) assert status == 0, f'FAILED with status {status}, stderr={stderr}' if stderr != '': @@ -454,6 +407,79 @@ def should_succeed( return stdout + @classmethod + def prepare_env(self, additional_env: Optional[dict] = None): + environ['PYTHONPATH'] = '.' + environ['PYTHONIOENCODING'] = 'utf-8' + env = environ.copy() + env.update(additional_env or {}) + return env + + def parse_command(self, env): + """ + Split `self.command` into a list of strings. If necessary, dump the env vars to a tmp file and substitute + one the command's argument with that file's path. + """ + command = self.command.split(' ') + if self.env_file_cmd_placeholder: + if any('\n' in var_value for var_value in env.values()): + raise ValueError( + 'Env vars containing new line characters will break env file format' + ) + env_file_path = mktemp() + pathlib.Path(env_file_path).write_text('\n'.join(f'{k}={v}' for k, v in env.items())) + command = [ + (c if c != self.env_file_cmd_placeholder else env_file_path) for c in command + ] + return command + + def execute( + self, + args: Optional[List[Union[str, Path, int]]] = None, + additional_env: Optional[dict] = None, + ): + """ + :param cmd: a command to run + :param args: command's arguments + :param additional_env: environment variables to pass to the command, overwriting parent process ones + :return: (status, stdout, stderr) + """ + # We'll run the b2 command-line by running the b2 module from + # the current directory or provided as parameter + env = self.prepare_env(additional_env) + command = self.parse_command(env) + + args: List[str] = [str(arg) for arg in args] if args else [] + command.extend(args) + + print('Running:', ' '.join(command)) + + stdout = StringReader() + stderr = StringReader() + + p = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=platform.system() != 'Windows', + env=env, + ) + p.stdin.close() + reader1 = threading.Thread(target=stdout.read_from, args=[p.stdout]) + reader1.start() + reader2 = threading.Thread(target=stderr.read_from, args=[p.stderr]) + reader2.start() + p.wait() + reader1.join() + reader2.join() + + stdout_decoded = remove_warnings(stdout.get_string().decode('utf-8', errors='replace')) + stderr_decoded = remove_warnings(stderr.get_string().decode('utf-8', errors='replace')) + + print_output(p.returncode, stdout_decoded, stderr_decoded) + return p.returncode, stdout_decoded, stderr_decoded + def should_succeed_json(self, args, additional_env: Optional[dict] = None): """ Runs the command-line with the given arguments. Raises an exception @@ -472,7 +498,7 @@ def should_fail(self, args, expected_pattern, additional_env: Optional[dict] = N Runs the command-line with the given args, expecting the given pattern to appear in stderr. """ - status, stdout, stderr = run_command(self.command, args, additional_env) + status, stdout, stderr = self.execute(args, additional_env) assert status != 0, 'ERROR: should have failed' if platform.python_implementation().lower() == 'pypy': diff --git a/test/integration/test_autocomplete.py b/test/integration/test_autocomplete.py index 8b77d5e3b..e0c30e065 100644 --- a/test/integration/test_autocomplete.py +++ b/test/integration/test_autocomplete.py @@ -56,13 +56,19 @@ def shell(env): @skip_on_windows -def test_autocomplete_b2_commands(autocomplete_installed, shell): +def test_autocomplete_b2_commands(autocomplete_installed, is_running_on_docker, shell): + if is_running_on_docker: + pytest.skip('Not supported on Docker') shell.send('b2 \t\t') shell.expect_exact(["authorize-account", "download-file-by-id", "get-bucket"], timeout=TIMEOUT) @skip_on_windows -def test_autocomplete_b2_only_matching_commands(autocomplete_installed, shell): +def test_autocomplete_b2_only_matching_commands( + autocomplete_installed, is_running_on_docker, shell +): + if is_running_on_docker: + pytest.skip('Not supported on Docker') shell.send('b2 download-\t\t') shell.expect_exact( @@ -74,9 +80,11 @@ def test_autocomplete_b2_only_matching_commands(autocomplete_installed, shell): @skip_on_windows def test_autocomplete_b2_bucket_n_file_name( - autocomplete_installed, shell, b2_tool, bucket_name, file_name + autocomplete_installed, shell, b2_tool, bucket_name, file_name, is_running_on_docker ): """Test that autocomplete suggests bucket names and file names.""" + if is_running_on_docker: + pytest.skip('Not supported on Docker') shell.send('b2 download_file_by_name \t\t') shell.expect_exact(bucket_name, timeout=TIMEOUT) shell.send(f'{bucket_name} \t\t') diff --git a/test/integration/test_b2_command_line.py b/test/integration/test_b2_command_line.py old mode 100644 new mode 100755 index e9fe7baa6..ea458203d --- a/test/integration/test_b2_command_line.py +++ b/test/integration/test_b2_command_line.py @@ -59,30 +59,27 @@ def get_bucketinfo() -> Tuple[str, str]: return '--bucketInfo', json.dumps({BUCKET_CREATED_AT_MILLIS: str(current_time_millis())}), -def test_download(b2_tool, bucket_name): - - file_to_upload = 'README.md' +def test_download(b2_tool, bucket_name, sample_file): uploaded_a = b2_tool.should_succeed_json( - ['upload-file', '--quiet', bucket_name, file_to_upload, 'a'] + ['upload-file', '--quiet', bucket_name, sample_file, 'a'] ) with TempDir() as dir_path: b2_tool.should_succeed( ['download-file-by-name', '--quiet', bucket_name, 'a', dir_path / 'a'] ) - assert read_file(dir_path / 'a') == read_file(file_to_upload) + assert read_file(dir_path / 'a') == read_file(sample_file) b2_tool.should_succeed( ['download-file-by-id', '--quiet', uploaded_a['fileId'], dir_path / 'b'] ) - assert read_file(dir_path / 'b') == read_file(file_to_upload) + assert read_file(dir_path / 'b') == read_file(sample_file) -def test_basic(b2_tool, bucket_name): +def test_basic(b2_tool, bucket_name, sample_file, is_running_on_docker): - file_to_upload = 'README.md' - file_mod_time_str = str(file_mod_time_millis(file_to_upload)) + file_mod_time_str = str(file_mod_time_millis(sample_file)) - file_data = read_file(file_to_upload) + file_data = read_file(sample_file) hex_sha1 = hashlib.sha1(file_data).hexdigest() list_of_buckets = b2_tool.should_succeed_json(['list-buckets', '--json']) @@ -90,32 +87,32 @@ def test_basic(b2_tool, bucket_name): [bucket_name], [b['bucketName'] for b in list_of_buckets if b['bucketName'] == bucket_name] ) - b2_tool.should_succeed(['upload-file', '--quiet', bucket_name, file_to_upload, 'a']) + b2_tool.should_succeed(['upload-file', '--quiet', bucket_name, sample_file, 'a']) b2_tool.should_succeed(['ls', '--long', '--replication', bucket_name]) - b2_tool.should_succeed(['upload-file', '--noProgress', bucket_name, file_to_upload, 'a']) - b2_tool.should_succeed(['upload-file', '--noProgress', bucket_name, file_to_upload, 'b/1']) - b2_tool.should_succeed(['upload-file', '--noProgress', bucket_name, file_to_upload, 'b/2']) + b2_tool.should_succeed(['upload-file', '--noProgress', bucket_name, sample_file, 'a']) + b2_tool.should_succeed(['upload-file', '--noProgress', bucket_name, sample_file, 'b/1']) + b2_tool.should_succeed(['upload-file', '--noProgress', bucket_name, sample_file, 'b/2']) b2_tool.should_succeed( [ 'upload-file', '--noProgress', '--sha1', hex_sha1, '--info', 'foo=bar=baz', '--info', - 'color=blue', bucket_name, file_to_upload, 'c' + 'color=blue', bucket_name, sample_file, 'c' ] ) b2_tool.should_fail( [ 'upload-file', '--noProgress', '--sha1', hex_sha1, '--info', 'foo-bar', '--info', - 'color=blue', bucket_name, file_to_upload, 'c' + 'color=blue', bucket_name, sample_file, 'c' ], r'ERROR: Bad file info: foo-bar' ) b2_tool.should_succeed( [ - 'upload-file', '--noProgress', '--contentType', 'text/plain', bucket_name, - file_to_upload, 'd' + 'upload-file', '--noProgress', '--contentType', 'text/plain', bucket_name, sample_file, + 'd' ] ) - b2_tool.should_succeed(['upload-file', '--noProgress', bucket_name, file_to_upload, 'rm']) - b2_tool.should_succeed(['upload-file', '--noProgress', bucket_name, file_to_upload, 'rm1']) + b2_tool.should_succeed(['upload-file', '--noProgress', bucket_name, sample_file, 'rm']) + b2_tool.should_succeed(['upload-file', '--noProgress', bucket_name, sample_file, 'rm1']) # with_wildcard allows us to target a single file. rm will be removed, rm1 will be left alone b2_tool.should_succeed(['rm', '--recursive', '--withWildcard', bucket_name, 'rm']) list_of_files = b2_tool.should_succeed_json( @@ -191,10 +188,10 @@ def test_basic(b2_tool, bucket_name): b2_tool.should_succeed(['make-url', second_c_version['fileId']]) b2_tool.should_succeed( - ['make-friendly-url', bucket_name, file_to_upload], + ['make-friendly-url', bucket_name, 'any-file-name'], '^https://.*/file/{}/{}\r?$'.format( bucket_name, - file_to_upload, + 'any-file-name', ), ) # \r? is for Windows, as $ doesn't match \r\n to_be_removed_bucket_name = b2_tool.generate_bucket_name() @@ -212,38 +209,39 @@ def test_basic(b2_tool, bucket_name): re.compile(r'^ERROR: Bucket with id=\w* not found\s*$') ) # Check logging settings - b2_tool.should_fail( - ['delete-bucket', to_be_removed_bucket_name, '--debugLogs'], - re.compile(r'^ERROR: Bucket with id=\w* not found\s*$') - ) - stack_trace_in_log = r'Traceback \(most recent call last\):.*Bucket with id=\w* not found' - - # the two regexes below depend on log message from urllib3, which is not perfect, but this test needs to - # check global logging settings - stderr_regex = re.compile( - r'DEBUG:urllib3.connectionpool:.* "POST /b2api/v2/b2_delete_bucket HTTP' - r'.*' + stack_trace_in_log, - re.DOTALL, - ) - log_file_regex = re.compile( - r'urllib3.connectionpool\tDEBUG\t.* "POST /b2api/v2/b2_delete_bucket HTTP' - r'.*' + stack_trace_in_log, - re.DOTALL, - ) - with open('b2_cli.log') as logfile: - log = logfile.read() - assert re.search(log_file_regex, log), log - os.remove('b2_cli.log') + if not is_running_on_docker: # It's difficult to read the log in docker in CI + b2_tool.should_fail( + ['delete-bucket', to_be_removed_bucket_name, '--debugLogs'], + re.compile(r'^ERROR: Bucket with id=\w* not found\s*$') + ) + stack_trace_in_log = r'Traceback \(most recent call last\):.*Bucket with id=\w* not found' + + # the two regexes below depend on log message from urllib3, which is not perfect, but this test needs to + # check global logging settings + stderr_regex = re.compile( + r'DEBUG:urllib3.connectionpool:.* "POST /b2api/v2/b2_delete_bucket HTTP' + r'.*' + stack_trace_in_log, + re.DOTALL, + ) + log_file_regex = re.compile( + r'urllib3.connectionpool\tDEBUG\t.* "POST /b2api/v2/b2_delete_bucket HTTP' + r'.*' + stack_trace_in_log, + re.DOTALL, + ) + with open('b2_cli.log') as logfile: + log = logfile.read() + assert re.search(log_file_regex, log), log + os.remove('b2_cli.log') - b2_tool.should_fail(['delete-bucket', to_be_removed_bucket_name, '--verbose'], stderr_regex) - assert not os.path.exists('b2_cli.log') + b2_tool.should_fail(['delete-bucket', to_be_removed_bucket_name, '--verbose'], stderr_regex) + assert not os.path.exists('b2_cli.log') - b2_tool.should_fail( - ['delete-bucket', to_be_removed_bucket_name, '--verbose', '--debugLogs'], stderr_regex - ) - with open('b2_cli.log') as logfile: - log = logfile.read() - assert re.search(log_file_regex, log), log + b2_tool.should_fail( + ['delete-bucket', to_be_removed_bucket_name, '--verbose', '--debugLogs'], stderr_regex + ) + with open('b2_cli.log') as logfile: + log = logfile.read() + assert re.search(log_file_regex, log), log def test_bucket(b2_tool, bucket_name): @@ -272,12 +270,12 @@ def test_bucket(b2_tool, bucket_name): ] -def test_key_restrictions(b2_api, b2_tool, bucket_name): +def test_key_restrictions(b2_api, b2_tool, bucket_name, sample_file): second_bucket_name = b2_tool.generate_bucket_name() b2_tool.should_succeed(['create-bucket', second_bucket_name, 'allPublic', *get_bucketinfo()],) # A single file for rm to fail on. - b2_tool.should_succeed(['upload-file', '--noProgress', bucket_name, 'README.md', 'test']) + b2_tool.should_succeed(['upload-file', '--noProgress', bucket_name, sample_file, 'test']) key_one_name = 'clt-testKey-01' + random_hex(6) created_key_stdout = b2_tool.should_succeed( @@ -728,21 +726,19 @@ def sync_up_helper(b2_tool, bucket_name, dir_, encryption=None): ) -def test_sync_down(b2_tool, bucket_name): - sync_down_helper(b2_tool, bucket_name, 'sync') +def test_sync_down(b2_tool, bucket_name, sample_file): + sync_down_helper(b2_tool, bucket_name, 'sync', sample_file) -def test_sync_down_no_prefix(b2_tool, bucket_name): - sync_down_helper(b2_tool, bucket_name, '') +def test_sync_down_no_prefix(b2_tool, bucket_name, sample_file): + sync_down_helper(b2_tool, bucket_name, '', sample_file) -def test_sync_down_sse_c_no_prefix(b2_tool, bucket_name): - sync_down_helper(b2_tool, bucket_name, '', SSE_C_AES) +def test_sync_down_sse_c_no_prefix(b2_tool, bucket_name, sample_file): + sync_down_helper(b2_tool, bucket_name, '', sample_file, SSE_C_AES) -def sync_down_helper(b2_tool, bucket_name, folder_in_bucket, encryption=None): - - file_to_upload = 'README.md' +def sync_down_helper(b2_tool, bucket_name, folder_in_bucket, sample_file, encryption=None): b2_sync_point = 'b2:%s' % bucket_name if folder_in_bucket: @@ -777,12 +773,12 @@ def sync_down_helper(b2_tool, bucket_name, folder_in_bucket, encryption=None): # Put a couple files in B2 b2_tool.should_succeed( - ['upload-file', '--noProgress', bucket_name, file_to_upload, b2_file_prefix + 'a'] + + ['upload-file', '--noProgress', bucket_name, sample_file, b2_file_prefix + 'a'] + upload_encryption_args, additional_env=upload_additional_env, ) b2_tool.should_succeed( - ['upload-file', '--noProgress', bucket_name, file_to_upload, b2_file_prefix + 'b'] + + ['upload-file', '--noProgress', bucket_name, sample_file, b2_file_prefix + 'b'] + upload_encryption_args, additional_env=upload_additional_env, ) @@ -793,13 +789,13 @@ def sync_down_helper(b2_tool, bucket_name, folder_in_bucket, encryption=None): should_equal(['a', 'b'], sorted(os.listdir(local_path))) b2_tool.should_succeed( - ['upload-file', '--noProgress', bucket_name, file_to_upload, b2_file_prefix + 'c'] + + ['upload-file', '--noProgress', bucket_name, sample_file, b2_file_prefix + 'c'] + upload_encryption_args, additional_env=upload_additional_env, ) # Sync the files with one file being excluded because of mtime - mod_time = str((file_mod_time_millis(file_to_upload) - 10) / 1000) + mod_time = str((file_mod_time_millis(sample_file) - 10) / 1000) b2_tool.should_succeed( [ 'sync', '--noProgress', '--excludeIfModifiedAfter', mod_time, b2_sync_point, @@ -830,51 +826,60 @@ def sync_down_helper(b2_tool, bucket_name, folder_in_bucket, encryption=None): ) -def test_sync_copy(b2_api, b2_tool, bucket_name): - prepare_and_run_sync_copy_tests(b2_api, b2_tool, bucket_name, 'sync') +def test_sync_copy(b2_api, b2_tool, bucket_name, sample_file): + prepare_and_run_sync_copy_tests(b2_api, b2_tool, bucket_name, 'sync', sample_file=sample_file) -def test_sync_copy_no_prefix_default_encryption(b2_api, b2_tool, bucket_name): +def test_sync_copy_no_prefix_default_encryption(b2_api, b2_tool, bucket_name, sample_file): prepare_and_run_sync_copy_tests( - b2_api, b2_tool, bucket_name, '', destination_encryption=None, expected_encryption=SSE_NONE + b2_api, + b2_tool, + bucket_name, + '', + sample_file=sample_file, + destination_encryption=None, + expected_encryption=SSE_NONE ) -def test_sync_copy_no_prefix_no_encryption(b2_api, b2_tool, bucket_name): +def test_sync_copy_no_prefix_no_encryption(b2_api, b2_tool, bucket_name, sample_file): prepare_and_run_sync_copy_tests( b2_api, b2_tool, bucket_name, '', + sample_file=sample_file, destination_encryption=SSE_NONE, expected_encryption=SSE_NONE ) -def test_sync_copy_no_prefix_sse_b2(b2_api, b2_tool, bucket_name): +def test_sync_copy_no_prefix_sse_b2(b2_api, b2_tool, bucket_name, sample_file): prepare_and_run_sync_copy_tests( b2_api, b2_tool, bucket_name, '', + sample_file=sample_file, destination_encryption=SSE_B2_AES, expected_encryption=SSE_B2_AES, ) -def test_sync_copy_no_prefix_sse_c(b2_api, b2_tool, bucket_name): +def test_sync_copy_no_prefix_sse_c(b2_api, b2_tool, bucket_name, sample_file): prepare_and_run_sync_copy_tests( b2_api, b2_tool, bucket_name, '', + sample_file=sample_file, destination_encryption=SSE_C_AES, expected_encryption=SSE_C_AES, source_encryption=SSE_C_AES_2, ) -def test_sync_copy_sse_c_single_bucket(b2_tool, bucket_name): +def test_sync_copy_sse_c_single_bucket(b2_tool, bucket_name, sample_file): run_sync_copy_with_basic_checks( b2_tool=b2_tool, b2_file_prefix='first_folder/', @@ -883,6 +888,7 @@ def test_sync_copy_sse_c_single_bucket(b2_tool, bucket_name): other_b2_sync_point=f'b2:{bucket_name}/second_folder', destination_encryption=SSE_C_AES_2, source_encryption=SSE_C_AES, + sample_file=sample_file, ) expected_encryption_first = encryption_summary( SSE_C_AES.as_dict(), @@ -910,6 +916,7 @@ def prepare_and_run_sync_copy_tests( b2_tool, bucket_name, folder_in_bucket, + sample_file, destination_encryption=None, expected_encryption=SSE_NONE, source_encryption=None, @@ -938,6 +945,7 @@ def prepare_and_run_sync_copy_tests( other_b2_sync_point=other_b2_sync_point, destination_encryption=destination_encryption, source_encryption=source_encryption, + sample_file=sample_file, ) if destination_encryption is None or destination_encryption in (SSE_NONE, SSE_B2_AES): @@ -970,9 +978,8 @@ def run_sync_copy_with_basic_checks( other_b2_sync_point, destination_encryption, source_encryption, + sample_file, ): - file_to_upload = 'README.md' - # Put a couple files in B2 if source_encryption is None or source_encryption.mode in ( EncryptionMode.NONE, EncryptionMode.SSE_B2 @@ -980,18 +987,18 @@ def run_sync_copy_with_basic_checks( b2_tool.should_succeed( [ 'upload-file', '--noProgress', '--destinationServerSideEncryption', 'SSE-B2', - bucket_name, file_to_upload, b2_file_prefix + 'a' + bucket_name, sample_file, b2_file_prefix + 'a' ] ) b2_tool.should_succeed( - ['upload-file', '--noProgress', bucket_name, file_to_upload, b2_file_prefix + 'b'] + ['upload-file', '--noProgress', bucket_name, sample_file, b2_file_prefix + 'b'] ) elif source_encryption.mode == EncryptionMode.SSE_C: for suffix in ['a', 'b']: b2_tool.should_succeed( [ 'upload-file', '--noProgress', '--destinationServerSideEncryption', 'SSE-C', - bucket_name, file_to_upload, b2_file_prefix + suffix + bucket_name, sample_file, b2_file_prefix + suffix ], additional_env={ 'B2_DESTINATION_SSE_C_KEY_B64': @@ -1120,16 +1127,14 @@ def test_default_sse_b2(b2_api, b2_tool, bucket_name): b2_api.clean_bucket(second_bucket_name) -def test_sse_b2(b2_tool, bucket_name): - file_to_upload = 'README.md' - +def test_sse_b2(b2_tool, bucket_name, sample_file): b2_tool.should_succeed( [ 'upload-file', '--destinationServerSideEncryption=SSE-B2', '--quiet', bucket_name, - file_to_upload, 'encrypted' + sample_file, 'encrypted' ] ) - b2_tool.should_succeed(['upload-file', '--quiet', bucket_name, file_to_upload, 'not_encrypted']) + b2_tool.should_succeed(['upload-file', '--quiet', bucket_name, sample_file, 'not_encrypted']) with TempDir() as dir_path: b2_tool.should_succeed( ['download-file-by-name', '--quiet', bucket_name, 'encrypted', dir_path / 'encrypted'] @@ -1189,26 +1194,30 @@ def test_sse_b2(b2_tool, bucket_name): should_equal({'mode': 'none'}, file_info['serverSideEncryption']) -def test_sse_c(b2_tool, bucket_name): +def test_sse_c(b2_tool, bucket_name, is_running_on_docker, sample_file): + + sse_c_key_id = 'user-generated-key-id \nąóźćż\nœøΩ≈ç\nßäöü' + if is_running_on_docker: + # TODO: fix this once we figure out how to pass env vars with \n in them to docker, docker-compose should work + sse_c_key_id = sse_c_key_id.replace('\n', '') - file_to_upload = 'README.md' secret = os.urandom(32) b2_tool.should_fail( [ 'upload-file', '--noProgress', '--quiet', '--destinationServerSideEncryption', 'SSE-C', - bucket_name, file_to_upload, 'gonna-fail-anyway' + bucket_name, sample_file, 'gonna-fail-anyway' ], 'Using SSE-C requires providing an encryption key via B2_DESTINATION_SSE_C_KEY_B64 env var' ) file_version_info = b2_tool.should_succeed_json( [ 'upload-file', '--noProgress', '--quiet', '--destinationServerSideEncryption', 'SSE-C', - bucket_name, file_to_upload, 'uploaded_encrypted' + bucket_name, sample_file, 'uploaded_encrypted' ], additional_env={ 'B2_DESTINATION_SSE_C_KEY_B64': base64.b64encode(secret).decode(), - 'B2_DESTINATION_SSE_C_KEY_ID': 'user-generated-key-id \nąóźćż\nœøΩ≈ç\nßäöü', + 'B2_DESTINATION_SSE_C_KEY_ID': sse_c_key_id, } ) should_equal( @@ -1219,10 +1228,7 @@ def test_sse_c(b2_tool, bucket_name): "mode": "SSE-C" }, file_version_info['serverSideEncryption'] ) - should_equal( - 'user-generated-key-id \nąóźćż\nœøΩ≈ç\nßäöü', - file_version_info['fileInfo'][SSE_C_KEY_ID_FILE_INFO_KEY_NAME] - ) + should_equal(sse_c_key_id, file_version_info['fileInfo'][SSE_C_KEY_ID_FILE_INFO_KEY_NAME]) b2_tool.should_fail( [ @@ -1262,7 +1268,7 @@ def test_sse_c(b2_tool, bucket_name): ], additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()} ) - assert read_file(dir_path / 'a') == read_file(file_to_upload) + assert read_file(dir_path / 'a') == read_file(sample_file) b2_tool.should_succeed( [ 'download-file-by-id', @@ -1275,7 +1281,7 @@ def test_sse_c(b2_tool, bucket_name): ], additional_env={'B2_SOURCE_SSE_C_KEY_B64': base64.b64encode(secret).decode()} ) - assert read_file(dir_path / 'b') == read_file(file_to_upload) + assert read_file(dir_path / 'b') == read_file(sample_file) b2_tool.should_fail( ['copy-file-by-id', file_version_info['fileId'], bucket_name, 'gonna-fail-anyway'], @@ -1457,7 +1463,7 @@ def test_sse_c(b2_tool, bucket_name): }, { 'file_name': 'uploaded_encrypted', - 'sse_c_key_id': 'user-generated-key-id \nąóźćż\nœøΩ≈ç\nßäöü', + 'sse_c_key_id': sse_c_key_id, 'serverSideEncryption': { "algorithm": "AES256", @@ -1557,7 +1563,7 @@ def test_license(b2_tool, with_packages): SOFTWARE.""" in license_text.replace(os.linesep, '\n'), repr(license_text[-2000:]) -def test_file_lock(b2_tool, application_key_id, application_key, b2_api): +def test_file_lock(b2_tool, application_key_id, application_key, b2_api, sample_file): lock_disabled_bucket_name = b2_tool.generate_bucket_name() b2_tool.should_succeed( [ @@ -1568,11 +1574,10 @@ def test_file_lock(b2_tool, application_key_id, application_key, b2_api): ], ) - file_to_upload = 'README.md' now_millis = current_time_millis() not_lockable_file = b2_tool.should_succeed_json( # file in a lock disabled bucket - ['upload-file', '--quiet', lock_disabled_bucket_name, file_to_upload, 'a'] + ['upload-file', '--quiet', lock_disabled_bucket_name, sample_file, 'a'] ) _assert_file_lock_configuration( @@ -1587,7 +1592,7 @@ def test_file_lock(b2_tool, application_key_id, application_key, b2_api): 'upload-file', '--quiet', lock_disabled_bucket_name, - file_to_upload, + sample_file, 'a', '--fileRetentionMode', 'governance', @@ -1640,7 +1645,7 @@ def test_file_lock(b2_tool, application_key_id, application_key, b2_api): } lockable_file = b2_tool.should_succeed_json( # file in a lock enabled bucket - ['upload-file', '--noProgress', '--quiet', lock_enabled_bucket_name, file_to_upload, 'a'] + ['upload-file', '--noProgress', '--quiet', lock_enabled_bucket_name, sample_file, 'a'] ) b2_tool.should_fail( @@ -1749,7 +1754,7 @@ def test_file_lock(b2_tool, application_key_id, application_key, b2_api): '--noProgress', '--quiet', lock_enabled_bucket_name, - file_to_upload, + sample_file, 'a', '--fileRetentionMode', 'governance', @@ -1765,7 +1770,7 @@ def test_file_lock(b2_tool, application_key_id, application_key, b2_api): '--noProgress', '--quiet', lock_enabled_bucket_name, - file_to_upload, + sample_file, 'a', '--fileRetentionMode', 'governance', @@ -1831,8 +1836,12 @@ def test_file_lock(b2_tool, application_key_id, application_key, b2_api): ) file_lock_without_perms_test( - b2_tool, lock_enabled_bucket_name, lock_disabled_bucket_name, lockable_file['fileId'], - not_lockable_file['fileId'] + b2_tool, + lock_enabled_bucket_name, + lock_disabled_bucket_name, + lockable_file['fileId'], + not_lockable_file['fileId'], + sample_file=sample_file ) b2_tool.should_succeed( @@ -1840,7 +1849,7 @@ def test_file_lock(b2_tool, application_key_id, application_key, b2_api): ) deleting_locked_files( - b2_tool, lock_enabled_bucket_name, lock_disabled_key_id, lock_disabled_key + b2_tool, lock_enabled_bucket_name, lock_disabled_key_id, lock_disabled_key, sample_file ) # ---- perform test cleanup ---- @@ -1870,7 +1879,7 @@ def make_lock_disabled_key(b2_tool): def file_lock_without_perms_test( b2_tool, lock_enabled_bucket_name, lock_disabled_bucket_name, lockable_file_id, - not_lockable_file_id + not_lockable_file_id, sample_file ): b2_tool.should_fail( @@ -1924,7 +1933,7 @@ def file_lock_without_perms_test( '--noProgress', '--quiet', lock_enabled_bucket_name, - 'README.md', + sample_file, 'bound_to_fail_anyway', '--fileRetentionMode', 'governance', @@ -1942,7 +1951,7 @@ def file_lock_without_perms_test( '--noProgress', '--quiet', lock_disabled_bucket_name, - 'README.md', + sample_file, 'bound_to_fail_anyway', '--fileRetentionMode', 'governance', @@ -1987,7 +1996,7 @@ def file_lock_without_perms_test( ) -def upload_locked_file(b2_tool, bucket_name): +def upload_locked_file(b2_tool, bucket_name, sample_file): return b2_tool.should_succeed_json( [ 'upload-file', @@ -1998,16 +2007,16 @@ def upload_locked_file(b2_tool, bucket_name): '--retainUntil', str(int(time.time()) + 1000), bucket_name, - 'README.md', + sample_file, 'a-locked', ] ) def deleting_locked_files( - b2_tool, lock_enabled_bucket_name, lock_disabled_key_id, lock_disabled_key + b2_tool, lock_enabled_bucket_name, lock_disabled_key_id, lock_disabled_key, sample_file ): - locked_file = upload_locked_file(b2_tool, lock_enabled_bucket_name) + locked_file = upload_locked_file(b2_tool, lock_enabled_bucket_name, sample_file) b2_tool.should_fail( [ # master key 'delete-file-version', @@ -2023,7 +2032,7 @@ def deleting_locked_files( '--bypassGovernance' ]) - locked_file = upload_locked_file(b2_tool, lock_enabled_bucket_name) + locked_file = upload_locked_file(b2_tool, lock_enabled_bucket_name, sample_file) b2_tool.should_succeed( [ @@ -2311,7 +2320,7 @@ def test_replication_setup(b2_api, b2_tool, bucket_name): 'asReplicationDestination']['sourceToDestinationKeyMapping'] -def test_replication_monitoring(b2_tool, bucket_name, b2_api): +def test_replication_monitoring(b2_tool, bucket_name, b2_api, sample_file): # ---------------- set up keys ---------------- key_one_name = 'clt-testKey-01' + random_hex(6) @@ -2337,7 +2346,7 @@ def test_replication_monitoring(b2_tool, bucket_name, b2_api): # ---------------- add test data ---------------- destination_bucket_name = bucket_name uploaded_a = b2_tool.should_succeed_json( - ['upload-file', '--quiet', destination_bucket_name, 'README.md', 'one/a'] + ['upload-file', '--quiet', destination_bucket_name, sample_file, 'one/a'] ) # ---------------- set up replication destination ---------------- @@ -2405,7 +2414,7 @@ def test_replication_monitoring(b2_tool, bucket_name, b2_api): # make test data uploaded_a = b2_tool.should_succeed_json( - ['upload-file', '--quiet', source_bucket_name, 'CHANGELOG.md', 'one/a'] + ['upload-file', '--quiet', source_bucket_name, sample_file, 'one/a'] ) b2_tool.should_succeed_json( [ @@ -2414,7 +2423,7 @@ def test_replication_monitoring(b2_tool, bucket_name, b2_api): source_bucket_name, '--legalHold', 'on', - 'README.md', + sample_file, 'two/b', ] ) @@ -2424,7 +2433,7 @@ def test_replication_monitoring(b2_tool, bucket_name, b2_api): upload_encryption_args = ['--destinationServerSideEncryption', 'SSE-B2'] upload_additional_env = {} b2_tool.should_succeed_json( - ['upload-file', '--quiet', source_bucket_name, 'README.md', 'two/c'] + + ['upload-file', '--quiet', source_bucket_name, sample_file, 'two/c'] + upload_encryption_args, additional_env=upload_additional_env, ) @@ -2436,7 +2445,7 @@ def test_replication_monitoring(b2_tool, bucket_name, b2_api): 'B2_DESTINATION_SSE_C_KEY_ID': SSE_C_AES.key.key_id, } b2_tool.should_succeed_json( - ['upload-file', '--quiet', source_bucket_name, 'README.md', 'two/d'] + + ['upload-file', '--quiet', source_bucket_name, sample_file, 'two/d'] + upload_encryption_args, additional_env=upload_additional_env, ) @@ -2447,7 +2456,7 @@ def test_replication_monitoring(b2_tool, bucket_name, b2_api): 'upload-file', '--quiet', source_bucket_name, - 'README.md', + sample_file, 'two/e', '--legalHold', 'on', @@ -2610,9 +2619,8 @@ def _assert_file_lock_configuration( assert legal_hold == actual_legal_hold -def test_cut(b2_tool, bucket_name): - file_to_upload = 'README.md' - file_data = read_file(file_to_upload) +def test_cut(b2_tool, bucket_name, sample_file): + file_data = read_file(sample_file) cut = 12345 cut_printable = '1970-01-01 00:00:12' args = [ @@ -2622,7 +2630,7 @@ def test_cut(b2_tool, bucket_name): str(cut), '--quiet', bucket_name, - file_to_upload, + sample_file, 'a', ] succeeded, stdout = b2_tool.run_command(args) @@ -2646,18 +2654,24 @@ def test_cut(b2_tool, bucket_name): @skip_on_windows -def test_upload_file__stdin_pipe_operator(bash_runner, b2_tool, bucket_name, request): +def test_upload_file__stdin_pipe_operator(request, bash_runner, b2_tool, bucket_name): """Test upload-file from stdin using pipe operator.""" content = request.node.name run = bash_runner( - f'echo -n {content!r} | b2 upload-file {bucket_name} - {request.node.name}.txt' + f'echo -n {content!r} ' + f'| ' + f'{" ".join(b2_tool.parse_command(b2_tool.prepare_env()))} upload-file {bucket_name} - {request.node.name}.txt' ) assert hashlib.sha1(content.encode()).hexdigest() in run.stdout @skip_on_windows -def test_upload_unbound_stream__redirect_operator(bash_runner, b2_tool, bucket_name, request): +def test_upload_unbound_stream__redirect_operator( + request, bash_runner, b2_tool, bucket_name, is_running_on_docker +): """Test upload-unbound-stream from stdin using redirect operator.""" + if is_running_on_docker: + pytest.skip('Not supported on Docker') content = request.node.name run = bash_runner( f'b2 upload-unbound-stream {bucket_name} <(echo -n {content}) {request.node.name}.txt' diff --git a/test/unit/test_console_tool.py b/test/unit/test_console_tool.py index 642c44728..9416345b3 100644 --- a/test/unit/test_console_tool.py +++ b/test/unit/test_console_tool.py @@ -38,6 +38,7 @@ B2_ENVIRONMENT_ENV_VAR, ) from b2.console_tool import ConsoleTool, Rm +from b2.version import VERSION from .test_base import TestBase @@ -2737,3 +2738,7 @@ def test_rm_skipping_over_errors(self): b/b1/test.csv ''' self._run_command(['ls', '--recursive', 'my-bucket'], expected_stdout) + + def test_version(self): + self._run_command(['version', '--short'], expected_stdout=f'{VERSION}\n') + self._run_command(['version'], expected_stdout=f'b2 command line tool, version {VERSION}\n')