build and push docker on release

.github/workflows/ci.yml

@@ -111,7 +111,6 @@ jobs:
       B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }}
       B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }}
     runs-on: ubuntu-latest
-    # Running on raw machine.
     steps:
       - uses: actions/checkout@v3
         with:
@@ -120,11 +119,26 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
+      - name: setup sudo NOX_PYTHONS
+        run: echo NOX_PYTHONS=$(sudo python3 --version | cut -d ' ' -f 2) >> "$GITHUB_ENV"
       - name: Install dependencies
-        run: python -m pip install --upgrade nox pip setuptools
-      - name: Run dockerized tests
+        run: sudo python -m pip install --upgrade nox pip setuptools
+      - name: Generate Dockerfile
+        run: nox -vs generate_dockerfile
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Build Docker
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          load: true
+          tags: backblazeit/b2:test
+          platforms: linux/amd64
+      - name: Run tests with docker
         if: ${{ env.B2_TEST_APPLICATION_KEY != '' && env.B2_TEST_APPLICATION_KEY_ID != '' }}
-        run: nox -vs docker_test
+        run: sudo NOX_PYTHONS=$NOX_PYTHONS B2_TEST_APPLICATION_KEY=${{ env.B2_TEST_APPLICATION_KEY }} B2_TEST_APPLICATION_KEY_ID=${{ env.B2_TEST_APPLICATION_KEY_ID }} nox -vs docker_test -- backblazeit/b2:test
   test-linux-bundle:
     needs: cleanup_buckets
     env:

.github/workflows/push_docker.yml

@@ -0,0 +1,54 @@
+name: Deploy docker
+
+on:
+  push:
+    tags: 'v*'  # push events to matching v*, i.e. v1.0, v20.15.10
+
+jobs:
+  deploy-docker:
+    runs-on: ubuntu-latest
+    env:
+      DEBIAN_FRONTEND: noninteractive
+      DOCKERHUB_USERNAME: secrets.DOCKERHUB_USERNAME
+      DOCKERHUB_TOKEN: secrets.DOCKERHUB_TOKEN
+      B2_TEST_APPLICATION_KEY: ${{ secrets.B2_TEST_APPLICATION_KEY }}
+      B2_TEST_APPLICATION_KEY_ID: ${{ secrets.B2_TEST_APPLICATION_KEY_ID }}
+      PYTHON_DEFAULT_VERSION: 3.11
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Python ${{ env.PYTHON_DEFAULT_VERSION }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_DEFAULT_VERSION }}
+      - name: Install dependencies
+        run: python -m pip install --upgrade nox pip setuptools
+      - name: Build Dockerfile
+        run: nox -vs generate_dockerfile
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to Docker Hub
+        if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}  # TODO: skip whole job without marking it as an error
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: install setuptools_scm
+        run: pip install setuptools_scm
+      - name: get version
+        id: package_version
+        run: echo package_version=`python -m setuptools_scm` >> $GITHUB_OUTPUT
+      - name: echo
+        run: echo ${{ steps.package_version.outputs.package_version }}
+      - name: Build and push
+        if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }}  # TODO: skip whole job without marking it as an error
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          push: true
+          tags: backblazeit/b2:latest,backblazeit/b2:${{ steps.package_version.outputs.package_version }}
+          platforms: linux/amd64
+

CHANGELOG.md

@@ -38,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [3.10.0] - 2023-09-10
 
 ### Added
+* docker tests and pushing the official docker image on release
 * Add ability to upload from an unbound source such as standard input or a named pipe
 * --bypassGovernance option to delete_file_version
 * Declare official support of Python 3.12

Dockerfile.template

@@ -9,30 +9,10 @@ LABEL vcs-url="${vcs_url}"
 LABEL vcs-ref="${vcs_ref}"
 LABEL build-date-iso8601="${build_date}"
 
-WORKDIR ${homedir}
+WORKDIR /root
 
 COPY ${tar_path}/${tar_name} .
 RUN ["pip", "install", "${tar_name}[full]"]
-ENV PATH=${homedir}/.local/bin:$$PATH
-
-
-FROM base as test
-
-WORKDIR ${tests_image_dir}
-COPY ${tests_path} ./${tests_path}
-COPY noxfile.py .
-
-# Files used by tests.
-${files_used_by_tests}
-
-RUN ["pip", "install", "nox"]
-ENTRYPOINT ["nox", "--no-venv", "--no-install", "-s"]
-
-
-FROM base
-
-RUN ["useradd", "--home", "${homedir}", "--shell", "/usr/sbin/nologin", "--badnames", "${username}"]
-USER ${username}
 
 ENTRYPOINT ["b2"]
 CMD ["--help"]

README.md

@@ -41,6 +41,16 @@ You can install the `b2` without them:
     pip install b2
 ```
 
+### Docker
+
+For a truly platform independent solution, use the official docker image: 
+
+```bash
+docker run backblazeit/b2:latest  ...
+```
+
+See examples in [Usage/Docker image](#docker-image)
+
 ### Installing from source
 
 If installing from the repository is needed in order to e.g. check if a pre-release version resolves a bug effectively, it can be installed with:
@@ -117,26 +127,33 @@ Note that using many threads could in some cases be detrimental to the other use
 
 ### Docker image
 
-An official Docker image is provided for these who want to use B2 Command Line Tool in a Docker environment.
+#### Authorization
+
+User can either authorize on each call (`list-buckets` is just an exemplary command here)
+
+```bash
+B2_APPLICATION_KEY=<key> B2_APPLICATION_KEY_ID=<key-id> docker run --rm -e B2_APPLICATION_KEY -e B2_APPLICATION_KEY_ID backblazeit/b2:latest list-buckets
+```
+
+or authorize once and keep the credentials persisted:
+
+```bash
+docker run --rm -it -v b2:/root backblazeit/b2:latest authorize-account
+docker run --rm -v b2:/root backblazeit/b2:latest list-buckets  # remember to include `-v` - authorization details are there
+```
+
+#### Downloading and uploading
 
-An example workflow could be (with passing environment variables):
+When uploading a single file, data can be passed to the container via a pipe:
 
 ```bash
-B2_APPLICATION_KEY=<key> B2_APPLICATION_KEY_ID=<key-id> docker run --rm -e B2_APPLICATION_KEY -e B2_APPLICATION_KEY_ID b2:latest authorize-account
-B2_APPLICATION_KEY=<key> B2_APPLICATION_KEY_ID=<key-id> docker run --rm -e B2_APPLICATION_KEY -e B2_APPLICATION_KEY_ID b2:latest create-bucket test-bucket allPrivate
-B2_APPLICATION_KEY=<key> B2_APPLICATION_KEY_ID=<key-id> docker run --rm -e B2_APPLICATION_KEY -e B2_APPLICATION_KEY_ID -v <absolute-local-path-to-data>:/data b2:latest upload-file test-bucket /data/local-file remote-file
-B2_APPLICATION_KEY=<key> B2_APPLICATION_KEY_ID=<key-id> docker run --rm -e B2_APPLICATION_KEY -e B2_APPLICATION_KEY_ID -v <absolute-local-path-to-data>:/data b2:latest ls test-bucket
-B2_APPLICATION_KEY=<key> B2_APPLICATION_KEY_ID=<key-id> docker run --rm -e B2_APPLICATION_KEY -e B2_APPLICATION_KEY_ID -v <absolute-local-path-to-data>:/data b2:latest download-file-by-name test-bucket remote-file /data/local-file-2
+cat source_file.txt | docker run --rm -v b2:/root backblazeit/b2:latest upload-unbound-stream bucket_name - target_file_name
 ```
 
-or mapping to a directory where account info will be kept:
+or by mounting local files in the docker container:
 
 ```bash
-docker run --rm -it -v <absolute-local-path-to-account-info>:/b2 b2:latest authorize-account
-docker run --rm -v <absolute-local-path-to-account-info>:/b2 b2:latest create-bucket test-bucket allPrivate
-docker run --rm -v <absolute-local-path-to-account-info>:/b2 -v <absolute-local-path-to-data>:/data b2:latest upload-file test-bucket /data/local-file remote-file
-docker run --rm -v <absolute-local-path-to-account-info>:/b2 -v <absolute-local-path-to-data>:/data b2:latest ls test-bucket
-docker run --rm -v <absolute-local-path-to-account-info>:/b2 -v <absolute-local-path-to-data>:/data b2:latest download-file-by-name test-bucket remote-file /data/local-file-2
+docker run --rm -v b2:/root -v /home/user/path/to/data:/data backblazeit/b2:latest upload-file bucket_name /data/source_file.txt target_file_name
 ```
 
 ## Contrib

b2/console_tool.py

@@ -3492,9 +3492,17 @@ class Version(Command):
 
     REQUIRES_AUTH = False
 
+    @classmethod
+    def _setup_parser(cls, parser):
+        parser.add_argument('--short', action='store_true')
+        super()._setup_parser(parser)
+
     def run(self, args):
         super().run(args)
-        self._print('b2 command line tool, version', VERSION)
+        if args.short:
+            self._print(VERSION)
+        else:
+            self._print('b2 command line tool, version', VERSION)
         return 0
 
 

noxfile.py

@@ -84,6 +84,11 @@
     )
 
 
+@nox.session(venv_backend='none')
+def install(session):
+    install_myself(session)
+
+
 def install_myself(session, extras=None):
     """Install from the source."""
 
@@ -182,25 +187,30 @@ def unit(session):
         session.notify('cover')
 
 
-@nox.session(python=PYTHON_VERSIONS)
-def integration(session):
+def run_integration_test(session, pytest_posargs):
     """Run integration tests."""
     install_myself(session, ['license'])
     session.run('pip', 'install', *REQUIREMENTS_TEST)
     session.run(
         'pytest',
+        'test/integration',
         '-s',
         '-n',
         'auto',
         '--log-level',
         'INFO',
         '-W',
         'ignore::DeprecationWarning:rst2ansi.visitor:',
-        *session.posargs,
-        'test/integration',
+        *pytest_posargs,
     )
 
 
+@nox.session(python=PYTHON_VERSIONS)
+def integration(session):
+    """Run integration tests."""
+    run_integration_test(session, session.posargs)
+
+
 @nox.session(python=PYTHON_VERSIONS)
 def test(session):
     """Run all tests."""
@@ -482,27 +492,22 @@ def _read_readme_name_and_description() -> Tuple[str, str]:
 
 
 @nox.session(python=PYTHON_DEFAULT_VERSION)
-def docker(session):
-    """Build the docker image."""
+def generate_dockerfile(session):
+    """Generate Dockerfile from Dockerfile.template"""
     build(session)
 
     install_myself(session)
     # This string is like `b2 command line tool, version <sem-ver-string>`
-    version = session.run('b2', 'version', silent=True).split(' ')[-1].strip()
+    version = session.run('b2', 'version', '--short', silent=True).strip()
 
     dist_path = 'dist'
-    tests_image_dir = '/test'
-    tests_path = 'test/'
 
     full_name, description = _read_readme_name_and_description()
     vcs_ref = session.run("git", "rev-parse", "HEAD", external=True, silent=True).strip()
     built_distribution = list(pathlib.Path('.').glob(f'{dist_path}/*'))[0]
-    username = 'b2'
 
     template_mapping = dict(
-        username='b2',
-        homedir=f'/{username}',
-        python_version=session.python,
+        python_version=PYTHON_DEFAULT_VERSION,
         vendor='Backblaze',
         name=full_name,
         description=description,
@@ -512,11 +517,8 @@ def docker(session):
         vcs_url='https://github.com/Backblaze/B2_Command_Line_Tool',
         vcs_ref=vcs_ref,
         build_date=datetime.datetime.utcnow().isoformat(),
-        tests_image_dir=tests_image_dir,
-        tests_path=tests_path,
         tar_path=dist_path,
         tar_name=built_distribution.name,
-        files_used_by_tests='\n'.join([f'COPY {filename} .' for filename in FILES_USED_IN_TESTS])
     )
 
     template_file = DOCKER_TEMPLATE.read_text()
@@ -525,23 +527,35 @@ def docker(session):
     pathlib.Path('./Dockerfile').write_text(dockerfile)
 
 
+def run_docker_tests(session, image_tag):
+    """Run unittests against a docker image."""
+    run_integration_test(
+        session, [
+            "--sut",
+            f"docker run -i -v b2:/root -v /tmp:/tmp:rw "
+            f"--env-file ENVFILE {image_tag}",
+            "--env-file-cmd-placeholder",
+            "ENVFILE",
+        ]
+    )
+
+
 @nox.session(python=PYTHON_DEFAULT_VERSION)
 def docker_test(session):
-    """Run unittests against the docker image."""
-    docker(session)
-
-    image_tag = 'b2:test'
-
-    session.run('docker', 'build', '-t', image_tag, '--target', 'test', '.', external=True)
-    docker_test_run = [
-        'docker',
-        'run',
-        '--rm',
-        '-e',
-        'B2_TEST_APPLICATION_KEY',
-        '-e',
-        'B2_TEST_APPLICATION_KEY_ID',
-        image_tag,
-    ]
-    session.run(*docker_test_run, 'unit', external=True)
-    session.run(*docker_test_run, 'integration', '--', '--cleanup', external=True)
+    """Run unittests against a docker image."""
+    if session.posargs:
+        image_tag = session.posargs[0]
+    else:
+        raise ValueError('Provide -- {docker_image_tag}')
+    run_docker_tests(session, image_tag)
+
+
+@nox.session(python=PYTHON_DEFAULT_VERSION)
+def build_and_test_docker(session):
+    """
+    For running locally, CI uses a different set of sessions
+    """
+    test_image_tag = 'b2:test'
+    generate_dockerfile(session)
+    session.run('docker', 'build', '-t', test_image_tag, '.', external=True)
+    run_docker_tests(session, test_image_tag)

pyproject.toml

@@ -1,8 +1,7 @@
-# Including project:version breaks `version` number generation under gha CD
-# [project]
-# requires-python = ">=3.7"
-# name = "b2"
-# version = "0.0.0"  # this is wrong, but setuptools>61 insists its here
+[project]
+requires-python = ">=3.7"
+name = "b2"
+dynamic = ["version"]
 
 [tool.ruff]
 target-version = "py37"  # to be replaced by project:requires-python when we will have that section in here