Speed up cleanup-removed-hosts-metadata script #340
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: main | |
| on: | |
| push: | |
| branches: [main] | |
| tags: "v*" | |
| pull_request: | |
| branches: [main] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.run_number }} | |
| cancel-in-progress: true | |
| env: | |
| BUILD_DEB_OUTPUT_DIR: out | |
| BUILD_PYTHON_OUTPUT_DIR: dist | |
| PROJECT_NAME: clickhouse-tools | |
| PROJECT_NAME_UNDESCORE: clickhouse_tools | |
| jobs: | |
| lint: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: ./.github/actions/setup_dependencies | |
| with: | |
| python-version: "3.10" | |
| - name: "lint: isort" | |
| run: make isort | |
| - name: "lint: black" | |
| run: make black | |
| - name: "lint: codespell" | |
| run: make codespell | |
| - name: "lint: ruff" | |
| run: make ruff | |
| - name: "lint: pylint" | |
| run: make pylint | |
| - name: "lint: mypy" | |
| run: make mypy | |
| tests: | |
| needs: lint | |
| runs-on: ubuntu-${{ matrix.target.ubuntu }} | |
| env: | |
| CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: | |
| - { python: "3.6", ubuntu: "20.04" } | |
| - { python: "3.10", ubuntu: "latest" } | |
| clickhouse: | |
| - "21.8.15.7" | |
| - "22.3.20.29" | |
| - "22.8.19.10" | |
| - "23.3.4.17" | |
| - "latest" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: ./.github/actions/setup_dependencies | |
| with: | |
| python-version: ${{ matrix.target.python }} | |
| - run: make test-unit | |
| - run: make test-integration | |
| - uses: actions/upload-artifact@v3 | |
| if: ${{ failure() }} | |
| with: | |
| name: integration-test-logs-py${{ matrix.target.python }}-clickhouse-${{ matrix.clickhouse }} | |
| path: tests/staging/logs/ | |
| if-no-files-found: ignore | |
| - name: publish test report | |
| uses: mikepenz/action-junit-report@v3 | |
| if: always() | |
| with: | |
| report_paths: 'tests/reports/*.xml' | |
| push_to_dockerhub: | |
| runs-on: ubuntu-latest | |
| env: | |
| CLICKHOUSE_VERSIONS: "21.8.15.7, 22.3.20.29, 22.8.19.10, 23.3.4.17, latest" | |
| if: ${{ github.event_name == 'push' }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: login to dockerhub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: build and push base images | |
| uses: docker/bake-action@v3 | |
| with: | |
| files: tests/bake.hcl | |
| push: true | |
| release: | |
| needs: tests | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| target: | |
| - {python: "3.6", ubuntu: "20.04"} | |
| - {python: "3.10", ubuntu: "latest"} | |
| runs-on: ubuntu-${{ matrix.target.ubuntu }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: ./.github/actions/setup_dependencies | |
| with: | |
| python-version: ${{ matrix.target.python }} | |
| - name: build project | |
| run: make build-python-packages | |
| - name: upload wheel | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.PROJECT_NAME_UNDESCORE }}_py${{ matrix.target.python }}.whl | |
| path: ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.whl | |
| if-no-files-found: error | |
| - name: upload sdist | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.PROJECT_NAME_UNDESCORE }}_py${{ matrix.target.python }}.tar.gz | |
| path: ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.tar.gz | |
| if-no-files-found: error | |
| - name: Set up QEMU (for building ARM debian package) | |
| uses: docker/setup-qemu-action@v3 | |
| - name: build DEB package | |
| env: | |
| DEB_BUILD_DISTRIBUTION: ubuntu:${{ matrix.target.ubuntu }} | |
| run: | | |
| # Speedup Debian package building | |
| echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io | |
| make build-deb-package | |
| - name: test DEB package | |
| run: | | |
| sudo make uninstall | |
| sudo apt-get install -q -y ./${{ env.BUILD_DEB_OUTPUT_DIR }}/*.deb | |
| sudo chadmin --help | |
| sudo ch-monitoring --no-ensure-monitoring-user --help | |
| sudo keeper-monitoring --help | |
| sudo ch-s3-credentials --help | |
| - name: upload DEB package artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.PROJECT_NAME }}_py-${{ matrix.target.python }}_ubuntu-${{ matrix.target.ubuntu }}.deb | |
| path: ${{ env.BUILD_DEB_OUTPUT_DIR }}/*.deb | |
| if-no-files-found: error | |
| - name: create a release | |
| uses: softprops/action-gh-release@v1 | |
| if: ${{ matrix.target.ubuntu == 'latest' && startsWith(github.ref, 'refs/tags/') }} | |
| with: | |
| generate_release_notes: true | |
| fail_on_unmatched_files: true | |
| files: | | |
| ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.whl | |
| ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.tar.gz | |
| ${{ env.BUILD_DEB_OUTPUT_DIR }}/*.deb | |
| - name: publish to PYPI | |
| if: ${{ matrix.target.ubuntu == 'latest' && startsWith(github.ref, 'refs/tags/') }} | |
| continue-on-error: true | |
| run: make publish | |
| env: | |
| POETRY_HTTP_BASIC_PYPI_USERNAME: "__token__" | |
| POETRY_HTTP_BASIC_PYPI_PASSWORD: "${{ secrets.PYPI_TOKEN }}" | |
| # Disable using keyring for poetry to get password from env variable | |
| PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring |