Benchmark #13336
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark | |
| on: | |
| pull_request: | |
| branches: ['*'] | |
| paths: | |
| - .github/workflows/benchmark.yml | |
| schedule: | |
| - cron: '0 4 * * 1-5' # Mon-Fri 4AM UTC | |
| workflow_dispatch: {} | |
| concurrency: 'benchmarks' # Ensure only one of this runs at a time | |
| jobs: | |
| run-benchmarks: | |
| name: Clickhouse queries | |
| runs-on: ubuntu-20.04 | |
| environment: clickhouse-benchmarks | |
| # Benchmarks are expensive to run so we only run them (periodically) against master branch and for PRs labeled `performance` | |
| if: ${{ github.repository == 'PostHog/posthog' && (github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'performance')) }} | |
| env: | |
| DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog' | |
| REDIS_URL: 'redis://localhost' | |
| DEBUG: '1' | |
| CLICKHOUSE_DATABASE: posthog | |
| CLICKHOUSE_HOST: ${{ secrets.BENCHMARKS_CLICKHOUSE_HOST }} | |
| CLICKHOUSE_USER: ${{ secrets.BENCHMARKS_CLICKHOUSE_USER }} | |
| CLICKHOUSE_PASSWORD: ${{ secrets.BENCHMARKS_CLICKHOUSE_PASSWORD }} | |
| CLICKHOUSE_SECURE: 'false' | |
| CLICKHOUSE_VERIFY: 'false' | |
| SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only | |
| BENCHMARK: '1' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| # Checkout repo with full history | |
| fetch-depth: 0 | |
| - name: Check out PostHog/benchmarks-results repo | |
| uses: actions/checkout@v3 | |
| with: | |
| path: ee/benchmarks/results | |
| repository: PostHog/benchmark-results | |
| token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} | |
| - name: Stop/Start stack with Docker Compose | |
| run: | | |
| docker compose -f docker-compose.dev.yml down | |
| docker compose -f docker-compose.dev.yml up -d | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11.9 | |
| cache: 'pip' | |
| cache-dependency-path: '**/requirements*.txt' | |
| token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} | |
| # uv is a fast pip alternative: https://github.com/astral-sh/uv/ | |
| - run: pip install uv | |
| - name: Install SAML (python3-saml) dependencies | |
| shell: bash | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl | |
| - name: Install python dependencies | |
| run: | | |
| uv pip install --system -r requirements-dev.txt | |
| uv pip install --system -r requirements.txt | |
| - name: Install asv | |
| run: uv pip install --system asv==0.5.1 virtualenv | |
| - name: Set up PostHog | |
| run: | | |
| python manage.py migrate & wait | |
| python manage.py setup_dev --no-data | |
| - name: Configure benchmarks | |
| run: asv machine --config ee/benchmarks/asv.conf.json --yes --machine ci-benchmarks | |
| - name: Run benchmarks | |
| run: asv run --config ee/benchmarks/asv.conf.json --show-stderr --strict | |
| - name: Compare results | |
| run: | | |
| asv compare $(cat ee/benchmarks/results/last-master-commit) HEAD --config ee/benchmarks/asv.conf.json --factor 1.2 | tee pr_vs_master.txt | |
| asv compare $(cat ee/benchmarks/results/last-master-commit) HEAD --config ee/benchmarks/asv.conf.json --factor 1.2 --only-changed | tee pr_vs_master_changed.txt | |
| - name: Save last benchmarked commit | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: echo "${{ github.sha }}" | tee ee/benchmarks/results/last-master-commit | |
| - name: Generate HTML report of results | |
| if: ${{ github.ref == 'refs/heads/master' }} | |
| run: asv publish --config ee/benchmarks/asv.conf.json | |
| - name: Commit update for benchmark results | |
| if: ${{ github.repository == 'PostHog/posthog' && github.ref == 'refs/heads/master' }} | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| repository: ee/benchmarks/results | |
| branch: master | |
| commit_message: 'Save benchmark results' | |
| commit_user_name: PostHog Bot | |
| commit_user_email: [email protected] | |
| commit_author: PostHog Bot <[email protected]> | |
| - name: Upload results as artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: benchmarks | |
| path: | | |
| pr_vs_master.txt | |
| pr_vs_master_changed.txt | |
| - name: Read benchmark output | |
| if: ${{ github.event_name == 'pull_request' }} | |
| id: pr_vs_master_changed | |
| uses: juliangruber/read-file-action@v1 | |
| with: | |
| path: pr_vs_master_changed.txt | |
| - name: Read benchmark output (full) | |
| if: ${{ github.event_name == 'pull_request' }} | |
| id: pr_vs_master | |
| uses: juliangruber/read-file-action@v1 | |
| with: | |
| path: pr_vs_master.txt | |
| - name: Find Comment | |
| if: ${{ github.event_name == 'pull_request' }} | |
| uses: peter-evans/find-comment@v2 | |
| id: fc | |
| with: | |
| issue-number: ${{ github.event.number }} | |
| comment-author: 'github-actions[bot]' | |
| body-includes: ClickHouse query benchmark results from GitHub Actions | |
| - name: Create or update comment | |
| if: ${{ github.event_name == 'pull_request' }} | |
| uses: peter-evans/create-or-update-comment@v3 | |
| with: | |
| comment-id: ${{ steps.fc.outputs.comment-id }} | |
| issue-number: ${{ github.event.number }} | |
| body: | | |
| ClickHouse query benchmark results from GitHub Actions | |
| Lower numbers are good, higher numbers are bad. A ratio less than 1 | |
| means a speed up and greater than 1 means a slowdown. Green lines | |
| beginning with `+` are slowdowns (the PR is slower then master or | |
| master is slower than the previous release). Red lines beginning | |
| with `-` are speedups. Blank means no changes. | |
| Significantly changed benchmark results (PR vs master) | |
| ```diff | |
| ${{ steps.pr_vs_master_changed.outputs.content }} | |
| ``` | |
| <details> | |
| <summary>Click to view full benchmark results</summary> | |
| ```diff | |
| ${{ steps.pr_vs_master.outputs.content }} | |
| ``` | |
| </details> | |
| edit-mode: replace |