test: use codspeed for continues benchmark (#54)

.github/dependabot.yml

@@ -0,0 +1,31 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "monthly"
+    groups:
+      # Specify a name for the group, which will be used in pull request titles
+      # and branch names
+      dependencies:
+        # Define patterns to include dependencies in the group (based on
+        # dependency name)
+        patterns:
+          - "*" # A wildcard that matches all dependencies in the package
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "monthly"
+    groups:
+      # Specify a name for the group, which will be used in pull request titles
+      # and branch names
+      dependencies:
+        # Define patterns to include dependencies in the group (based on
+        # dependency name)
+        patterns:
+          - "*" # A wildcard that matches all dependencies in the package

.github/workflows/codespeed.yml

@@ -0,0 +1,36 @@
+name: Bench
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  benchmark:
+    name: Continuous Benchmarking with CodSpeed
+    runs-on: ubuntu-latest
+    steps:
+      - name: Collect Workflow Telemetry
+        uses: runforesight/workflow-telemetry-action@v2
+        with:
+          comment_on_pr: false
+      - uses: actions/checkout@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v3
+        with:
+          run: |
+            uv run pytest --codspeed --benchmark-warmup=off --benchmark-warmup-iterations=10 --benchmark-min-rounds=2 -v tests/test_perf.py   
+
+      - name: Minimize uv cache
+        run: uv cache prune --ci

.github/workflows/test.yml

@@ -10,41 +10,6 @@ on:
     branches: ["main"]
 
 jobs:
-  coverage-perf:
-    runs-on: "ubuntu-latest"
-    steps:
-      - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version-file: pyproject.toml
-      - name: Install uv
-        uses: astral-sh/setup-uv@v3
-        with:
-          enable-cache: true
-      - run: make bench
-
-      - name: Upload coverage reports to Codecov
-        run: |
-          curl -Os https://uploader.codecov.io/latest/linux/codecov
-          chmod +x codecov
-          ./codecov -t ${{ secrets.CODECOV_TOKEN }}
-      - name: Continuous Benchmark
-        uses: benchmark-action/[email protected]
-        if: ${{ github.ref == 'refs/heads/main' }}
-        with:
-          name: cyeva Benchmark
-          tool: "pytest"
-          output-file-path: output.json
-          github-token: ${{ secrets.OPS_ACTIONS_CI_TOKEN }}
-          gh-repository: "github.com/caiyunapp/cyeva"
-          auto-push: true
-          alert-threshold: "150%"
-          comment-on-alert: true
-          fail-on-alert: false
-          benchmark-data-dir-path: "performance/"
-          alert-comment-cc-users: "@clarmy"
-
   Test:
     runs-on: ${{ matrix.os }}
     strategy:
@@ -64,3 +29,9 @@ jobs:
         with:
           enable-cache: true
       - run: make all
+      - name: Upload coverage reports to Codecov
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          curl -Os https://uploader.codecov.io/latest/linux/codecov
+          chmod +x codecov
+          ./codecov -t ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -131,4 +131,6 @@ dmypy.json
 .DS_Store
 .vscode
 *.ipynb
-_build
+_build
+
+output.json

Makefile

@@ -42,7 +42,7 @@ test: lint
 	uv run pytest --cov=cyeva/ ./tests/functions ./tests/test_issues.py
 
 bench: sync
-	uv run pytest --cov=cyeva/ -p no:warnings --memray --benchmark-json output.json tests/test_perf.py
+	uv run pytest --benchmark-warmup=off --benchmark-warmup-iterations=10 --benchmark-min-rounds=2 --benchmark-json output.json tests/test_perf.py   
 
 freeze-doc-requirements:
 	cd docs;uv lock;uv export > requirements.txt

pyproject.toml

@@ -34,6 +34,7 @@ dev-dependencies = [
     "pytest-cov>=5.0.0",
     "pytest-benchmark>=4.0.0",
     "pytest-memray>=1.7.0; sys_platform != 'win32'",
+    "pytest-codspeed>=2.2.1",
 ]
 package = true
 

tests/test_perf.py

@@ -31,17 +31,6 @@ def inner():
     benchmark(inner)
 
 
-def test_calc_mae_1e7(benchmark):
-    # ten million level
-    obs = np.random.rand(int(1e7))
-    fct = np.random.rand(int(1e7))
-
-    def inner():
-        calc_mae(obs, fct)
-
-    benchmark(inner)
-
-
 def test_calc_mbe_1e3(benchmark):
     # thousand level
     obs = np.random.rand(int(1e3))
@@ -64,17 +53,6 @@ def inner():
     benchmark(inner)
 
 
-def test_calc_mbe_1e7(benchmark):
-    # ten million level
-    obs = np.random.rand(int(1e7))
-    fct = np.random.rand(int(1e7))
-
-    def inner():
-        calc_mbe(obs, fct)
-
-    benchmark(inner)
-
-
 def test_calc_threshold_hit_ratio_1e3(benchmark):
     # thousand level
     obs = np.random.rand(int(1e3))
@@ -97,17 +75,6 @@ def inner():
     benchmark(inner)
 
 
-def test_calc_threshold_hit_ratio_1e7(benchmark):
-    # ten million level
-    obs = np.random.rand(int(1e7))
-    fct = np.random.rand(int(1e7))
-
-    def inner():
-        calc_threshold_hit_ratio(obs, fct, 0.5)
-
-    benchmark(inner)
-
-
 def test_calc_threshold_false_alarm_ratio_1e3(benchmark):
     obs = np.random.rand(int(1e3))
     fct = np.random.rand(int(1e3))
@@ -129,17 +96,6 @@ def inner():
     benchmark(inner)
 
 
-def test_calc_threshold_false_alarm_ratio_1e7(benchmark):
-    # ten million level
-    obs = np.random.rand(int(1e7))
-    fct = np.random.rand(int(1e7))
-
-    def inner():
-        calc_threshold_false_alarm_ratio(obs, fct, 0.5)
-
-    benchmark(inner)
-
-
 def test_calc_threshold_ts_1e3(benchmark):
     # thousand level
     obs = np.random.rand(int(1e3))
@@ -160,14 +116,3 @@ def inner():
         calc_threshold_ts(obs, fct, 0.5)
 
     benchmark(inner)
-
-
-def test_calc_threshold_ts_1e7(benchmark):
-    # ten million level
-    obs = np.random.rand(int(1e7))
-    fct = np.random.rand(int(1e7))
-
-    def inner():
-        calc_threshold_ts(obs, fct, 0.5)
-
-    benchmark(inner)