test benchmarking

.github/workflows/benchmark.yml

@@ -3,77 +3,80 @@ name: benchmark
 on:
   workflow_dispatch:
   pull_request:
+    paths:
+      - "adbpyg_adapter/*.py"
+      - "benchmark/*.py"
   push:
       branches:
         - master
       paths:
-        - "src/*.py"
+        - "adbpyg_adapter/*.py"
         - "benchmark/*.py"
 
 jobs:
-  # write_traces:
-  #   if: github.event_name == 'push'
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #       with:
-  #         fetch-depth: 0
-
-  #     - name: Create new branch
-  #       run: git checkout -b actions/benchmark
-
-  #     - name: Set branch upstream
-  #       run: git push -u origin actions/benchmark
-  #       env:
-  #         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  #     - name: Setup Python
-  #       uses: actions/setup-python@v4
-  #       with:
-  #         cache: 'pip'
-  #         cache-dependency-path: setup.py
-
-  #     - name: Set up ArangoDB Instance via Docker
-  #       run: docker run -d --name arangodb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb
+  generate_master_traces:
+    if: github.event_name == 'push'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Create new branch
+        run: git checkout -b actions/benchmark
+
+      - name: Set branch upstream
+        run: git push -u origin actions/benchmark
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          cache: 'pip'
+          cache-dependency-path: setup.py
+
+      - name: Start ArangoDB Instance
+        run: docker run -d --name arangodb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb
+
+      - name: Start Jaeger Instance
+        run: docker run -d --name jaeger --rm -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 -p 16686:16686 -p 4317:4317 -p 4318:4318 -p 9411:9411 jaegertracing/all-in-one:latest
 
-  #     - name: Start Jaeger Instance
-  #       run: docker run -d --name jaeger --rm \
-  #         -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
-  #         -p 16686:16686 \
-  #         -p 4317:4317 \
-  #         -p 4318:4318 \
-  #         -p 9411:9411 \
-  #         jaegertracing/all-in-one:latest
-
-  #     - name: Install packages
-  #       run: |
-  #         pip install torch==2.1.0
-  #         pip install torch-scatter torch-sparse -f https://data.pyg.org/whl/torch-$(python -c 'import torch; print(torch.__version__.split("+")[0])')+cpu.html
-  #         pip install -e '.[dev, tracing]'
-
-  #     - name: Run Python Script
-  #       run: python benchmark/write.py --output_dir head
-
-  #     - name: Make commit for auto-generated benchmark files
-  #       uses: EndBug/add-and-commit@v9
-  #       env:
-  #         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  #       with:
-  #         add: "./benchmark/main/*.json"
-  #         new_branch: actions/benchmark
-  #         message: "generate benchmark files for $GITHUB_SHA"
-
-  #     - name: Create pull request for the auto generated changelog
-  #       run: |
-  #         echo "PR_URL=$(gh pr create \
-  #           --title "benchmark: $GITHUB_SHA" \
-  #           --body "beep boop, i am a robot ($GITHUB_SHA)" \
-  #           --label documentation)" >> $GITHUB_ENV
-  #       env:
-  #         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install packages
+        run: |
+          pip install torch==2.1.0
+          pip install torch-scatter torch-sparse -f https://data.pyg.org/whl/torch-$(python -c 'import torch; print(torch.__version__.split("+")[0])')+cpu.html
+          pip install -e '.[dev, tracing]'
+
+      - name: Run Python Script
+        run: python benchmark/write.py --output_dir master
+
+      - name: Echo PyG to ArangoDB
+        run: cat benchmark/traces/master/pyg_to_arangodb.json | jq .
+
+      - name: Echo ArangoDB to PyG 
+        run: cat benchmark/traces/master/arangodb_to_pyg.json | jq .
+
+      - name: Make commit for auto-generated benchmark files
+        uses: EndBug/add-and-commit@v9
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          add: "./benchmark/traces/master/*.json"
+          new_branch: actions/benchmark
+          message: "generate benchmark files for $GITHUB_SHA"
+
+      - name: Create pull request for the auto generated benchmark
+        run: |
+          echo "PR_URL=$(gh pr create \
+            --title "benchmark: $GITHUB_SHA" \
+            --body "beep boop, i am a robot ($GITHUB_SHA)" \
+            --label documentation)" >> $GITHUB_ENV
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  #     - name: Alert developer of open PR
-  #       run: echo "Benchmark $PR_URL is ready to be merged by developer."
+      - name: Alert developer of open PR
+        run: echo "Benchmark $PR_URL is ready to be merged by developer."
 
   compare_traces:
     if: github.event_name == 'pull_request'
@@ -104,11 +107,40 @@ jobs:
       - name: Write PR traces
         run: python benchmark/write.py --output_dir branch
 
-      - name: Compare PR traces against Master traces
-        run: python benchmark/compare.py
-
-      - name: Echo PyG to ArangoDB Comparison
-        run: cat benchmark/diff/pyg_to_arangodb.json | jq
+      - name: Compare PR & Master Traces
+        run: |
+          echo "ROOT_SPAN_COMPARISON<<EOF" >> $GITHUB_ENV
+          python benchmark/compare.py >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
 
-      - name: Echo ArangoDB to PyG Comparison
-        run: cat benchmark/diff/arangodb_to_pyg.json | jq
+      - name: Prepare PR Comment
+        run: |
+          echo 'MESSAGE<<EOF' >> $GITHUB_ENV
+          echo "Benchmark (${{ github.event.pull_request.head.sha }})" >> $GITHUB_ENV
+          echo '' >> $GITHUB_ENV
+          echo '```json' >> $GITHUB_ENV
+          echo "$ROOT_SPAN_COMPARISON" | jq . >> $GITHUB_ENV
+          echo '```' >> $GITHUB_ENV
+          echo '' >> $GITHUB_ENV
+          echo "See the full diff [here](https://github.com/arangoml/pyg-adapter/actions/runs/$GITHUB_RUN_ID)" >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+
+      - name: Post PR Comment
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            const prNumber = context.payload.pull_request.number;
+            const message = process.env.MESSAGE;
+            github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+              body: message
+            });
+
+      - name: Echo Full PyG to ArangoDB Diff
+        run: cat benchmark/traces/diff/pyg_to_arangodb.json | jq .
+
+      - name: Echo Full ArangoDB to PyG Diff
+        run: cat benchmark/traces/diff/arangodb_to_pyg.json | jq .

.github/workflows/build.yml

@@ -34,7 +34,7 @@ jobs:
         run: |
           pip install torch==2.1.0
           pip install torch-scatter torch-sparse -f https://data.pyg.org/whl/torch-$(python -c 'import torch; print(torch.__version__.split("+")[0])')+cpu.html
-          pip install -e '.[dev]'
+          pip install -e '.[dev, tracing]'
 
       - name: Run black
         run: black --check --verbose --diff --color ${{env.PACKAGE_DIR}} ${{env.TESTS_DIR}}

.github/workflows/release.yml

@@ -19,10 +19,10 @@ jobs:
           python-version: "3.10"
 
       - name: Install release packages
-        run: pip install setuptools wheel twine setuptools-scm[toml]
+        run: pip install build twine
 
       - name: Build distribution
-        run: python setup.py sdist bdist_wheel
+        run: python -m build
 
       - name: Publish to Test PyPi
         env:

.gitignore

@@ -122,4 +122,4 @@ tests/data/pyg
 
 # "Current" Benchmark Results
 benchmark/traces/branch/*.json
-benchmark/diff/*.json
+benchmark/traces/diff/*.json