Merge branch 'main' into main

.github/actions/reload/action.yaml

@@ -36,6 +36,11 @@ runs:
         echo "SHOW WAREHOUSES;" | bendsql
         echo "BENDSQL_DSN=${BENDSQL_DSN}&warehouse=${{ inputs.warehouse }}" >> $GITHUB_ENV
 
+    - name: Get start time
+      shell: bash
+      run: |
+        echo "START_TIME=$(date +%s)" >> $GITHUB_ENV
+
     - name: Run reload script
       shell: bash
       run: |
@@ -44,6 +49,12 @@ runs:
         AWS_KEY_ID: ${{ inputs.aws_key_id }}
         AWS_SECRET_KEY: ${{ inputs.aws_secret_key }}
 
+    - name: Get time elapsed
+      if: ${{ always() }}
+      shell: bash
+      run: |
+        echo "ELAPSED_SECONDS=$(expr $(date +%s) - $START_TIME)" >> $GITHUB_ENV
+
     - name: Notify Feishu Group
       uses: foxundermoon/feishu-action@v2
       if: ${{ always() }}
@@ -53,6 +64,6 @@ runs:
         content: |
           post:
             zh_cn:
-              title: Reload ${{ inputs.script }} ${{ github.action_status }}
+              title: Reload ${{ inputs.script }} ${{ github.action_status }} after ${{ env.ELAPSED_SECONDS }} seconds
               content:
               - [{"tag": "a", "text": "link", "href": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"}]

.github/actions/setup-bendsql/action.yaml

@@ -7,9 +7,8 @@ runs:
     - name: Get BendSQL from release
       shell: bash
       run: |
-        curl -fsSLo ./bendsql.tar.gz https://github.com/datafuselabs/bendsql/releases/download/v0.3.0/bendsql-x86_64-unknown-linux-gnu.tar.gz
-        tar -xvzf ./bendsql.tar.gz
-        chmod +x ./bendsql
-        sudo cp ./bendsql /usr/local/bin/bendsql
-        rm -rf ./bendsql
-        rm -rf ./bendsql.tar.gz
+        sudo curl -L -o /usr/share/keyrings/datafuselabs-keyring.gpg https://repo.databend.rs/deb/datafuselabs.gpg
+        sudo curl -L -o /etc/apt/sources.list.d/datafuselabs.list https://repo.databend.rs/deb/datafuselabs.list
+        sudo apt update
+        sudo apt install -y bendsql
+        bendsql --version

.github/workflows/perf.yaml

@@ -27,20 +27,22 @@ on:
 jobs:
   perf:
     name: Databend Release Perf
-    runs-on: [self-hosted, X64, Linux, 4c8g]
+    runs-on: [self-hosted, X64, Linux, 4c8g, gcp]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           ref: ${{ github.head_ref }}
 
       - name: Get Databend version
+        env:
+          GH_TOKEN: ${{ github.token }}
         run: |
           if [ -z "${{ github.event.inputs.imageTag }}" ]; then
-            LASTEST_TAG=$(curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -s GET https://api.github.com/repos/datafuselabs/databend/tags\?per_page\=1 | jq -r '.[].name')
-            echo "DATABEND_VERSION=${LASTEST_TAG}" >> $GITHUB_ENV
+            LASTEST_RELEASE=$(gh api repos/datafuselabs/databend/tags -X GET -f per_page=1 | jq -r .[].name)
+            echo "DATABEND_VERSION=${LASTEST_RELEASE}" >> $GITHUB_ENV
           else
             echo "DATABEND_VERSION=${{ github.event.inputs.imageTag }}" >> $GITHUB_ENV
           fi
@@ -86,7 +88,6 @@ jobs:
             --size XLarge
 
           mv collector/numbers.json collector/numbers/${{ env.PERF_DATE }}-numbers.json
-          mv collector/ontime.json collector/ontime/${{ env.PERF_DATE }}-ontime.json
           mv collector/hits.json collector/hits/${{ env.PERF_DATE }}-hits.json
 
       - name: Clean warehouse

.github/workflows/reload.hits.yaml

@@ -1,11 +1,11 @@
 name: Reload hits
 
 on:
-  # pull_request:
-  #   branches:
-  #     - main
-  #   paths:
-  #     - "reload/hits.sh"
+  pull_request:
+    branches:
+      - main
+    paths:
+      - "reload/hits.sh"
   push:
     branches:
       - main

.github/workflows/reuse.reload.yaml

@@ -10,10 +10,10 @@ on:
 jobs:
   reload:
     name: Reload
-    runs-on: [self-hosted, X64, Linux, 4c8g]
+    runs-on: [self-hosted, X64, Linux, 4c8g, gcp]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           ref: ${{ github.head_ref }}

benchmarks/hits.yaml

@@ -59,32 +59,32 @@ statements:
   - name: Q28
     query: "SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;"
   - name: Q29
-    query: "SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\\.)?([^/]+)/.*$', '$1') AS k, AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;"
+    query: "SELECT REGEXP_REPLACE(Referrer, '^https?://(?:www\\.)?([^/]+)/.*$', '$1') AS k, AVG(length(Referrer)) AS l, COUNT(*) AS c, MIN(Referrer) FROM hits WHERE Referrer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25;"
   - name: Q30
     query: "SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), SUM(ResolutionWidth + 2), SUM(ResolutionWidth + 3), SUM(ResolutionWidth + 4), SUM(ResolutionWidth + 5), SUM(ResolutionWidth + 6), SUM(ResolutionWidth + 7), SUM(ResolutionWidth + 8), SUM(ResolutionWidth + 9), SUM(ResolutionWidth + 10), SUM(ResolutionWidth + 11), SUM(ResolutionWidth + 12), SUM(ResolutionWidth + 13), SUM(ResolutionWidth + 14), SUM(ResolutionWidth + 15), SUM(ResolutionWidth + 16), SUM(ResolutionWidth + 17), SUM(ResolutionWidth + 18), SUM(ResolutionWidth + 19), SUM(ResolutionWidth + 20), SUM(ResolutionWidth + 21), SUM(ResolutionWidth + 22), SUM(ResolutionWidth + 23), SUM(ResolutionWidth + 24), SUM(ResolutionWidth + 25), SUM(ResolutionWidth + 26), SUM(ResolutionWidth + 27), SUM(ResolutionWidth + 28), SUM(ResolutionWidth + 29), SUM(ResolutionWidth + 30), SUM(ResolutionWidth + 31), SUM(ResolutionWidth + 32), SUM(ResolutionWidth + 33), SUM(ResolutionWidth + 34), SUM(ResolutionWidth + 35), SUM(ResolutionWidth + 36), SUM(ResolutionWidth + 37), SUM(ResolutionWidth + 38), SUM(ResolutionWidth + 39), SUM(ResolutionWidth + 40), SUM(ResolutionWidth + 41), SUM(ResolutionWidth + 42), SUM(ResolutionWidth + 43), SUM(ResolutionWidth + 44), SUM(ResolutionWidth + 45), SUM(ResolutionWidth + 46), SUM(ResolutionWidth + 47), SUM(ResolutionWidth + 48), SUM(ResolutionWidth + 49), SUM(ResolutionWidth + 50), SUM(ResolutionWidth + 51), SUM(ResolutionWidth + 52), SUM(ResolutionWidth + 53), SUM(ResolutionWidth + 54), SUM(ResolutionWidth + 55), SUM(ResolutionWidth + 56), SUM(ResolutionWidth + 57), SUM(ResolutionWidth + 58), SUM(ResolutionWidth + 59), SUM(ResolutionWidth + 60), SUM(ResolutionWidth + 61), SUM(ResolutionWidth + 62), SUM(ResolutionWidth + 63), SUM(ResolutionWidth + 64), SUM(ResolutionWidth + 65), SUM(ResolutionWidth + 66), SUM(ResolutionWidth + 67), SUM(ResolutionWidth + 68), SUM(ResolutionWidth + 69), SUM(ResolutionWidth + 70), SUM(ResolutionWidth + 71), SUM(ResolutionWidth + 72), SUM(ResolutionWidth + 73), SUM(ResolutionWidth + 74), SUM(ResolutionWidth + 75), SUM(ResolutionWidth + 76), SUM(ResolutionWidth + 77), SUM(ResolutionWidth + 78), SUM(ResolutionWidth + 79), SUM(ResolutionWidth + 80), SUM(ResolutionWidth + 81), SUM(ResolutionWidth + 82), SUM(ResolutionWidth + 83), SUM(ResolutionWidth + 84), SUM(ResolutionWidth + 85), SUM(ResolutionWidth + 86), SUM(ResolutionWidth + 87), SUM(ResolutionWidth + 88), SUM(ResolutionWidth + 89) FROM hits;"
   - name: Q31
-    query: "SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;"
+    query: "SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(Refresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10;"
   - name: Q32
-    query: "SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;"
+    query: "SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(Refresh), AVG(ResolutionWidth) FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;"
   - name: Q33
-    query: "SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;"
+    query: "SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(Refresh), AVG(ResolutionWidth) FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10;"
   - name: Q34
     query: "SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10;"
   - name: Q35
     query: "SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10;"
   - name: Q36
     query: "SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10;"
   - name: Q37
-    query: "SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;"
+    query: "SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND Refresh = 0 AND URL <> '' GROUP BY URL ORDER BY PageViews DESC LIMIT 10;"
   - name: Q38
-    query: "SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;"
+    query: "SELECT Title, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND DontCountHits = 0 AND Refresh = 0 AND Title <> '' GROUP BY Title ORDER BY PageViews DESC LIMIT 10;"
   - name: Q39
-    query: "SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;"
+    query: "SELECT URL, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND Refresh = 0 AND IsLink <> 0 AND IsDownload = 0 GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;"
   - name: Q40
-    query: "SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;"
+    query: "SELECT TrafficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referrer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND Refresh = 0 GROUP BY TrafficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000;"
   - name: Q41
-    query: "SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;"
+    query: "SELECT URLHash, EventDate, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND Refresh = 0 AND TrafficSourceID IN (-1, 6) AND ReferrerHash = 3594120000172545465 GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100;"
   - name: Q42
-    query: "SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;"
+    query: "SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND Refresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000;"
   - name: Q43
-    query: "SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND IsRefresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;"
+    query: "SELECT DATE_TRUNC('minute', EventTime) AS M, COUNT(*) AS PageViews FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' AND Refresh = 0 AND DontCountHits = 0 GROUP BY DATE_TRUNC('minute', EventTime) ORDER BY DATE_TRUNC('minute', EventTime) LIMIT 10 OFFSET 1000;"

collector/numbers/2023-07-10-numbers.json

@@ -0,0 +1 @@
+{"metadata":{"table":"numbers","tag":"","size":"XLarge"},"schema":[{"name":"Q1","sql":"SELECT avg(number) FROM numbers_mt(10000000000);","min":0.716,"max":0.838,"median":0.7204999999999999,"std_dev":0.03523194573111169,"read_row":10000000000,"read_byte":80000000000,"time":[0.838,0.727,0.719,0.721,0.718,0.729,0.721,0.72,0.716,0.72],"error":[],"mean":0.732117492700239},{"name":"Q2","sql":"SELECT sum(number) FROM numbers_mt(10000000000);","min":0.714,"max":0.76,"median":0.719,"std_dev":0.013023440405668552,"read_row":10000000000,"read_byte":80000000000,"time":[0.76,0.714,0.722,0.716,0.72,0.715,0.716,0.718,0.722,0.73],"error":[],"mean":0.7231856644697298},{"name":"Q3","sql":"SELECT min(number) FROM numbers_mt(10000000000);","min":0.847,"max":0.901,"median":0.8494999999999999,"std_dev":0.015738170160472926,"read_row":10000000000,"read_byte":80000000000,"time":[0.85,0.849,0.86,0.847,0.848,0.901,0.847,0.85,0.847,0.852],"error":[],"mean":0.8549593106072945},{"name":"Q4","sql":"SELECT max(number) FROM numbers_mt(10000000000);","min":1.031,"max":1.036,"median":1.0339999999999998,"std_dev":0.002022374841615706,"read_row":10000000000,"read_byte":80000000000,"time":[1.032,1.036,1.036,1.031,1.033,1.036,1.033,1.031,1.035,1.036],"error":[],"mean":1.0338980214652433},{"name":"Q5","sql":"SELECT count(number) FROM numbers_mt(10000000000);","min":0.47,"max":0.574,"median":0.4745,"std_dev":0.03797117327657916,"read_row":10000000000,"read_byte":80000000000,"time":[0.574,0.474,0.47,0.472,0.564,0.472,0.473,0.476,0.475,0.487],"error":[],"mean":0.49233893939731693}]}