Merge pull request #226 from Gabriel39/doris_perf

Update Apache Doris

doris/benchmark.sh

@@ -1,15 +1,15 @@
 #!/bin/bash
 set -ex
 
-# This benchmark should run on Amazon Linux
+# This benchmark should run on Ubuntu 20.04
 
 # Install
 ROOT=$(pwd)
 
 if [[ -n "$1" ]]; then
     url="$1"
 else
-    url='https://doris-release.s3.us-east-1.amazonaws.com/1.2/doris-1.2.0.alpha-x86_64.tar.gz'
+    url='https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-2.1.7-rc01-bin-x64.tar.gz'
 fi
 # Download
 file_name="$(basename ${url})"
@@ -22,34 +22,34 @@ if [[ "$url" == "http"* ]]; then
 fi
 dir_name="${file_name/.tar.gz/}"
 
-# Try to stop SelectDB and remove it first if execute this script multiple times
+# Try to stop Doris and remove it first if execute this script multiple times
 set +e
-"$dir_name"/fe/bin/stop_fe.sh
-"$dir_name"/be/bin/stop_be.sh
+"$dir_name"/apache-doris-2.1.7-rc01-bin-x64/fe/bin/stop_fe.sh
+"$dir_name"/apache-doris-2.1.7-rc01-bin-x64/be/bin/stop_be.sh
 rm -rf "$dir_name"
 set -e
 
 # Uncompress
 mkdir "$dir_name"
 tar zxf "$file_name" -C "$dir_name"
-DORIS_HOME="$ROOT/$dir_name/"
+DORIS_HOME="$ROOT/$dir_name/apache-doris-2.1.7-rc01-bin-x64"
 export DORIS_HOME
 
 # Install dependencies
-sudo yum install -y mysql java-11-amazon-corretto.x86_64
-export JAVA_HOME="/usr/lib/jvm/java-11-amazon-corretto.x86_64/"
+sudo apt update
+sudo apt install -y openjdk-17-jdk
+sudo apt install -y mysql-client
+export JAVA_HOME="/usr/lib/jvm/java-17-openjdk-amd64/"
 export PATH=$JAVA_HOME/bin:$PATH
 
-IPADDR=$(hostname -i)
+sudo systemctl disable unattended-upgrades
+sudo systemctl stop unattended-upgrades
 
-# Start Frontend
-echo "priority_networks = ${IPADDR}/24" >>"$DORIS_HOME"/fe/conf/fe_custom.conf
 "$DORIS_HOME"/fe/bin/start_fe.sh --daemon
 
 # Start Backend
-echo "priority_networks = ${IPADDR}/24
-load_process_max_memory_limit_percent=80" >>"$DORIS_HOME"/be/conf/be_custom.conf
 sudo sysctl -w vm.max_map_count=2000000
+ulimit -n 65535
 "$DORIS_HOME"/be/bin/start_be.sh --daemon
 
 # Wait for Frontend ready
@@ -65,7 +65,7 @@ while true; do
 done
 
 # Setup cluster, add Backend to cluster
-mysql -h 127.0.0.1 -P9030 -uroot -e "ALTER SYSTEM ADD BACKEND '${IPADDR}:9050' "
+mysql -h 127.0.0.1 -P9030 -uroot -e "ALTER SYSTEM ADD BACKEND '127.0.0.1:9050' "
 
 # Wait for Backend ready
 while true; do
@@ -79,6 +79,8 @@ while true; do
     fi
 done
 
+echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null
+
 # Create Database and table
 mysql -h 127.0.0.1 -P9030 -uroot -e "CREATE DATABASE hits"
 sleep 5

doris/create.sql

@@ -106,5 +106,5 @@ CREATE TABLE hits (
     CLID INT NOT NULL
 )  
 DUPLICATE KEY (CounterID, EventDate, UserID, EventTime, WatchID) 
-DISTRIBUTED BY HASH(UserID) BUCKETS 16
+DISTRIBUTED BY HASH(UserID) BUCKETS 192
 PROPERTIES ( "replication_num"="1");

doris/results/c6a.metal.json

@@ -0,0 +1,55 @@
+{
+    "system": "Apache Doris",
+    "date": "2024-09-19",
+    "machine": "c6a.metal, 500gb gp2",
+    "cluster_size": 1,
+    "comment": "",
+    "tags": ["C++", "column-oriented", "MySQL compatible", "ClickHouse derivative"],
+    "load_time": 451,
+    "data_size": 17361511279,
+    "result": [
+[0.07,0.02,0.02],
+[0.11,0.02,0.02],
+[1.10,0.03,0.04],
+[1.69,0.05,0.05],
+[1.70,0.15,0.16],
+[1.53,0.20,0.20],
+[0.04,0.02,0.02],
+[0.07,0.03,0.03],
+[2.58,0.24,0.23],
+[3.55,0.25,0.25],
+[2.53,0.08,0.07],
+[3.11,0.07,0.08],
+[1.87,0.22,0.19],
+[3.67,0.28,0.25],
+[2.20,0.24,0.23],
+[1.49,0.11,0.12],
+[3.82,0.30,0.30],
+[2.97,0.08,0.08],
+[4.91,0.50,0.50],
+[0.05,0.01,0.02],
+[11.30,0.23,0.11],
+[13.35,0.12,0.06],
+[25.53,0.17,0.08],
+[7.30,1.37,0.05],
+[2.58,0.19,0.24],
+[2.17,0.04,0.05],
+[3.20,0.04,0.03],
+[11.55,0.24,0.22],
+[9.48,0.98,0.92],
+[0.49,0.05,0.05],
+[4.99,0.13,0.14],
+[6.96,0.16,0.15],
+[4.94,0.88,0.83],
+[11.61,1.07,0.99],
+[11.65,1.06,0.95],
+[0.87,0.18,0.17],
+[1.75,0.04,0.04],
+[2.14,0.04,0.03],
+[1.82,0.02,0.02],
+[2.23,0.08,0.08],
+[1.37,0.03,0.03],
+[2.01,0.03,0.02],
+[1.11,0.03,0.03]
+	]
+}

doris/run.sh

@@ -6,6 +6,7 @@ touch result.csv
 truncate -s0 result.csv
 
 while read -r query; do
+    curl http://127.0.0.1:8040/api/clear_cache/all
     sync
     echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null