From 49adae70c850bd84a4012a6f5f28742ff9e90710 Mon Sep 17 00:00:00 2001 From: Suraj Naik Date: Mon, 17 Jun 2024 07:25:24 +0000 Subject: [PATCH 1/8] [CORE] Support JDK 11 --- .github/workflows/velox_docker.yml | 11 +++++++++-- pom.xml | 15 +++++++++++++-- tools/gluten-it/pom.xml | 9 +++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 6c1be4344c71..cdd81f89b743 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -87,7 +87,7 @@ jobs: matrix: os: [ "ubuntu:20.04", "ubuntu:22.04" ] spark: [ "spark-3.2", "spark-3.3", "spark-3.4", "spark-3.5" ] - java: [ "java-8", "java-17" ] + java: [ "java-8", "java-11", "java-17" ] # Spark supports JDK17 since 3.3 and later, see https://issues.apache.org/jira/browse/SPARK-33772 exclude: - spark: spark-3.2 @@ -116,10 +116,13 @@ jobs: run: | if [ "${{ matrix.java }}" = "java-17" ]; then apt-get update && apt-get install -y openjdk-17-jdk maven + apt remove openjdk-11* -y + elif [ "${{ matrix.java }}" = "java-11" ]; then + apt-get update && apt-get install -y openjdk-11-jdk maven else apt-get update && apt-get install -y openjdk-8-jdk maven + apt remove openjdk-11* -y fi - apt remove openjdk-11* -y ls -l /root/.m2/repository/org/apache/arrow/arrow-dataset/15.0.0-gluten/ - name: Build and run TPCH/DS run: | @@ -175,6 +178,8 @@ jobs: run: | if [ "${{ matrix.java }}" = "java-17" ]; then yum update -y && yum install -y java-17-openjdk-devel wget + elif [ "${{ matrix.java }}" = "java-11" ]; then + yum update -y && yum install -y java-11-openjdk-devel wget else yum update -y && yum install -y java-1.8.0-openjdk-devel wget fi @@ -186,6 +191,8 @@ jobs: echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV if [ "${{ matrix.java }}" = "java-17" ]; then echo "JAVA_HOME=/usr/lib/jvm/java-17-openjdk" >> $GITHUB_ENV + elif [ "${{ matrix.java }}" = "java-11" ]; then + echo "JAVA_HOME=/usr/lib/jvm/java-11-openjdk" >> $GITHUB_ENV else echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV fi diff --git a/pom.xml b/pom.xml index 88cbb724e053..87b60e40ea85 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ 1.8 ${java.version} ${java.version} - 2.9.3 + 2.9.3 2.12 2.12.15 3 @@ -122,6 +122,16 @@ 1.8 + + java-11 + + 11 + + + 11 + 3.1.8 + + java-17 @@ -129,6 +139,7 @@ 17 + 3.1.8 @@ -301,7 +312,7 @@ com.github.ben-manes.caffeine caffeine - ${caffeine.version.java8} + ${caffeine.version} org.apache.spark diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml index ccb59ade8aa5..3f1760069792 100644 --- a/tools/gluten-it/pom.xml +++ b/tools/gluten-it/pom.xml @@ -119,6 +119,15 @@ 1.8 + + java-11 + + 11 + + + 11 + + java-17 From f8f5f3adcefed210ebaf4559c90389d1e8595d9a Mon Sep 17 00:00:00 2001 From: Suraj Naik Date: Mon, 17 Jun 2024 07:29:13 +0000 Subject: [PATCH 2/8] Update docs --- docs/developers/NewToGluten.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md index 54f379148d39..393da2df2e8d 100644 --- a/docs/developers/NewToGluten.md +++ b/docs/developers/NewToGluten.md @@ -43,7 +43,7 @@ export PATH="$PATH:$JAVA_HOME/bin" ## OpenJDK 17 -By default, Gluten compiles package using JDK8. Enable maven profile by `-Pjava-17` to use JDK17, and please make sure your JAVA_HOME points to jdk17. +By default, Gluten compiles package using JDK8. Enable maven profile by `-Pjava-17` to use JDK17 or `-Pjava-11` to use JDK 11, and please make sure your JAVA_HOME points to jdk17 or jdk11 respectively. Apache Spark and Arrow requires setting java args `-Dio.netty.tryReflectionSetAccessible=true`, see [SPARK-29924](https://issues.apache.org/jira/browse/SPARK-29924) and [ARROW-6206](https://issues.apache.org/jira/browse/ARROW-6206). So please add following configs in `spark-defaults.conf`: From 02b96a09deaf2b5662e9cbbacedf099756437ecc Mon Sep 17 00:00:00 2001 From: Suraj Naik Date: Mon, 17 Jun 2024 10:15:00 +0000 Subject: [PATCH 3/8] Explicitly remove java 11 --- .github/workflows/velox_docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index cdd81f89b743..4a5fcc092069 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -517,6 +517,7 @@ jobs: - name: Setup java and maven run: | apt-get update && apt-get install -y openjdk-8-jdk maven wget + apt remove openjdk-11* -y - name: Build for Spark ${{ matrix.spark }} run: | cd $GITHUB_WORKSPACE/ From 1d10af78a3f0090603c3579320aa3e798e8d2b53 Mon Sep 17 00:00:00 2001 From: Suraj Naik Date: Mon, 17 Jun 2024 10:19:38 +0000 Subject: [PATCH 4/8] Set JAVA_HOME --- .github/workflows/velox_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 4a5fcc092069..6c5793bf731c 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -518,13 +518,13 @@ jobs: run: | apt-get update && apt-get install -y openjdk-8-jdk maven wget apt remove openjdk-11* -y + export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 - name: Build for Spark ${{ matrix.spark }} run: | cd $GITHUB_WORKSPACE/ $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -Pceleborn -DskipTests - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with ${{ matrix.celeborn }} run: | - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 EXTRA_PROFILE="" if [ "${{ matrix.celeborn }}" = "celeborn-0.4.0" ]; then EXTRA_PROFILE="-Pceleborn-0.4" From 16e40a437e1186130e5f36df2e1d4ba6ae4eaf5d Mon Sep 17 00:00:00 2001 From: Suraj Naik Date: Mon, 17 Jun 2024 11:42:46 +0000 Subject: [PATCH 5/8] Fix --- .github/workflows/velox_docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 6c5793bf731c..973f24f4bdc9 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -525,6 +525,7 @@ jobs: $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -Pceleborn -DskipTests - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with ${{ matrix.celeborn }} run: | + export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 EXTRA_PROFILE="" if [ "${{ matrix.celeborn }}" = "celeborn-0.4.0" ]; then EXTRA_PROFILE="-Pceleborn-0.4" From 4641ec64e50cf262ba0f45dbde0da9ec9dec8336 Mon Sep 17 00:00:00 2001 From: Suraj Naik Date: Wed, 19 Jun 2024 04:56:17 +0000 Subject: [PATCH 6/8] Run CI using Java-11 only for Spark 3.5 --- .github/workflows/velox_docker.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 973f24f4bdc9..adef619787c7 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -96,8 +96,16 @@ jobs: java: java-17 - spark: spark-3.5 java: java-17 + - spark: spark-3.2 + java: java-11 + - spark: spark-3.3 + java: java-11 + - spark: spark-3.4 + java: java-11 - os: ubuntu:22.04 java: java-17 + - os: ubuntu:22.04 + java: java-11 runs-on: ubuntu-20.04 container: ${{ matrix.os }} steps: @@ -144,7 +152,7 @@ jobs: matrix: os: [ "centos:7", "centos:8" ] spark: [ "spark-3.2", "spark-3.3", "spark-3.4", "spark-3.5" ] - java: [ "java-8", "java-17" ] + java: [ "java-8", "java-11", "java-17" ] # Spark supports JDK17 since 3.3 and later, see https://issues.apache.org/jira/browse/SPARK-33772 exclude: - spark: spark-3.2 @@ -153,8 +161,16 @@ jobs: java: java-17 - spark: spark-3.5 java: java-17 + - spark: spark-3.2 + java: java-11 + - spark: spark-3.3 + java: java-11 + - spark: spark-3.4 + java: java-11 - os: centos:7 java: java-17 + - os: centos:7 + java: java-11 runs-on: ubuntu-20.04 container: ${{ matrix.os }} steps: From e080415ff2d05cd1070c3216564d886a2965ae23 Mon Sep 17 00:00:00 2001 From: Suraj Naik Date: Wed, 19 Jun 2024 05:27:16 +0000 Subject: [PATCH 7/8] Fix github env --- .github/workflows/velox_docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index adef619787c7..d5427360588a 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -534,14 +534,13 @@ jobs: run: | apt-get update && apt-get install -y openjdk-8-jdk maven wget apt remove openjdk-11* -y - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 + export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 >> $GITHUB_ENV - name: Build for Spark ${{ matrix.spark }} run: | cd $GITHUB_WORKSPACE/ $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -Pceleborn -DskipTests - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with ${{ matrix.celeborn }} run: | - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 EXTRA_PROFILE="" if [ "${{ matrix.celeborn }}" = "celeborn-0.4.0" ]; then EXTRA_PROFILE="-Pceleborn-0.4" From 35daae8bfb5ca68c2607eadafb32e895ff4eaf5b Mon Sep 17 00:00:00 2001 From: Suraj Naik Date: Wed, 19 Jun 2024 05:29:26 +0000 Subject: [PATCH 8/8] Fix github env --- .github/workflows/velox_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index 431d17678d00..eff1d4da0906 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -534,7 +534,7 @@ jobs: run: | apt-get update && apt-get install -y openjdk-8-jdk maven wget apt remove openjdk-11* -y - export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 >> $GITHUB_ENV + echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV - name: Build for Spark ${{ matrix.spark }} run: | cd $GITHUB_WORKSPACE/