[CORE] Support JDK 11 (#6112)

apache · Jun 20, 2024 · bfd089e · bfd089e
1 parent 716dd9a
commit bfd089e
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 7 deletions.
diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml
@@ -87,7 +87,7 @@ jobs:
       matrix:
         os: [ "ubuntu:20.04", "ubuntu:22.04" ]
         spark: [ "spark-3.2", "spark-3.3", "spark-3.4", "spark-3.5" ]
-        java: [ "java-8", "java-17" ]
+        java: [ "java-8", "java-11", "java-17" ]
         # Spark supports JDK17 since 3.3 and later, see https://issues.apache.org/jira/browse/SPARK-33772
         exclude:
           - spark: spark-3.2
@@ -96,8 +96,16 @@ jobs:
             java: java-17
           - spark: spark-3.5
             java: java-17
+          - spark: spark-3.2
+            java: java-11
+          - spark: spark-3.3
+            java: java-11
+          - spark: spark-3.4
+            java: java-11
           - os: ubuntu:22.04
             java: java-17
+          - os: ubuntu:22.04
+            java: java-11
     runs-on: ubuntu-20.04
     container: ${{ matrix.os }}
     steps:
@@ -116,10 +124,13 @@ jobs:
         run: |
           if [ "${{ matrix.java }}" = "java-17" ]; then
             apt-get update && apt-get install -y openjdk-17-jdk maven
+            apt remove openjdk-11* -y
+          elif [ "${{ matrix.java }}" = "java-11" ]; then
+            apt-get update && apt-get install -y openjdk-11-jdk maven
           else
             apt-get update && apt-get install -y openjdk-8-jdk maven
+            apt remove openjdk-11* -y
           fi
-          apt remove openjdk-11* -y
           ls -l /root/.m2/repository/org/apache/arrow/arrow-dataset/15.0.0-gluten/
       - name: Build and run TPCH/DS
         run: |
@@ -141,7 +152,7 @@ jobs:
       matrix:
         os: [ "centos:7", "centos:8" ]
         spark: [ "spark-3.2", "spark-3.3", "spark-3.4", "spark-3.5" ]
-        java: [ "java-8", "java-17" ]
+        java: [ "java-8", "java-11", "java-17" ]
         # Spark supports JDK17 since 3.3 and later, see https://issues.apache.org/jira/browse/SPARK-33772
         exclude:
           - spark: spark-3.2
@@ -150,8 +161,16 @@ jobs:
             java: java-17
           - spark: spark-3.5
             java: java-17
+          - spark: spark-3.2
+            java: java-11
+          - spark: spark-3.3
+            java: java-11
+          - spark: spark-3.4
+            java: java-11
           - os: centos:7
             java: java-17
+          - os: centos:7
+            java: java-11
     runs-on: ubuntu-20.04
     container: ${{ matrix.os }}
     steps:
@@ -175,6 +194,8 @@ jobs:
         run: |
           if [ "${{ matrix.java }}" = "java-17" ]; then
             yum update -y && yum install -y java-17-openjdk-devel wget
+          elif [ "${{ matrix.java }}" = "java-11" ]; then
+            yum update -y && yum install -y java-11-openjdk-devel wget
           else
             yum update -y && yum install -y java-1.8.0-openjdk-devel wget
           fi
@@ -186,6 +207,8 @@ jobs:
           echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
           if [ "${{ matrix.java }}" = "java-17" ]; then
             echo "JAVA_HOME=/usr/lib/jvm/java-17-openjdk" >> $GITHUB_ENV
+          elif [ "${{ matrix.java }}" = "java-11" ]; then
+            echo "JAVA_HOME=/usr/lib/jvm/java-11-openjdk" >> $GITHUB_ENV
           else
             echo "JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk" >> $GITHUB_ENV
           fi
@@ -510,13 +533,14 @@ jobs:
       - name: Setup java and maven
         run: |
           apt-get update && apt-get install -y openjdk-8-jdk maven wget
+          apt remove openjdk-11* -y
+          echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV
       - name: Build for Spark ${{ matrix.spark }}
         run: |
           cd $GITHUB_WORKSPACE/
           $MVN_CMD clean install -P${{ matrix.spark }} -Pbackends-velox -Pceleborn -DskipTests
       - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with ${{ matrix.celeborn }}
         run: |
-          export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
           EXTRA_PROFILE=""
           if [ "${{ matrix.celeborn }}" = "celeborn-0.4.0" ]; then
             EXTRA_PROFILE="-Pceleborn-0.4"

diff --git a/docs/developers/NewToGluten.md b/docs/developers/NewToGluten.md
@@ -43,7 +43,7 @@ export PATH="$PATH:$JAVA_HOME/bin"
 
 ## OpenJDK 17
 
-By default, Gluten compiles package using JDK8. Enable maven profile by `-Pjava-17` to use JDK17, and please make sure your JAVA_HOME points to jdk17.
+By default, Gluten compiles package using JDK8. Enable maven profile by `-Pjava-17` to use JDK17 or `-Pjava-11` to use JDK 11, and please make sure your JAVA_HOME points to jdk17 or jdk11 respectively.
 
 Apache Spark and Arrow requires setting java args `-Dio.netty.tryReflectionSetAccessible=true`, see [SPARK-29924](https://issues.apache.org/jira/browse/SPARK-29924) and [ARROW-6206](https://issues.apache.org/jira/browse/ARROW-6206).
 So please add following configs in `spark-defaults.conf`:

diff --git a/pom.xml b/pom.xml
@@ -41,7 +41,7 @@
     <java.version>1.8</java.version>
     <maven.compiler.source>${java.version}</maven.compiler.source>
     <maven.compiler.target>${java.version}</maven.compiler.target>
-    <caffeine.version.java8>2.9.3</caffeine.version.java8>
+    <caffeine.version>2.9.3</caffeine.version>
     <scala.binary.version>2.12</scala.binary.version>
     <scala.version>2.12.15</scala.version>
     <spark.major.version>3</spark.major.version>
@@ -122,13 +122,24 @@
         <java.version>1.8</java.version>
       </properties>
     </profile>
+    <profile>
+      <id>java-11</id>
+      <activation>
+        <jdk>11</jdk>
+      </activation>
+      <properties>
+        <java.version>11</java.version>
+        <caffeine.version>3.1.8</caffeine.version>
+      </properties>
+    </profile>
     <profile>
       <id>java-17</id>
       <activation>
         <jdk>17</jdk>
       </activation>
       <properties>
         <java.version>17</java.version>
+        <caffeine.version>3.1.8</caffeine.version>
       </properties>
     </profile>
     <profile>
@@ -301,7 +312,7 @@
       <dependency>
         <groupId>com.github.ben-manes.caffeine</groupId>
         <artifactId>caffeine</artifactId>
-        <version>${caffeine.version.java8}</version>
+        <version>${caffeine.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.spark</groupId>

diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml
@@ -119,6 +119,15 @@
         <java.version>1.8</java.version>
       </properties>
     </profile>
+    <profile>
+      <id>java-11</id>
+      <activation>
+        <jdk>11</jdk>
+      </activation>
+      <properties>
+        <java.version>11</java.version>
+      </properties>
+    </profile>
     <profile>
       <id>java-17</id>
       <activation>