[VL] Add a docker build job and reuse pre-built arrow libs

.github/workflows/docker_image.yml

@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build and Push Docker Image
+
+on:
+  pull_request:
+    paths:
+      - '.github/workflows/docker_image.yml'
+  schedule:
+    - cron: '0 20 * * 0'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: dev/vcpkg/docker/gha-centos-7.dockerfile
+          push: true
+          tags: apache/gluten:vcpkg-centos-7

.github/workflows/velox_docker.yml

@@ -51,7 +51,7 @@ concurrency:
 jobs:
   build-native-lib-centos-7:
     runs-on: ubuntu-20.04
-    container: apache/gluten:gluten-vcpkg-builder_2024_08_05 # centos7 with dependencies installed
+    container: apache/gluten:vcpkg-centos-7
     steps:
       - uses: actions/checkout@v2
       - name: Generate cache key
@@ -63,7 +63,6 @@ jobs:
         with:
           path: |
             ./cpp/build/releases/
-            /root/.m2/repository/org/apache/arrow/
           key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
       - name: Build Gluten native libraries
         if: ${{ steps.cache.outputs.cache-hit != 'true' }}

.github/workflows/velox_docker_cache.yml

@@ -30,7 +30,7 @@ concurrency:
 jobs:
   cache-native-lib-centos-7:
     runs-on: ubuntu-20.04
-    container: apache/gluten:gluten-vcpkg-builder_2024_08_05 # centos7 with dependencies installed
+    container: apache/gluten:vcpkg-centos-7
     steps:
       - uses: actions/checkout@v2
       - name: Generate cache key
@@ -43,7 +43,6 @@ jobs:
           lookup-only: true
           path: |
             ./cpp/build/releases/
-            /root/.m2/repository/org/apache/arrow/
           key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
       - name: Build Gluten native libraries
         if: steps.check-cache.outputs.cache-hit != 'true'
@@ -57,7 +56,6 @@ jobs:
         with:
           path: |
             ./cpp/build/releases/
-            /root/.m2/repository/org/apache/arrow/  
           key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
 
   cache-native-lib-centos-8:

dev/ci-velox-buildstatic-centos-7.sh

@@ -2,12 +2,9 @@
 
 set -e
 
-yum install sudo patch java-1.8.0-openjdk-devel -y
-cd $GITHUB_WORKSPACE/ep/build-velox/src
-./get_velox.sh
 source /opt/rh/devtoolset-9/enable
 cd $GITHUB_WORKSPACE/
 source ./dev/vcpkg/env.sh
-sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt
 export NUM_THREADS=4
-./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON
+./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF --build_arrow=OFF --enable_s3=ON \
+                           --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON

dev/vcpkg/Makefile

@@ -31,7 +31,7 @@ docker-image:
 
 docker-image-gha:
 	docker build \
-		--file docker/Dockerfile.gha \
+		--file docker/gha-centos-7.dockerfile \
 		--tag "$(DOCKER_IMAGE)" \
 		--build-arg HTTPS_PROXY="" \
 		--build-arg HTTP_PROXY="" \

dev/vcpkg/docker/Dockerfile.gha → dev/vcpkg/docker/gha-centos-7.dockerfile

@@ -11,7 +11,7 @@ RUN sed -i \
        -e 's/mirror\.centos\.org/vault.centos.org/' \
        /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
 
-RUN yum install -y git patch wget sudo
+RUN yum install -y git patch wget sudo java-1.8.0-openjdk-devel
 
 # build
 RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten
@@ -22,4 +22,7 @@ RUN cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh
 
 # vcpkg env
 ENV VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite
-RUN source /opt/rh/devtoolset-9/enable && cd /opt/gluten && source dev/vcpkg/env.sh
+
+# Build arrow, then install the native libs to system paths and jar package to .m2/ directory.
+RUN cd /opt/gluten && source ./dev/vcpkg/env.sh && bash ./dev/builddeps-veloxbe.sh build_arrow && \
+    rm -rf ep/_ep/ && rm -rf /tmp/velox-deps/

dev/vcpkg/ports/gflags/portfile.cmake

@@ -22,6 +22,9 @@ vcpkg_configure_cmake(
         -DBUILD_gflags_nothreads_LIB:BOOL=OFF
         -DGFLAGS_USE_TARGET_NAMESPACE:BOOL=ON
         -DCMAKE_DEBUG_POSTFIX=d
+        -DGFLAGS_BUILD_STATIC_LIBS:BOOL=ON
+        -DGFLAGS_BUILD_SHARED_LIBS:BOOL=ON
+        -DGFLAGS_BUILD_gflags_LIB:BOOL=ON
 )
 
 vcpkg_install_cmake()