velox_docker_cache.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Velox backend Dependency Cache

on:
  push:
    branches:
      - 'main'

env:
  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true

concurrency:
  group: ${{ github.repository }}-${{ github.workflow }}
  cancel-in-progress: false

jobs:
  cache-native-lib-centos-7:
    runs-on: ubuntu-20.04
    container: apache/gluten:gluten-vcpkg-builder_2024_07_11 # centos7 with dependencies installed
    steps:
      - uses: actions/checkout@v2
      - name: Generate cache key
        run: |
          echo ${{ hashFiles('./ep/build-velox/src/**', './dev/**', './cpp/*', './github/workflows/*') }} > cache-key
      - name: Check existing caches
        id: check-cache
        uses: actions/cache/restore@v3
        with:
          lookup-only: true
          path: |
            ./cpp/build/releases/
            /root/.m2/repository/org/apache/arrow/
          key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
      - name: Build Gluten native libraries
        if: steps.check-cache.outputs.cache-hit != 'true'
        run: |
          df -a
          bash dev/ci-velox-buildstatic-centos-7.sh
      - name: Cache
        if: steps.check-cache.outputs.cache-hit != 'true'
        id: cache
        uses: actions/cache/save@v3
        with:
          path: |
            ./cpp/build/releases/
            /root/.m2/repository/org/apache/arrow/  
          key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}

  cache-native-lib-centos-8:
    runs-on: ubuntu-20.04
    container: ghcr.io/facebookincubator/velox-dev:centos8
    steps:
      - uses: actions/checkout@v2
      - name: Generate cache key
        run: |
          echo ${{ hashFiles('./ep/build-velox/src/**', './dev/**', './cpp/*', './github/workflows/*') }} > cache-key
      - name: Check existing caches
        id: check-cache
        uses: actions/cache/restore@v3
        with:
          lookup-only: true
          path: |
            ./cpp/build/releases/
            ./cpp/build/velox/udf/examples/
            ./cpp/build/velox/benchmarks/
            /root/.m2/repository/org/apache/arrow/
          key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
      - name: Build Gluten native libraries
        if: steps.check-cache.outputs.cache-hit != 'true'
        run: |
          df -a
          bash dev/ci-velox-buildshared-centos-8.sh
      - name: Cache
        if: steps.check-cache.outputs.cache-hit != 'true'
        id: cache
        uses: actions/cache/save@v3
        with:
          path: |
            ./cpp/build/releases/
            ./cpp/build/velox/udf/examples/
            ./cpp/build/velox/benchmarks/
            /root/.m2/repository/org/apache/arrow/
          key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}

  # ccache-native-lib-ubuntu-velox-ut:
  #   runs-on: ubuntu-20.04
  #   env:
  #     CCACHE_DIR: "${{ github.workspace }}/.ccache"
  #   container: ghcr.io/facebookincubator/velox-dev:amd64-ubuntu-22.04-avx
  #   steps:
  #     - uses: actions/checkout@v2
  #     - name: Get Ccache
  #       uses: actions/cache/restore@v3
  #       with:
  #         path: '${{ env.CCACHE_DIR }}'
  #         key: ccache-ubuntu-release-default
  #     - name: Ensure Cache Dirs Exists
  #       working-directory: ${{ github.workspace }}
  #       run: |
  #         mkdir -p '${{ env.CCACHE_DIR }}'
  #     - name: Build Gluten native libraries
  #       run: |
  #         rm -rf /opt/miniconda-for-velox/
  #         cd ep/build-velox/src && \
  #         ./get_velox.sh
  #         cd ../build/velox_ep/
  #         make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON" 

  #     - name: CCache after
  #       run: |
  #         ccache -vs

  #     - uses: actions/cache/save@v3
  #       with:
  #         path: '${{ env.CCACHE_DIR }}'
  #         key: ccache-ubuntu-release-default
#  ccache-native-lib-centos-velox-ut:
#    runs-on: ubuntu-20.04
#    env:
#      CCACHE_DIR: "${{ github.workspace }}/.ccache"
#    container: ghcr.io/facebookincubator/velox-dev:centos8
#    steps:
#      - uses: actions/checkout@v2
#      - name: Setup java and maven
#        run: |
#          yum install sudo patch java-1.8.0-openjdk-devel wget -y && \
#          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
#          tar -xvf apache-maven-3.8.8-bin.tar.gz
#          mv apache-maven-3.8.8 /usr/lib/maven
#      - name: Get Ccache
#        uses: actions/cache/restore@v3
#        with:
#          path: '${{ env.CCACHE_DIR }}'
#          key: ccache-centos-release-default
#      - name: Ensure Cache Dirs Exists
#        working-directory: ${{ github.workspace }}
#        run: |
#          mkdir -p '${{ env.CCACHE_DIR }}'
#      - name: Build Gluten native libraries
#        run: |
#          rm -rf /opt/miniconda-for-velox/
#          cd ep/build-velox/src && \
#          ./get_velox.sh
#          cd ../build/velox_ep/
#          source /opt/rh/gcc-toolset-9/enable
#          make EXTRA_CMAKE_FLAGS="-DVELOX_ENABLE_PARQUET=ON -DVELOX_BUILD_TESTING=ON -DVELOX_BUILD_TEST_UTILS=ON"
#
#      - name: CCache after
#        run: |
#          ccache -s
#
#      - uses: actions/cache/save@v3
#        with:
#          path: '${{ env.CCACHE_DIR }}'
#          key: ccache-centos-release-default