Skip to content

Commit

Permalink
[GLUTEN-5112] Support build bundle package through github action (#5056)
Browse files Browse the repository at this point in the history
This PR adds a new workflow to support build bundle package through github action
(Fixes: #5112)
  • Loading branch information
wangyum authored Mar 26, 2024
1 parent a3f86ed commit 0d03c32
Showing 1 changed file with 87 additions and 0 deletions.
87 changes: 87 additions & 0 deletions .github/workflows/build_bundle_package.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Build bundle package

concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true

on:
workflow_dispatch:
inputs:
os:
description: 'OS version: ubuntu:20.04, ubuntu:22.04, centos:7 or centos:8'
required: true
default: 'ubuntu:20.04'
spark:
description: 'Spark version: spark-3.2, spark-3.3, spark-3.4 or spark-3.5'
required: true
default: 'spark-3.5'
hadoop:
description: 'Hadoop version: 2.7.4, 3.2.0, 3.2.2, 3.3.1, 3.3.3 or 3.3.6'
required: true
default: '3.3.3'

jobs:
build-native-lib:
runs-on: ubuntu-20.04
container: inteldpo/gluten-centos-packaging:latest
steps:
- uses: actions/checkout@v3
- name: Build Gluten velox third party
run: |
yum install sudo patch java-1.8.0-openjdk-devel -y && \
cd $GITHUB_WORKSPACE/ep/build-velox/src && \
./get_velox.sh && \
source /opt/rh/devtoolset-9/enable && \
source /opt/gluten//dev/vcpkg/env.sh && \
cd $GITHUB_WORKSPACE/ && \
sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt && \
export NUM_THREADS=4
./dev/builddeps-veloxbe.sh --build_tests=OFF --build_benchmarks=OFF --enable_s3=OFF \
--enable_gcs=OFF --enable_hdfs=ON --enable_abfs=OFF
- uses: actions/upload-artifact@v2
with:
path: ./cpp/build/releases/
name: velox-native-lib-${{github.sha}}
retention-days: 1

build-pabkages:
needs: build-native-lib
runs-on: ubuntu-20.04
container: ${{ github.event.inputs.os }}
steps:
- uses: actions/checkout@v3
- name: Download All Artifacts
uses: actions/download-artifact@v2
with:
name: velox-native-lib-${{github.sha}}
path: ./cpp/build/releases
- name: Setup java and maven
run: |
apt-get update && \
apt-get install -y openjdk-8-jdk maven && \
apt remove openjdk-11* -y
- name: Build for Spark ${{ github.event.inputs.spark }}
run: |
cd $GITHUB_WORKSPACE/ && \
mvn clean install -P${{ github.event.inputs.spark }} -Dhadoop.version=${{ github.event.inputs.hadoop }} -Pbackends-velox -Prss -DskipTests -Dmaven.source.skip
- name: Upload bundle package
uses: actions/upload-artifact@v4
with:
name: gluten-velox-bundle-package
path: package/target/gluten-velox-bundle-*.jar
retention-days: 7

0 comments on commit 0d03c32

Please sign in to comment.