Migrate circleci worklfows to github actions

.github/actions/fuzzer-run/action.yml

@@ -0,0 +1,42 @@
+name: fuzzer-run
+description: Fuzzer run
+inputs:
+  fuzzer-output:
+    description: Fuzzer output
+    required: true
+  fuzzer-repro:
+    description: Fuzzer repro
+    required: true
+  fuzzer-name:
+    description: Fuzzer name
+    required: true
+  fuzzer-exe:
+    description: Fuzzer command
+    required: true
+  fuzzer-args:
+    description: Fuzzer arguments
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Build
+      shell: bash
+      run: |
+        make debug NUM_THREADS=16 MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=4
+    - name: "Run ${{ inputs.fuzzer-name }} Fuzzer"
+      shell: bash
+      run: |
+        eval ' ${{ inputs.fuzzer-exe }} ${{ inputs.fuzzer-args }} ' \
+              2>&1 | tee "${{ inputs.fuzzer-output }}" || ( \
+                tail -n 1000 "${{ inputs.fuzzer-output }}" ; \
+                echo "FAIL: ${{ inputs.fuzzer-name }} run failed"; \
+                exit 1; \
+              )
+            echo -e "\n ${{ inputs.fuzzer-name }} run finished successfully."
+    - name: "Upload artifacts"
+      uses: actions/[email protected]
+      with:
+        name: ${{ join( inputs.fuzzer-name, '-') }}fuzzer-artifacts
+        path: |
+          ${{ inputs.fuzzer-output }}
+          ${{ inputs.fuzzer-repro }}

.github/actions/macos-build/action.yml

@@ -0,0 +1,49 @@
+name: macos-build
+description: MacOS build
+runs:
+  using: composite
+  steps:
+    - name: Cache dependencies
+      uses: actions/cache@v2
+      with:
+        path: ~/deps
+        key: velox-${{ runner.os }}-deps-v1-${{ hashFiles('.github/workflows/longer-fuzzer.yml') }}-${{ hashFiles('scripts/setup-macos.sh') }}
+    - name: "Calculate merge-base date for CCache"
+      run: git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/main HEAD) | tee merge-base-date
+      shell: bash
+    - name: Ccache cache
+      uses: actions/cache@v2
+      with:
+        path: .ccache
+        key: velox-ccache-${{ runner.os }}-${{ hashFiles('merge-base-date') }}
+    - name: "Install dependencies"
+      run: |
+        set -xu
+        mkdir -p ~/deps ~/deps-src
+        curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C ~/deps
+        PATH=~/deps/bin:${PATH} DEPENDENCY_DIR=~/deps-src INSTALL_PREFIX=~/deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-macos.sh
+        rm -rf ~/deps/.git ~/deps/Library/Taps/  # Reduce cache size by 70%.
+      shell: bash
+    - name: "Build on MacOS"
+      run: |
+        export PATH=~/deps/bin:~/deps/opt/bison/bin:~/deps/opt/flex/bin:${PATH}
+        mkdir -p .ccache
+        export CCACHE_DIR=$(pwd)/.ccache
+        ccache -sz -M 5Gi
+        brew install [email protected]
+        brew link --overwrite --force [email protected]
+        export PATH="/Users/distiller/deps/opt/[email protected]/bin:$PATH"
+        export OPENSSL_ROOT_DIR=$(brew --prefix [email protected])
+        cmake \
+            -B _build/debug \
+            -GNinja \
+            -DTREAT_WARNINGS_AS_ERRORS=1 \
+            -DENABLE_ALL_WARNINGS=1 \
+            -DVELOX_ENABLE_PARQUET=ON \
+            -DCMAKE_BUILD_TYPE=Debug \
+            -DCMAKE_PREFIX_PATH=~/deps \
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+            -DFLEX_INCLUDE_DIR=~/deps/opt/flex/include
+        ninja -C _build/debug
+        ccache -s
+      shell: bash

.github/actions/setup-environment/action.yml

@@ -0,0 +1,31 @@
+name: setup-environment
+description: Setup environment for Velox CI
+runs:
+  using: composite
+  steps:
+    # - name: Install dependencies
+    #   shell: bash
+    #   run: |
+    #     sudo ./scripts/setup-ubuntu.sh
+    - name: Setup environment
+      shell: bash
+      run: |-
+        git config --global --add safe.directory $GITHUB_WORKSPACE
+        # Calculate ccache key.
+        git show -s --format=%cd --date="format:%Y%m%d" $(git merge-base origin/main HEAD) | tee merge-base-date
+
+        # Set up xml gtest output.
+        mkdir -p /tmp/test_xml_output/
+        echo "export XML_OUTPUT_FILE=\"/tmp/test_xml_output/\"" >> $GITHUB_ENV
+
+        # Set up ccache configs.
+        mkdir -p .ccache
+        echo "export CCACHE_DIR=$(realpath .ccache)" >> $GITHUB_ENV
+        ccache -sz -M 5Gi
+        if [ -e /opt/rh/gcc-toolset-9/enable ]; then
+          source /opt/rh/gcc-toolset-9/enable
+        fi
+    - name: ccache
+      uses: hendrikmuhs/[email protected]
+      with:
+        key: velox-ccache-debug-${{ runner.os }}-${{ hashFiles('merge-base-date') }}

.github/workflows/doc-gen-job.yml

@@ -0,0 +1,40 @@
+name: doc-gen-job
+on:
+  workflow_dispatch:
+  pull_request:
+
+jobs:
+  doc-gen-job:
+    runs-on: 8-core
+    container:
+      image : ghcr.io/facebookincubator/velox-dev:circleci-avx
+    env:
+      CC:  /opt/rh/gcc-toolset-9/root/bin/gcc
+      CXX: /opt/rh/gcc-toolset-9/root/bin/g++
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          submodules: recursive
+      - name: Build docs and update gh-pages
+        run: |
+          git config --global user.email "[email protected]"
+          git config --global user.name "velox"
+          git config --global --add safe.directory $GITHUB_WORKSPACE
+          git fetch origin
+          git checkout origin/main
+          conda init bash
+          source ~/.bashrc
+          conda create -y --name docgenenv python=3.7
+          conda activate docgenenv
+          pip install sphinx sphinx-tabs breathe sphinx_rtd_theme chardet
+          source /opt/rh/gcc-toolset-9/enable
+          ./scripts/gen-docs.sh docgenenv
+          git checkout gh-pages
+          cp -R velox/docs/_build/html/* docs
+          git add docs
+          if [ -n "$(git status --porcelain --untracked-files=no)" ]
+          then
+            git commit -m "Update documentation"
+            git push
+          fi

.github/workflows/format-check.yml

@@ -0,0 +1,23 @@
+name: format-check
+on:
+  workflow_dispatch:
+  pull_request:
+
+jobs:
+  format-check:
+    runs-on: ubuntu-latest
+    container:
+      image : ghcr.io/facebookincubator/velox-dev:check-avx
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+      - name: Check formatting
+        run: |
+          if ! make format-check; then
+            make format-fix
+            echo -e "\n==== Apply using:"
+            echo "patch -p1 \<<EOF"
+            git --no-pager diff
+            echo "EOF"
+            false
+          fi

.github/workflows/header-check.yml

@@ -0,0 +1,23 @@
+name: header-check
+on:
+  workflow_dispatch:
+  pull_request:
+
+jobs:
+  header-check:
+    runs-on: ubuntu-latest
+    container:
+      image : ghcr.io/facebookincubator/velox-dev:check-avx
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+      - name: Check license headers
+        run: |
+          if ! make header-check; then
+            make header-fix
+            echo -e "\n==== Apply using:"
+            echo "patch -p1 \<<EOF"
+            git --no-pager diff
+            echo "EOF"
+            false
+          fi

.github/workflows/linux-adapters.yml

@@ -0,0 +1,75 @@
+name: linux-adapters
+on:
+  workflow_dispatch:
+  pull_request:
+
+jobs:
+  linux-adapters:
+    runs-on: 32-core
+    container:
+      image : ghcr.io/facebookincubator/velox-dev:circleci-avx
+    env:
+      CC:  /opt/rh/gcc-toolset-9/root/bin/gcc
+      CXX: /opt/rh/gcc-toolset-9/root/bin/g++
+      VELOX_DEPENDENCY_SOURCE: BUNDLED
+      ICU_SOURCE: BUNDLED
+      simdjson_SOURCE: BUNDLED
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          submodules: recursive
+      - uses: ./.github/actions/setup-environment
+      - name: Install Adapter Dependencies
+        run: |
+          mkdir -p ~/adapter-deps/install/bin
+          set -xu
+          DEPENDENCY_DIR=~/adapter-deps PROMPT_ALWAYS_RESPOND=n ./scripts/setup-adapters.sh
+      - name: Install Minio Server
+        run: |
+          set -xu
+          cd ~/adapter-deps/install/bin/
+          wget https://dl.min.io/server/minio/release/linux-amd64/archive/minio-20220526054841.0.0.x86_64.rpm
+          rpm -i minio-20220526054841.0.0.x86_64.rpm
+          rm minio-20220526054841.0.0.x86_64.rpm
+      - name: Install Hadoop Dependency
+        run: |
+          set -xu
+          yum -y install java-1.8.0-openjdk
+      - name: Build including all Benchmarks
+        run: |
+          EXTRA_CMAKE_FLAGS=(
+            "-DVELOX_ENABLE_BENCHMARKS=ON"
+            "-DVELOX_ENABLE_ARROW=ON"
+            "-DVELOX_ENABLE_PARQUET=ON"
+            "-DVELOX_ENABLE_HDFS=ON"
+            "-DVELOX_ENABLE_S3=ON"
+            "-DVELOX_ENABLE_GCS=ON"
+            "-DVELOX_ENABLE_ABFS=ON"
+            "-DVELOX_ENABLE_SUBSTRAIT=ON"
+            "-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON"
+          )
+          make release EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS[*]}" AWSSDK_ROOT_DIR=~/adapter-deps/install GCSSDK_ROOT_DIR=~/adapter-deps/install NUM_THREADS=16 MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=8
+          ccache -s
+      - name: Run Unit Tests
+        run: |
+          conda init bash
+          conda create -y --name testbench python=3.7
+          conda activate testbench
+          pip install https://github.com/googleapis/storage-testbench/archive/refs/tags/v0.36.0.tar.gz
+          export LC_ALL=C
+          export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk
+          export HADOOP_ROOT_LOGGER="WARN,DRFA"
+          export LIBHDFS3_CONF=$(pwd)/.circleci/hdfs-client.xml
+          export HADOOP_HOME='/usr/local/hadoop'
+          export PATH=~/adapter-deps/install/bin:/usr/local/hadoop/bin:${PATH}
+          # The following is used to install Azurite in the CI for running Abfs Hive Connector unit tests.
+          # Azurite is an emulator for local Azure Storage development, and it is a required component for running Abfs Hive Connector unit tests. 
+          # It can be installed using npm. The following is used to install Node.js and npm for Azurite installation.
+          curl -sL https://rpm.nodesource.com/setup_10.x | bash -
+          sudo apt-get install -y nodejs
+          npm install -g azurite
+          cd _build/release && ctest -j 16 -VV --output-on-failure

.github/workflows/linux-build-options.yml

@@ -0,0 +1,26 @@
+name: linux-build-options
+on:
+    workflow_dispatch:
+    pull_request:
+
+jobs:
+  linux-build-options:
+    runs-on: 32-core
+    container:
+      image : ghcr.io/facebookincubator/velox-dev:circleci-avx
+    env:
+      CC:  /opt/rh/gcc-toolset-9/root/bin/gcc
+      CXX: /opt/rh/gcc-toolset-9/root/bin/g++
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          submodules: recursive
+      - uses: "./.github/actions/setup-environment"
+      - name: Build Velox Minimal
+        run: |
+          make min_debug NUM_THREADS=16 MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=16
+      - name: Build Velox Without Testing
+        run: |
+          make clean
+          make debug EXTRA_CMAKE_FLAGS="-DVELOX_BUILD_TESTING=OFF" NUM_THREADS=16 MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=16