From fa275f1c007cf3cce60272367d7c952bc867570c Mon Sep 17 00:00:00 2001
From: Parth Chandra <parthc@apache.org>
Date: Thu, 19 Sep 2024 12:24:08 -0700
Subject: [PATCH] feat: implement scripts for binary release build (#932)

* feat: implement scripts for binary release build

* Install to temp local maven repo and updates for MacOS

* newline

* Use independent docker images for different architectures instead of
a multi-arch image

* update docs and cleanup

* remove unused code

* fail build script on error

* Build all profiles

* remove duplicate target from makefile

---------

Co-authored-by: Andy Grove <agrove@apache.org>
---
 Makefile                                      |  16 ++
 dev/release/README.md                         |  53 +++++
 dev/release/build-release-comet.sh            | 202 ++++++++++++++++++
 dev/release/comet-rm/Dockerfile               |  91 ++++++++
 .../comet-rm/build-comet-native-libs.sh       |  55 +++++
 pom.xml                                       |   5 +
 6 files changed, 422 insertions(+)
 create mode 100755 dev/release/build-release-comet.sh
 create mode 100644 dev/release/comet-rm/Dockerfile
 create mode 100755 dev/release/comet-rm/build-comet-native-libs.sh

diff --git a/Makefile b/Makefile
index d4e2887f7..80f334a34 100644
--- a/Makefile
+++ b/Makefile
@@ -46,6 +46,22 @@ format:
 	./mvnw compile test-compile scalafix:scalafix -Psemanticdb $(PROFILES)
 	./mvnw spotless:apply $(PROFILES)
 
+# build native libs for amd64 architecture Linux/MacOS on a Linux/amd64 machine/container
+core-amd64-libs:
+	cd native && cargo build -j 2 --release
+ifdef HAS_OSXCROSS
+	rustup target add x86_64-apple-darwin
+	cd native && cargo build -j 2 --target x86_64-apple-darwin --release
+endif
+
+# build native libs for arm64 architecture Linux/MacOS on a Linux/arm64 machine/container
+core-arm64-libs:
+	cd native && cargo build -j 2 --release
+ifdef HAS_OSXCROSS
+	rustup target add aarch64-apple-darwin
+	cd native && cargo build -j 2 --target aarch64-apple-darwin --release
+endif
+
 core-amd64:
 	rustup target add x86_64-apple-darwin
 	cd native && RUSTFLAGS="-Ctarget-cpu=skylake -Ctarget-feature=-prefer-256-bit" CC=o64-clang CXX=o64-clang++ cargo build --target x86_64-apple-darwin --release
diff --git a/dev/release/README.md b/dev/release/README.md
index 62291b51b..d11345294 100644
--- a/dev/release/README.md
+++ b/dev/release/README.md
@@ -81,6 +81,50 @@ python3 generate-changelog.py 0.0.0 HEAD 0.1.0 > ../changelog/0.1.0.md
 Create a PR against the _main_ branch to add this change log and once this is approved and merged, cherry-pick the
 commit into the release branch.
 
+### Build the jars 
+
+#### Setup to do the build
+  The build process requires Docker. Download the latest Docker Desktop from https://www.docker.com/products/docker-desktop/.
+  If you have multiple docker contexts running switch to the context of the Docker Desktop. For example - 
+
+  ```shell
+$ docker context ls
+NAME              DESCRIPTION                               DOCKER ENDPOINT                               ERROR
+default           Current DOCKER_HOST based configuration   unix:///var/run/docker.sock
+desktop-linux     Docker Desktop                            unix:///Users/parth/.docker/run/docker.sock
+my_custom_context *                                         tcp://192.168.64.2:2376
+
+$ docker context use desktop-linux
+  ```
+#### Run the build script
+ The `build-release-comet.sh` script will create a docker image for each architecture and use the image 
+to build the platform specific binaries. These builder images are created every time this script is run.
+The script optionally allows overriding of the repository and branch to build the binaries from (Note that 
+ the local git repo is not used in the building of the binaries, but it is used to build the final uber jar).
+
+```shell
+Usage: build-release-comet.sh [options]
+
+This script builds comet native binaries inside a docker image. The image is named
+"comet-rm" and will be generated by this script
+
+Options are:
+
+-r [repo]   : git repo (default: https://github.com/apache/datafusion-comet.git)
+-b [branch] : git branch (default: release)
+-t [tag]    : tag for the spark-rm docker image to use for building (default: "latest").
+```
+
+Example:
+
+```shell
+cd dev/release && ./build-release-comet.sh && cd ../..
+```
+
+#### Build output
+ The build output is installed to a temporary local maven repository. The build script will print the name of the repository
+location at the end. This location will be required at the time of deploying the artifacts to a staging repository
+
 ### Tag the Release Candidate
 
 Tag the release branch with `0.1.0-rc1` and push to the `apache` repo
@@ -105,6 +149,15 @@ Run the create-tarball script on the release candidate tag (`0.1.0-rc1`) to crea
 GH_TOKEN=<TOKEN> ./dev/release/create-tarball.sh 0.1.0 1
 ```
 
+### Publish the maven artifacts
+#### Setup maven
+##### One time project setup
+Setting up your project in the ASF Nexus Repository from here: https://infra.apache.org/publishing-maven-artifacts.html
+##### Release Manager Setup
+Set up your development environment from here:  https://infra.apache.org/publishing-maven-artifacts.html
+
+TODO: build and publish a release candidate to nexus.
+
 ### Start an Email Voting Thread
 
 Send the email that is generated in the previous step to `dev@datafusion.apache.org`.
diff --git a/dev/release/build-release-comet.sh b/dev/release/build-release-comet.sh
new file mode 100755
index 000000000..07466bf16
--- /dev/null
+++ b/dev/release/build-release-comet.sh
@@ -0,0 +1,202 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+COMET_HOME_DIR=$SCRIPT_DIR/../..
+
+function usage {
+  local NAME=$(basename $0)
+  cat <<EOF
+Usage: $NAME [options]
+
+This script builds comet native binaries inside a docker image. The image is named
+"comet-rm" and will be generated by this script
+
+Options are:
+
+  -r [repo]   : git repo (default: ${REPO})
+  -b [branch] : git branch (default: ${BRANCH})
+  -t [tag]    : tag for the spark-rm docker image to use for building (default: "latest").
+EOF
+exit 1
+}
+
+function cleanup()
+{
+  if [ $CLEANUP != 0 ]
+  then
+    echo Cleaning up ...
+    if [ "$(docker ps -a | grep comet-arm64-builder-container)" != "" ]
+    then
+      docker rm comet-arm64-builder-container
+    fi
+    if [ "$(docker ps -a | grep comet-amd64-builder-container)" != "" ]
+    then
+      docker rm comet-amd64-builder-container
+    fi
+    CLEANUP=0
+  fi
+}
+
+trap cleanup SIGINT SIGTERM EXIT
+
+CLEANUP=1
+
+REPO="https://github.com/apache/datafusion-comet.git"
+BRANCH="release"
+MACOS_SDK=
+HAS_MACOS_SDK="false"
+IMGTAG=latest
+
+while getopts "b:hr:t:" opt; do
+  case $opt in
+    r) REPO="$OPTARG";;
+    b) BRANCH="$OPTARG";;
+    t) IMGTAG="$OPTARG" ;;
+    h) usage ;;
+    \?) error "Invalid option. Run with -h for help." ;;
+  esac
+done
+
+echo "Building binaries from $REPO/$BRANCH"
+
+WORKING_DIR="$SCRIPT_DIR/comet-rm/workdir"
+cp $SCRIPT_DIR/../cargo.config $WORKING_DIR
+
+# TODO: Search for Xcode (Once building macos binaries works)
+#PS3="Select Xcode:"
+#select xcode_path in  `find . -name "${MACOS_SDK}"`
+#do
+#  echo "found Xcode in $xcode_path"
+#  cp $xcode_path $WORKING_DIR
+#  break
+#done
+
+if [ -f "${WORKING_DIR}/${MACOS_SDK}" ]
+then
+  HAS_MACOS_SDK="true"
+fi
+
+BUILDER_IMAGE_ARM64="comet-rm-arm64:$IMGTAG"
+BUILDER_IMAGE_AMD64="comet-rm-amd64:$IMGTAG"
+
+# Build the docker image in which we will do the build
+docker build \
+  --platform=linux/arm64 \
+  -t "$BUILDER_IMAGE_ARM64" \
+  --build-arg HAS_MACOS_SDK=${HAS_MACOS_SDK} \
+  --build-arg MACOS_SDK=${MACOS_SDK} \
+  "$SCRIPT_DIR/comet-rm"
+
+docker build \
+  --platform=linux/amd64 \
+  -t "$BUILDER_IMAGE_AMD64" \
+  --build-arg HAS_MACOS_SDK=${HAS_MACOS_SDK} \
+  --build-arg MACOS_SDK=${MACOS_SDK} \
+  "$SCRIPT_DIR/comet-rm"
+
+# Clean previous Java build
+pushd $COMET_HOME_DIR && ./mvnw clean && popd
+
+# Run the builder container for each architecture. The entrypoint script will build the binaries
+
+# AMD64
+echo "Building amd64 binary"
+docker run \
+   --name comet-amd64-builder-container \
+   --memory 24g \
+   --cpus 6 \
+   -it \
+   --platform linux/amd64 \
+   $BUILDER_IMAGE_AMD64 "${REPO}" "${BRANCH}" amd64
+
+if [ $? != 0 ]
+then
+  echo "Building amd64 binary failed."
+  exit 1
+fi
+
+# ARM64
+echo "Building arm64 binary"
+docker run \
+   --name comet-arm64-builder-container \
+   --memory 24g \
+   --cpus 6 \
+   -it \
+   --platform linux/arm64 \
+   $BUILDER_IMAGE_ARM64 "${REPO}" "${BRANCH}" arm64
+
+if [ $? != 0 ]
+then
+  echo "Building arm64 binary failed."
+  exit 1
+fi
+
+echo "Building binaries completed"
+echo "Copying to java build directories"
+
+JVM_TARGET_DIR=$COMET_HOME_DIR/common/target/classes/org/apache/comet
+mkdir -p $JVM_TARGET_DIR
+
+mkdir -p $JVM_TARGET_DIR/linux/amd64
+docker cp \
+  comet-amd64-builder-container:"/opt/comet-rm/comet/native/target/release/libcomet.so" \
+  $JVM_TARGET_DIR/linux/amd64/
+
+if [ "$HAS_MACOS_SDK" == "true" ]
+then
+  mkdir -p $JVM_TARGET_DIR/darwin/x86_64
+  docker cp \
+    comet-amd64-builder-container:"/opt/comet-rm/comet/native/target/x86_64-apple-darwin/release/libcomet.dylib" \
+    $JVM_TARGET_DIR/darwin/x86_64/
+fi
+
+mkdir -p $JVM_TARGET_DIR/linux/aarch64
+docker cp \
+  comet-arm64-builder-container:"/opt/comet-rm/comet/native/target/release/libcomet.so" \
+  $JVM_TARGET_DIR/linux/aarch64/
+
+if [ "$HAS_MACOS_SDK" == "true" ]
+then
+  mkdir -p $JVM_TARGET_DIR/linux/aarch64
+  docker cp \
+    comet-arm64-builder-container:"/opt/comet-rm/comet/native/target/aarch64-apple-darwin/release/libcomet.dylib" \
+    $JVM_TARGET_DIR/darwin/aarch64/
+fi
+
+# Build final jar
+echo "Building uber jar and publishing it locally"
+pushd $COMET_HOME_DIR
+
+GIT_HASH=$(git rev-parse --short HEAD)
+LOCAL_REPO=$(mktemp -d /tmp/comet-staging-repo-XXXXX)
+
+./mvnw  "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-3.4 -P scala-2.12  -DskipTests install
+./mvnw  "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-3.4 -P scala-2.13  -DskipTests install
+./mvnw  "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-3.3 -P scala-2.12  -DskipTests install
+./mvnw  "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-3.3 -P scala-2.13  -DskipTests install
+./mvnw  "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-3.5 -P scala-2.12  -DskipTests install
+./mvnw  "-Dmaven.repo.local=${LOCAL_REPO}" -P spark-3.5 -P scala-2.13  -DskipTests install
+
+echo "Installed to local repo: ${LOCAL_REPO}"
+
+popd
diff --git a/dev/release/comet-rm/Dockerfile b/dev/release/comet-rm/Dockerfile
new file mode 100644
index 000000000..9b430d2a9
--- /dev/null
+++ b/dev/release/comet-rm/Dockerfile
@@ -0,0 +1,91 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+ARG HAS_MACOS_SDK="false"
+
+FROM ubuntu:20.04 AS base
+
+USER root
+
+# For apt to be noninteractive
+ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBCONF_NONINTERACTIVE_SEEN=true
+
+ENV LC_ALL=C
+# Install pr-requisites for rust
+RUN export LC_ALL=C \
+    && apt-get update \
+    && apt-get install --no-install-recommends -y \
+        ca-certificates \
+        build-essential \
+        curl \
+        wget \
+        git \
+        llvm \
+        clang \
+        libssl-dev \
+        lzma-dev \
+        liblzma-dev \
+        openssh-client \
+        cmake \
+        cpio \
+        libxml2-dev \
+        patch \
+        bzip2 \
+        libbz2-dev \
+        zlib1g-dev
+
+
+# Install rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+RUN cargo install cargo2junit
+
+# Stage to add OSXCross if MacOSSDK is provided
+FROM base AS with-macos-sdk-true
+ARG MACOS_SDK
+
+COPY workdir/$MACOS_SDK /opt/xcode/
+
+RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
+    rustup target add aarch64-apple-darwin; \
+elif [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
+    rustup target add x86_64-apple-darwin; \
+fi
+
+# Build OSXCross
+RUN cd /opt && git clone --depth 1 https://github.com/tpoechtrager/osxcross.git \
+    && cd /opt/osxcross \
+    && ./tools/gen_sdk_package_pbzx.sh /opt/xcode/${MACOS_SDK} \
+    && cd .. \
+    && cp /opt/osxcross/*.tar.xz tarballs \
+    && UNATTENDED=1 ./build.sh
+ENV PATH="/opt/osxcross/target/bin:${PATH}"
+# Use osxcross toolchain for cargo
+COPY workdir/cargo.config /root/.cargo/config
+ENV HAS_OSXCROSS="true"
+
+# Placeholder Stage if MacOSSDK is not provided
+FROM base AS with-macos-sdk-false
+RUN echo "Building without MacOS"
+
+
+FROM with-macos-sdk-${HAS_MACOS_SDK} AS final
+
+COPY build-comet-native-libs.sh /opt/comet-rm/build-comet-native-libs.sh
+WORKDIR /opt/comet-rm
+
+ENTRYPOINT [ "/opt/comet-rm/build-comet-native-libs.sh"]
\ No newline at end of file
diff --git a/dev/release/comet-rm/build-comet-native-libs.sh b/dev/release/comet-rm/build-comet-native-libs.sh
new file mode 100755
index 000000000..78f1ae12d
--- /dev/null
+++ b/dev/release/comet-rm/build-comet-native-libs.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# builds a comet binary
+
+set -e
+
+REPO=$1
+BRANCH=$2
+ARCH=$3
+
+function usage {
+  local NAME=$(basename $0)
+  echo "Usage: ${NAME} [git repo] [branch] [arm64 | amd64]"
+  exit 1
+}
+
+if [ $# -ne 3 ]
+then
+  usage
+fi
+
+if [ "$ARCH" != "arm64" ] && [ "$ARCH" != "amd64" ]
+then
+  usage
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+
+echo "Building architecture: ${ARCH} for ${REPO}/${BRANCH}"
+rm -fr comet
+git clone "$REPO" comet
+cd comet
+git checkout "$BRANCH"
+
+# build comet binaries
+
+make core-${ARCH}-libs
diff --git a/pom.xml b/pom.xml
index 4f3f39d14..2893b705d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -23,6 +23,11 @@ under the License.
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache</groupId>
+    <artifactId>apache</artifactId>
+    <version>23</version>
+  </parent>
   <groupId>org.apache.datafusion</groupId>
   <artifactId>comet-parent-spark${spark.version.short}_${scala.binary.version}</artifactId>
   <version>0.3.0-SNAPSHOT</version>