Fall 2024 recompile of the natives (#1416)

This time around, instead of the very ad hoc approach I have historically taken, this time we actually have a convenient script that takes care of basically all the hard work necessary, including cross-compilation.

We compile natives for Ubuntu 20.04 LTS (OpenSSL 1.1.x native and libdeflate) and Ubuntu 22.04 LTS (OpenSSL 3.x.x native), for both x86_64 and aarch64.

The macOS natives have also been recompiled on macOS Sonoma.

native/build-support/build-all-linux-natives.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -e
+
+# make sure we're in the correct directory - the top-level `native` directory
+cd "$(dirname "$0")/.." || exit 1
+
+ARCHS=(x86_64 aarch64)
+BASE_DOCKERFILE_VARIANTS=(ubuntu-focal ubuntu-jammy)
+
+for variant in "${BASE_DOCKERFILE_VARIANTS[@]}"; do
+  docker_platforms=""
+  for arch in "${ARCHS[@]}"; do
+    docker_platforms="$docker_platforms --platform linux/${arch}"
+  done
+
+  echo "Building base build image for $variant..."
+  docker build -t velocity-native-build:$variant $docker_platforms -f build-support/$variant.Dockerfile .
+done
+
+for arch in "${ARCHS[@]}"; do
+  for variant in "${BASE_DOCKERFILE_VARIANTS[@]}"; do
+    echo "Building native crypto for $arch on $variant..."
+
+    docker run --rm -v "$(pwd)":/app --platform linux/${arch} velocity-native-build:$variant /bin/bash -c "cd /app && ./build-support/compile-linux-crypto.sh"
+  done
+
+  # Use only the oldest variant for the compression library
+  variant=${BASE_DOCKERFILE_VARIANTS[0]}
+  echo "Building native compression for $arch on $variant..."
+  docker run --rm -v "$(pwd)":/app --platform linux/${arch} velocity-native-build:$variant /bin/bash -c "cd /app && ./build-support/compile-linux-compress.sh"
+done

native/compile-linux.sh → native/build-support/compile-linux-compress.sh

@@ -8,18 +8,16 @@ fi
 
 if [ ! -d libdeflate ]; then
   echo "Cloning libdeflate..."
-  git clone https://github.com/ebiggers/libdeflate.git
+  git clone --branch v1.21 --single-branch https://github.com/ebiggers/libdeflate.git
 fi
 
 echo "Compiling libdeflate..."
 cd libdeflate || exit
-cmake -B build && cmake --build build --target libdeflate_static
+rm -rf build && cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -B build && cmake --build build --target libdeflate_static
 cd ..
 
 CFLAGS="-O2 -I$JAVA_HOME/include/ -I$JAVA_HOME/include/linux/ -fPIC -shared -Wl,-z,noexecstack -Wall -Werror -fomit-frame-pointer"
 ARCH=$(uname -m)
 mkdir -p src/main/resources/linux_$ARCH
 $CC $CFLAGS -Ilibdeflate src/main/c/jni_util.c src/main/c/jni_zlib_deflate.c src/main/c/jni_zlib_inflate.c \
-    libdeflate/build/libdeflate.a -o src/main/resources/linux_$ARCH/velocity-compress.so
-$CC $CFLAGS -shared src/main/c/jni_util.c src/main/c/jni_cipher_openssl.c \
-    -o src/main/resources/linux_$ARCH/velocity-cipher.so -lcrypto
+    libdeflate/build/libdeflate.a -o src/main/resources/linux_$ARCH/velocity-compress.so

native/build-support/compile-linux-crypto.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+openssl_version=$(openssl version | awk '{print $2}')
+
+# Extract the major and minor version numbers
+major_version=$(echo "$openssl_version" | cut -d. -f1)
+minor_version=$(echo "$openssl_version" | cut -d. -f2)
+
+# Determine the appropriate file name based on the version
+if [ "$major_version" -eq 1 ] && [ "$minor_version" -eq 1 ]; then
+    filename="velocity-cipher-ossl11x.so"
+elif [ "$major_version" -eq 3 ]; then
+    filename="velocity-cipher-ossl30x.so"
+else
+    echo "Unsupported OpenSSL version: $openssl_version"
+    exit 1
+fi
+
+if [ ! "$CC" ]; then
+  export CC=gcc
+fi
+
+output_file="velocity-cipher.so"
+if [ -n "$OPENSSL_VERSION" ]; then
+  output_file="velocity-cipher-ossl${OPENSSL_VERSION}.so"
+fi
+
+CFLAGS="-O2 -I$JAVA_HOME/include/ -I$JAVA_HOME/include/linux/ -fPIC -shared -Wl,-z,noexecstack -Wall -Werror -fomit-frame-pointer"
+ARCH=$(uname -m)
+mkdir -p src/main/resources/linux_$ARCH
+$CC $CFLAGS -shared src/main/c/jni_util.c src/main/c/jni_cipher_openssl.c \
+    -o src/main/resources/linux_$ARCH/$filename -lcrypto

native/compile-macos.sh → native/build-support/compile-macos.sh

@@ -6,7 +6,7 @@ fi
 
 if [ ! -d libdeflate ]; then
   echo "Cloning libdeflate..."
-  git clone https://github.com/ebiggers/libdeflate.git
+  git clone --branch v1.21 --single-branch https://github.com/ebiggers/libdeflate.git
 fi
 
 echo "Compiling libdeflate..."

native/build-support/ubuntu-focal.Dockerfile

@@ -0,0 +1,20 @@
+# Use the official Eclipse Temurin 17.0.12_7-jdk-focal image as the base image.
+# We compile for Ubuntu Focal Fossa (20.04 LTS) as it is still supported until 2025, and the crypto
+# native is specific to a given OpenSSL version.
+FROM eclipse-temurin:17.0.12_7-jdk-focal
+
+# Install required dependencies
+RUN apt-get update && apt-get install -y \
+    libssl-dev \
+    curl \
+    git \
+    unzip \
+    build-essential \
+    cmake \
+    openssl \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+# Create a non-root user
+RUN useradd -m -s /bin/bash -u 1000 -U user
+USER user

native/build-support/ubuntu-jammy.Dockerfile

@@ -0,0 +1,19 @@
+# Use the official Eclipse Temurin 17.0.12_7-jdk-jammy image as the base image.
+# We compile for Ubuntu Jammy Jellyfish (22.04 LTS) as it supports OpenSSL 3.0.
+FROM eclipse-temurin:17.0.12_7-jdk-jammy
+
+# Install required dependencies
+RUN apt-get update && apt-get install -y \
+    libssl-dev \
+    curl \
+    git \
+    unzip \
+    build-essential \
+    cmake \
+    openssl \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+# Create a non-root user
+RUN useradd -m -s /bin/bash -u 1000 -U user
+USER user

native/src/main/java/com/velocitypowered/natives/util/Natives.java

@@ -103,21 +103,21 @@ private static Path createTemporaryNativeFilename(String ext) throws IOException
               copyAndLoadNative("/linux_x86_64/velocity-cipher.so"), // Any local version
               "OpenSSL local (Linux x86_64)", NativeVelocityCipher.FACTORY),
           new NativeCodeLoader.Variant<>(NativeConstraints.LINUX_X86_64,
-              copyAndLoadNative("/linux_x86_64/velocity-cipher-ossl30x.so"), // Debian "Bookworm"
-              "OpenSSL 3.0.x (Linux x86_64)", NativeVelocityCipher.FACTORY),
+              copyAndLoadNative("/linux_x86_64/velocity-cipher-ossl30x.so"), // Ubuntu 22.04
+              "OpenSSL 3.x.x (Linux x86_64)", NativeVelocityCipher.FACTORY),
           new NativeCodeLoader.Variant<>(NativeConstraints.LINUX_X86_64,
-              copyAndLoadNative("/linux_x86_64/velocity-cipher-ossl11x.so"), // Debian 9
+              copyAndLoadNative("/linux_x86_64/velocity-cipher-ossl11x.so"), // Ubuntu 20.04
               "OpenSSL 1.1.x (Linux x86_64)", NativeVelocityCipher.FACTORY),
 
           new NativeCodeLoader.Variant<>(NativeConstraints.LINUX_AARCH64,
               copyAndLoadNative("/linux_aarch64/velocity-cipher.so"),
-              "OpenSSL (Linux aarch64)", NativeVelocityCipher.FACTORY), // Any local version
+              "OpenSSL local (Linux aarch64)", NativeVelocityCipher.FACTORY), // Any local version
           new NativeCodeLoader.Variant<>(NativeConstraints.LINUX_AARCH64,
               copyAndLoadNative("/linux_aarch64/velocity-cipher-ossl30x.so"),
-              "OpenSSL (Linux aarch64)", NativeVelocityCipher.FACTORY), // Fedora 36
+              "OpenSSL 3.x.x (Linux aarch64)", NativeVelocityCipher.FACTORY), // Ubuntu 22.04
           new NativeCodeLoader.Variant<>(NativeConstraints.LINUX_AARCH64,
               copyAndLoadNative("/linux_aarch64/velocity-cipher-ossl11x.so"),
-              "OpenSSL 1.1.x (Linux aarch64)", NativeVelocityCipher.FACTORY), // Debian 11
+              "OpenSSL 1.1.x (Linux aarch64)", NativeVelocityCipher.FACTORY), // Ubuntu 20.04
 
           new NativeCodeLoader.Variant<>(NativeConstraints.MACOS_AARCH64,
               copyAndLoadNative("/macos_arm64/velocity-cipher.dylib"),