[GLUTEN-4917][VL] Update dependencies in static packaging (apache#5339)

zhanglistar · Apr 10, 2024 · cf5c60a · cf5c60a
1 parent b274a90
commit cf5c60a
Show file tree

Hide file tree

Showing 7 changed files with 79 additions and 4 deletions.
diff --git a/dev/package-vcpkg.sh b/dev/package-vcpkg.sh
@@ -8,3 +8,7 @@ GLUTEN_DIR="$CURRENT_DIR/.."
 cd "$GLUTEN_DIR"
 source ./dev/vcpkg/env.sh
 ./dev/buildbundle-veloxbe.sh --build_tests=ON --build_benchmarks=ON --enable_s3=ON  --enable_hdfs=ON
+mvn clean package -Pbackends-velox -Prss -Pspark-3.2 -DskipTests
+mvn clean package -Pbackends-velox -Prss -Pspark-3.3 -DskipTests
+mvn clean package -Pbackends-velox -Prss -Pspark-3.4 -DskipTests
+mvn clean package -Pbackends-velox -Prss -Pspark-3.5 -DskipTests
diff --git a/dev/vcpkg/Makefile b/dev/vcpkg/Makefile
@@ -27,5 +27,12 @@ docker-image:
 		--tag "$(DOCKER_IMAGE)" \
 		.
 
+docker-image-gha:
+	docker build \
+		--file docker/Dockerfile.gha \
+		--tag "$(DOCKER_IMAGE)" \
+		.
+
+
 $(CCACHE_DIR) $(VCPKG_BINARY_CACHE_DIR) $(MAVEN_M2_DIR): %:
-	mkdir -p $@
+	mkdir -p $@
diff --git a/dev/vcpkg/docker/Dockerfile.gha b/dev/vcpkg/docker/Dockerfile.gha
@@ -0,0 +1,14 @@
+FROM centos:7
+
+RUN yum install -y git patch wget sudo
+
+# build
+RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten
+
+RUN echo "check_certificate = off" >> ~/.wgetrc
+# deps
+RUN cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh
+
+# vcpkg env
+ENV VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite
+RUN source /opt/rh/devtoolset-9/enable && cd /opt/gluten && source dev/vcpkg/env.sh
diff --git a/dev/vcpkg/ports/duckdb/portfile.cmake b/dev/vcpkg/ports/duckdb/portfile.cmake
@@ -0,0 +1,25 @@
+vcpkg_download_distfile(ARCHIVE
+    URLS "https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz"
+    FILENAME "v0.8.1.tar.gz"
+    SHA512 06e480aeefdb73bdc1b3d977d5b6f56b40d8893ac4de8702ca54e47ee14a0fa2532e3f9b1aaa49000d6b0b82b93e46e9e2bbaa7a1a6761542f91443a903ddb5d
+)
+
+vcpkg_extract_source_archive_ex(
+    OUT_SOURCE_PATH SOURCE_PATH
+    ARCHIVE "${ARCHIVE}"
+)
+
+vcpkg_configure_cmake(
+    SOURCE_PATH "${SOURCE_PATH}"
+    PREFER_NINJA
+    OPTIONS
+        -DBUILD_UNITTESTS=OFF
+        -DENABLE_SANITIZER=OFF
+        -DENABLE_UBSAN=OFF
+        -DBUILD_SHELL=OFF
+        -DEXPORT_DLL_SYMBOLS=OFF
+        -DCMAKE_BUILD_TYPE=Release
+)
+vcpkg_install_cmake(ADD_BIN_TO_PATH)
+
+vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE")
diff --git a/dev/vcpkg/ports/duckdb/vcpkg.json b/dev/vcpkg/ports/duckdb/vcpkg.json
@@ -0,0 +1,7 @@
+{
+  "name": "duckdb",
+  "version": "0.8.1",
+  "dependencies": [
+
+  ]
+}
diff --git a/dev/vcpkg/vcpkg-configuration.json b/dev/vcpkg/vcpkg-configuration.json
@@ -1,5 +1,13 @@
 {
   "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg-configuration.schema.json",
   "overlay-ports": [ "./ports" ],
-  "overlay-triplets": [ "./triplets" ]
+  "overlay-triplets": [ "./triplets" ],
+  "registries": [
+    {
+      "kind": "git",
+      "repository": "https://github.com/Microsoft/vcpkg",
+      "baseline": "8ccb84df727bdf83045e53c319af05c554838b80",
+      "packages": [ "boost*", "boost-*"]
+    }
+  ]
 }
diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json
@@ -2,7 +2,7 @@
   "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json",
   "builtin-baseline": "a7b6122f6b6504d16d96117336a0562693579933",
   "dependencies": ["jemalloc"],
-  "default-features": ["velox", "velox-s3", "velox-gcs", "velox-hdfs", "velox-abfs"],
+  "default-features": ["velox", "velox-s3", "velox-gcs", "velox-hdfs", "velox-abfs", "duckdb"],
   "features": {
     "velox": {
       "description": "Velox backend",
@@ -43,8 +43,10 @@
         "boost-thread",
         "boost-multiprecision",
         "boost-uuid",
+        "boost-variant",
         "gtest",
         "xsimd",
+        "simdjson",
         "xxhash",
         "protobuf",
         "benchmark",
@@ -92,6 +94,12 @@
         "azure-storage-files-datalake-cpp"
       ]
     },
+    "duckdb": {
+      "description": "DUCKDB",
+      "dependencies": [
+        "duckdb"
+      ]
+    },
     "hbm": {
       "description": "HBM",
       "dependencies": [
@@ -109,6 +117,8 @@
     }
   },
   "overrides": [
-    { "name": "fmt", "version": "10.1.1" }
+    { "name": "fmt", "version": "10.1.1" },
+    { "name": "xsimd", "version": "10.0.0" },
+    { "name": "simdjson", "version": "3.2.0" }
   ]
 }