From 200bde5ce187c547650b70b149fbc8db2420583b Mon Sep 17 00:00:00 2001 From: nemonlou Date: Thu, 29 Aug 2024 16:03:12 +0800 Subject: [PATCH] [GLUTEN-7023][CH] Shade dependency jars(update pacakge script and getstart doc) --- docs/get-started/ClickHouse.md | 8 ++------ ep/build-clickhouse/src/package.sh | 4 ---- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/docs/get-started/ClickHouse.md b/docs/get-started/ClickHouse.md index 12a662270c029..65e7136d15282 100644 --- a/docs/get-started/ClickHouse.md +++ b/docs/get-started/ClickHouse.md @@ -148,9 +148,7 @@ The prerequisites are the same as the one mentioned above. Compile Gluten with C ``` tar zxf spark-3.2.2-bin-hadoop2.7.tgz cd spark-3.2.2-bin-hadoop2.7 -rm -f jars/protobuf-java-2.5.0.jar -#download protobuf-java-3.23.4.jar, delta-core_2.12-2.0.1.jar and delta-storage-2.0.1.jar -wget https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.23.4/protobuf-java-3.23.4.jar -P ./jars +#download delta-core_2.12-2.0.1.jar and delta-storage-2.0.1.jar wget https://repo1.maven.org/maven2/io/delta/delta-core_2.12/2.0.1/delta-core_2.12-2.0.1.jar -P ./jars wget https://repo1.maven.org/maven2/io/delta/delta-storage/2.0.1/delta-storage-2.0.1.jar -P ./jars cp gluten-XXXXX-spark-3.2-jar-with-dependencies.jar jars/ @@ -161,9 +159,7 @@ cp gluten-XXXXX-spark-3.2-jar-with-dependencies.jar jars/ ``` tar zxf spark-3.3.1-bin-hadoop2.7.tgz cd spark-3.3.1-bin-hadoop2.7 -rm -f jars/protobuf-java-2.5.0.jar -#download protobuf-java-3.23.4.jar, delta-core_2.12-2.2.0.jar and delta-storage-2.2.0.jar -wget https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.23.4/protobuf-java-3.23.4.jar -P ./jars +#download delta-core_2.12-2.2.0.jar and delta-storage-2.2.0.jar wget https://repo1.maven.org/maven2/io/delta/delta-core_2.12/2.2.0/delta-core_2.12-2.2.0.jar -P ./jars wget https://repo1.maven.org/maven2/io/delta/delta-storage/2.2.0/delta-storage-2.2.0.jar -P ./jars cp gluten-XXXXX-spark-3.3-jar-with-dependencies.jar jars/ diff --git a/ep/build-clickhouse/src/package.sh b/ep/build-clickhouse/src/package.sh index db9f38d5bf318..1af0eb18c61f8 100755 --- a/ep/build-clickhouse/src/package.sh +++ b/ep/build-clickhouse/src/package.sh @@ -95,10 +95,6 @@ wget https://repo1.maven.org/maven2/io/delta/delta-core_2.12/${delta_version_33} wget https://repo1.maven.org/maven2/io/delta/delta-storage/${delta_version_33}/delta-storage-${delta_version_33}.jar -P "${PACKAGE_DIR_PATH}"/jars/spark33 # download common 3rd party jars -protobuf_version=$(mvn -q -P${DEFAULT_SPARK_PROFILE} -Dexec.executable="echo" -Dexec.args='${protobuf.version}' --non-recursive exec:exec) -wget https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/${protobuf_version}/protobuf-java-${protobuf_version}.jar -P "${PACKAGE_DIR_PATH}"/jars/spark32 -cp "${PACKAGE_DIR_PATH}"/jars/spark32/protobuf-java-${protobuf_version}.jar "${PACKAGE_DIR_PATH}"/jars/spark33 - celeborn_version=$(mvn -q -P${DEFAULT_SPARK_PROFILE} -Dexec.executable="echo" -Dexec.args='${celeborn.version}' --non-recursive exec:exec) wget https://repo1.maven.org/maven2/org/apache/celeborn/celeborn-client-spark-3-shaded_2.12/${celeborn_version}/celeborn-client-spark-3-shaded_2.12-${celeborn_version}.jar -P "${PACKAGE_DIR_PATH}"/jars/spark32 cp "${PACKAGE_DIR_PATH}"/jars/spark32/celeborn-client-spark-3-shaded_2.12-${celeborn_version}.jar "${PACKAGE_DIR_PATH}"/jars/spark33