From f6e256a0a916d9242dfbee93e60b4693fc1629c2 Mon Sep 17 00:00:00 2001 From: Zhen Wang <643348094@qq.com> Date: Thu, 5 Sep 2024 13:53:28 +0800 Subject: [PATCH] [GLUTEN-6571][VL] Add platform and arch subdirectory for base lib package (#6942) Closes #6571 --- backends-velox/pom.xml | 1 + .../backendsapi/velox/VeloxListenerApi.scala | 16 +++- .../utils/SharedLibraryLoaderCentos7.scala | 44 +++++----- .../utils/SharedLibraryLoaderCentos8.scala | 52 +++++------ .../utils/SharedLibraryLoaderCentos9.scala | 52 +++++------ .../utils/SharedLibraryLoaderDebian11.scala | 55 ++++++------ .../utils/SharedLibraryLoaderDebian12.scala | 67 +++++++------- .../utils/SharedLibraryLoaderUbuntu2004.scala | 87 ++++++++++--------- .../utils/SharedLibraryLoaderUbuntu2204.scala | 57 ++++++------ dev/buildbundle-veloxbe.sh | 13 +++ .../gluten/vectorized/JniLibLoader.java | 38 ++++---- pom.xml | 54 ++++++++++++ 12 files changed, 317 insertions(+), 219 deletions(-) diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml index 3acf27c316ca..c678168252b6 100755 --- a/backends-velox/pom.xml +++ b/backends-velox/pom.xml @@ -187,6 +187,7 @@ ${cpp.releases.dir} + org/apache/gluten/${platform}/${arch} ${resource.dir} diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala index bb5e7deec58a..d2c4293906e4 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala @@ -138,8 +138,8 @@ class VeloxListenerApi extends ListenerApi with Logging { JniLibLoader.loadFromPath(libPath, false) } else { val baseLibName = conf.get(GlutenConfig.GLUTEN_LIB_NAME, "gluten") - loader.mapAndLoad(baseLibName, false) - loader.mapAndLoad(VeloxBackend.BACKEND_NAME, false) + loader.load(s"$baseLibPackage/${System.mapLibraryName(baseLibName)}", false) + loader.load(s"$baseLibPackage/${System.mapLibraryName(VeloxBackend.BACKEND_NAME)}", false) } // Initial native backend with configurations. @@ -152,6 +152,18 @@ class VeloxListenerApi extends ListenerApi with Logging { GlutenRowSplitter.setInstance(new VeloxRowSplitter()) } + private lazy val baseLibPackage: String = { + val osName = System.getProperty("os.name") match { + case n if n.contains("Linux") => "linux" + case n if n.contains("Mac") => "darwin" + case _ => + // Default to linux + "linux" + } + val arch = System.getProperty("os.arch") + s"org/apache/gluten/$osName/$arch" + } + private def shutdown(): Unit = { // TODO shutdown implementation in velox to release resources } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala index a7750dcb3e65..384d00fe6d55 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala @@ -20,26 +20,28 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderCentos7 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) - loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) - loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) - loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - loader.loadAndCreateLink( - "libboost_program_options.so.1.84.0", - "libboost_program_options.so", - false) - loader.loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) - loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - loader.loadAndCreateLink("libdouble-conversion.so.1", "libdouble-conversion.so", false) - loader.loadAndCreateLink("libevent-2.0.so.5", "libevent-2.0.so", false) - loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - loader.loadAndCreateLink("libglog.so.0", "libglog.so", false) - loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false) - loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) - loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - loader.loadAndCreateLink("libre2.so.10", "libre2.so", false) - loader.loadAndCreateLink("libzstd.so.1", "libzstd.so", false) - loader.loadAndCreateLink("liblz4.so.1", "liblz4.so", false) + def loadAndCreateLink(libName: String, linkName: String): Unit = { + val mapLibName = System.mapLibraryName(libName) + loader.loadAndCreateLink(mapLibName, linkName, false) + } + + loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so") + loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so") + loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so") + loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so") + loadAndCreateLink("libboost_program_options.so.1.84.0", "libboost_program_options.so") + loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so") + loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so") + loadAndCreateLink("libdouble-conversion.so.1", "libdouble-conversion.so") + loadAndCreateLink("libevent-2.0.so.5", "libevent-2.0.so") + loadAndCreateLink("libgflags.so.2.2", "libgflags.so") + loadAndCreateLink("libglog.so.0", "libglog.so") + loadAndCreateLink("libntlm.so.0", "libntlm.so") + loadAndCreateLink("libgsasl.so.7", "libgsasl.so") + loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so") + loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so") + loadAndCreateLink("libre2.so.10", "libre2.so") + loadAndCreateLink("libzstd.so.1", "libzstd.so") + loadAndCreateLink("liblz4.so.1", "liblz4.so") } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala index bd8bf15bec9f..0167ed4e2f51 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala @@ -20,30 +20,32 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderCentos8 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) - loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) - loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) - loader.loadAndCreateLink("libicudata.so.60", "libicudata.so", false) - loader.loadAndCreateLink("libicuuc.so.60", "libicuuc.so", false) - loader.loadAndCreateLink("libicui18n.so.60", "libicui18n.so", false) - loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - loader.loadAndCreateLink( - "libboost_program_options.so.1.84.0", - "libboost_program_options.so", - false) - loader.loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) - loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - loader.loadAndCreateLink("libevent-2.1.so.6", "libevent-2.1.so", false) - loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - loader.loadAndCreateLink("libglog.so.1", "libglog.so", false) - loader.loadAndCreateLink("libdwarf.so.1", "libdwarf.so", false) - loader.loadAndCreateLink("libidn.so.11", "libidn.so", false) - loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false) - loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) - loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - loader.loadAndCreateLink("libre2.so.0", "libre2.so", false) - loader.loadAndCreateLink("libsodium.so.23", "libsodium.so", false) + def loadAndCreateLink(libName: String, linkName: String): Unit = { + val mapLibName = System.mapLibraryName(libName) + loader.loadAndCreateLink(mapLibName, linkName, false) + } + + loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so") + loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so") + loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so") + loadAndCreateLink("libicudata.so.60", "libicudata.so") + loadAndCreateLink("libicuuc.so.60", "libicuuc.so") + loadAndCreateLink("libicui18n.so.60", "libicui18n.so") + loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so") + loadAndCreateLink("libboost_program_options.so.1.84.0", "libboost_program_options.so") + loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so") + loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so") + loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so") + loadAndCreateLink("libevent-2.1.so.6", "libevent-2.1.so") + loadAndCreateLink("libgflags.so.2.2", "libgflags.so") + loadAndCreateLink("libglog.so.1", "libglog.so") + loadAndCreateLink("libdwarf.so.1", "libdwarf.so") + loadAndCreateLink("libidn.so.11", "libidn.so") + loadAndCreateLink("libntlm.so.0", "libntlm.so") + loadAndCreateLink("libgsasl.so.7", "libgsasl.so") + loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so") + loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so") + loadAndCreateLink("libre2.so.0", "libre2.so") + loadAndCreateLink("libsodium.so.23", "libsodium.so") } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala index 06fb25c8b0e4..92fd1f5c600c 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala @@ -20,30 +20,32 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderCentos9 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) - loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) - loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) - loader.loadAndCreateLink("libicudata.so.67", "libicudata.so", false) - loader.loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false) - loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false) - loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - loader.loadAndCreateLink( - "libboost_program_options.so.1.84.0", - "libboost_program_options.so", - false) - loader.loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) - loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) - loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - loader.loadAndCreateLink("libglog.so.1", "libglog.so", false) - loader.loadAndCreateLink("libdwarf.so.0", "libdwarf.so", false) - loader.loadAndCreateLink("libidn.so.12", "libidn.so", false) - loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false) - loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) - loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - loader.loadAndCreateLink("libre2.so.9", "libre2.so", false) - loader.loadAndCreateLink("libsodium.so.23", "libsodium.so", false) + def loadAndCreateLink(libName: String, linkName: String): Unit = { + val mapLibName = System.mapLibraryName(libName) + loader.loadAndCreateLink(mapLibName, linkName, false) + } + + loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so") + loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so") + loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so") + loadAndCreateLink("libicudata.so.67", "libicudata.so") + loadAndCreateLink("libicuuc.so.67", "libicuuc.so") + loadAndCreateLink("libicui18n.so.67", "libicui18n.so") + loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so") + loadAndCreateLink("libboost_program_options.so.1.84.0", "libboost_program_options.so") + loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so") + loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so") + loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so") + loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so") + loadAndCreateLink("libgflags.so.2.2", "libgflags.so") + loadAndCreateLink("libglog.so.1", "libglog.so") + loadAndCreateLink("libdwarf.so.0", "libdwarf.so") + loadAndCreateLink("libidn.so.12", "libidn.so") + loadAndCreateLink("libntlm.so.0", "libntlm.so") + loadAndCreateLink("libgsasl.so.7", "libgsasl.so") + loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so") + loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so") + loadAndCreateLink("libre2.so.9", "libre2.so") + loadAndCreateLink("libsodium.so.23", "libsodium.so") } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala index a300cc5b9d05..3aebb725bf39 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala @@ -20,30 +20,35 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderDebian11 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader.loadAndCreateLink("libicudata.so.67", "libicudata.so", false) - loader.loadAndCreateLink("libre2.so.9", "libre2.so", false) - loader.loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false) - loader.loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false) - loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) - loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) - loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false) - loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false) - loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) - loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false) - loader.loadAndCreateLink("libssh2.so.1", "libssh2.so", false) - loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false) - loader.loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false) - loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) - loader.loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false) - loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - loader.loadAndCreateLink("libglog.so.0", "libglog.so", false) - loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) - loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) - loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false) - loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + def loadAndCreateLink(libName: String, linkName: String): Unit = { + val mapLibName = System.mapLibraryName(libName) + loader.loadAndCreateLink(mapLibName, linkName, false) + } + + loadAndCreateLink("libicudata.so.67", "libicudata.so") + loadAndCreateLink("libre2.so.9", "libre2.so") + loadAndCreateLink("libicuuc.so.67", "libicuuc.so") + loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so") + loadAndCreateLink("libsasl2.so.2", "libsasl2.so") + loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so") + loadAndCreateLink("libicui18n.so.67", "libicui18n.so") + loadAndCreateLink("libunwind.so.8", "libunwind.so") + loadAndCreateLink("libgflags.so.2.2", "libgflags.so") + loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so") + loadAndCreateLink("librtmp.so.1", "librtmp.so") + loadAndCreateLink("libssh2.so.1", "libssh2.so") + loadAndCreateLink("libpsl.so.5", "libpsl.so") + loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so") + loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so") + loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so") + loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so") + loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so") + loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so") + loadAndCreateLink("libglog.so.0", "libglog.so") + loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so") + loadAndCreateLink("libsnappy.so.1", "libsnappy.so") + loadAndCreateLink("libcurl.so.4", "libcurl.so") + loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so") + loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so") } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala index 8e24ef0bad52..80b20129f364 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala @@ -20,36 +20,41 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderDebian12 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader.loadAndCreateLink("libcrypto.so.3", "libcrypto.so", false) - loader.loadAndCreateLink("libkrb5support.so.0", "libkrb5support.so", false) - loader.loadAndCreateLink("libssl.so.3", "libssl.so", false) - loader.loadAndCreateLink("libicudata.so.72", "libicudata.so", false) - loader.loadAndCreateLink("libk5crypto.so.3", "libk5crypto.so", false) - loader.loadAndCreateLink("libkeyutils.so.1", "libkeyutils.so", false) - loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) - loader.loadAndCreateLink("libthrift-0.17.0.so", "libthrift.so", false) - loader.loadAndCreateLink("libicuuc.so.72", "libicuuc.so", false) - loader.loadAndCreateLink("libkrb5.so.3", "libkrb5.so", false) - loader.loadAndCreateLink("liblber-2.5.so.0", "liblber-2.4.so", false) - loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) - loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) - loader.loadAndCreateLink("libicui18n.so.72", "libicui18n.so", false) - loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false) - loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) - loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false) - loader.loadAndCreateLink("libssh2.so.1", "libssh2.so", false) - loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false) - loader.loadAndCreateLink("libgssapi_krb5.so.2", "libgssapi_krb5.so", false) - loader.loadAndCreateLink("libldap-2.5.so.0", "libldap_r-2.4.so", false) - loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) - loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - loader.loadAndCreateLink("libglog.so.1", "libglog.so", false) - loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) - loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false) - loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + def loadAndCreateLink(libName: String, linkName: String): Unit = { + val mapLibName = System.mapLibraryName(libName) + loader.loadAndCreateLink(mapLibName, linkName, false) + } + + loadAndCreateLink("libcrypto.so.3", "libcrypto.so") + loadAndCreateLink("libkrb5support.so.0", "libkrb5support.so") + loadAndCreateLink("libssl.so.3", "libssl.so") + loadAndCreateLink("libicudata.so.72", "libicudata.so") + loadAndCreateLink("libk5crypto.so.3", "libk5crypto.so") + loadAndCreateLink("libkeyutils.so.1", "libkeyutils.so") + loadAndCreateLink("libsnappy.so.1", "libsnappy.so") + loadAndCreateLink("libthrift-0.17.0.so", "libthrift.so") + loadAndCreateLink("libicuuc.so.72", "libicuuc.so") + loadAndCreateLink("libkrb5.so.3", "libkrb5.so") + loadAndCreateLink("liblber-2.5.so.0", "liblber-2.4.so") + loadAndCreateLink("libsasl2.so.2", "libsasl2.so") + loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so") + loadAndCreateLink("libicui18n.so.72", "libicui18n.so") + loadAndCreateLink("libgflags.so.2.2", "libgflags.so") + loadAndCreateLink("libunwind.so.8", "libunwind.so") + loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so") + loadAndCreateLink("librtmp.so.1", "librtmp.so") + loadAndCreateLink("libssh2.so.1", "libssh2.so") + loadAndCreateLink("libpsl.so.5", "libpsl.so") + loadAndCreateLink("libgssapi_krb5.so.2", "libgssapi_krb5.so") + loadAndCreateLink("libldap-2.5.so.0", "libldap_r-2.4.so") + loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so") + loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so") + loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so") + loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so") + loadAndCreateLink("libglog.so.1", "libglog.so") + loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so") + loadAndCreateLink("libcurl.so.4", "libcurl.so") + loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so") + loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so") } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala index a03a0a7e87c8..150eae6a1cdf 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala @@ -20,46 +20,51 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderUbuntu2004 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader.loadAndCreateLink("libroken.so.18", "libroken.so", false) - loader.loadAndCreateLink("libasn1.so.8", "libasn1.so", false) - loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) - loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) - loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false) - loader.loadAndCreateLink("libcrypto.so.1.1", "libcrypto.so", false) - loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) - loader.loadAndCreateLink("libnettle.so.7", "libnettle.so", false) - loader.loadAndCreateLink("libhogweed.so.5", "libhogweed.so", false) - loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false) - loader.loadAndCreateLink("libssh.so.4", "libssh.so", false) - loader.loadAndCreateLink("libssl.so.1.1", "libssl.so", false) - loader.loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false) - loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) - loader.loadAndCreateLink("libwind.so.0", "libwind.so", false) - loader.loadAndCreateLink("libheimbase.so.1", "libheimbase.so", false) - loader.loadAndCreateLink("libhcrypto.so.4", "libhcrypto.so", false) - loader.loadAndCreateLink("libhx509.so.5", "libhx509.so", false) - loader.loadAndCreateLink("libkrb5.so.26", "libkrb5.so", false) - loader.loadAndCreateLink("libheimntlm.so.0", "libheimntlm.so", false) - loader.loadAndCreateLink("libgssapi.so.3", "libgssapi.so", false) - loader.loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false) - loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false) - loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) - loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false) - loader.loadAndCreateLink("libglog.so.0", "libglog.so", false) - loader.loadAndCreateLink("libidn.so.11", "libidn.so", false) - loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false) - loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) - loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - loader.loadAndCreateLink("libicudata.so.66", "libicudata.so", false) - loader.loadAndCreateLink("libicuuc.so.66", "libicuuc.so", false) - loader.loadAndCreateLink("libxml2.so.2", "libxml2.so", false) - loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - loader.loadAndCreateLink("libre2.so.5", "libre2.so", false) - loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) - loader.loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false) + def loadAndCreateLink(libName: String, linkName: String): Unit = { + val mapLibName = System.mapLibraryName(libName) + loader.loadAndCreateLink(mapLibName, linkName, false) + } + + loadAndCreateLink("libroken.so.18", "libroken.so") + loadAndCreateLink("libasn1.so.8", "libasn1.so") + loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so") + loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so") + loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so") + loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so") + loadAndCreateLink("libpsl.so.5", "libpsl.so") + loadAndCreateLink("libcrypto.so.1.1", "libcrypto.so") + loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so") + loadAndCreateLink("libnettle.so.7", "libnettle.so") + loadAndCreateLink("libhogweed.so.5", "libhogweed.so") + loadAndCreateLink("librtmp.so.1", "librtmp.so") + loadAndCreateLink("libssh.so.4", "libssh.so") + loadAndCreateLink("libssl.so.1.1", "libssl.so") + loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so") + loadAndCreateLink("libsasl2.so.2", "libsasl2.so") + loadAndCreateLink("libwind.so.0", "libwind.so") + loadAndCreateLink("libheimbase.so.1", "libheimbase.so") + loadAndCreateLink("libhcrypto.so.4", "libhcrypto.so") + loadAndCreateLink("libhx509.so.5", "libhx509.so") + loadAndCreateLink("libkrb5.so.26", "libkrb5.so") + loadAndCreateLink("libheimntlm.so.0", "libheimntlm.so") + loadAndCreateLink("libgssapi.so.3", "libgssapi.so") + loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so") + loadAndCreateLink("libcurl.so.4", "libcurl.so") + loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so") + loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so") + loadAndCreateLink("libgflags.so.2.2", "libgflags.so") + loadAndCreateLink("libunwind.so.8", "libunwind.so") + loadAndCreateLink("libglog.so.0", "libglog.so") + loadAndCreateLink("libidn.so.11", "libidn.so") + loadAndCreateLink("libntlm.so.0", "libntlm.so") + loadAndCreateLink("libgsasl.so.7", "libgsasl.so") + loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so") + loadAndCreateLink("libicudata.so.66", "libicudata.so") + loadAndCreateLink("libicuuc.so.66", "libicuuc.so") + loadAndCreateLink("libxml2.so.2", "libxml2.so") + loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so") + loadAndCreateLink("libre2.so.5", "libre2.so") + loadAndCreateLink("libsnappy.so.1", "libsnappy.so") + loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so") } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala index 4bb2b59b33ea..7183f13ddb02 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala @@ -20,31 +20,36 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderUbuntu2204 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - loader.loadAndCreateLink("libicudata.so.70", "libicudata.so", false) - loader.loadAndCreateLink("libicuuc.so.70", "libicuuc.so", false) - loader.loadAndCreateLink("libicui18n.so.70", "libicui18n.so", false) - loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) - loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false) - loader.loadAndCreateLink("libssh.so.4", "libssh.so", false) - loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) - loader.loadAndCreateLink("liblber-2.5.so.0", "liblber-2.5.so", false) - loader.loadAndCreateLink("libldap-2.5.so.0", "libldap-2.5.so", false) - loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false) - loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) - loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false) - loader.loadAndCreateLink("libglog.so.0", "libglog.so", false) - loader.loadAndCreateLink("libidn.so.12", "libidn.so", false) - loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false) - loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) - loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - loader.loadAndCreateLink("libxml2.so.2", "libxml2.so", false) - loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - loader.loadAndCreateLink("libre2.so.9", "libre2.so", false) - loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) - loader.loadAndCreateLink("libthrift-0.16.0.so", "libthrift.so", false) + def loadAndCreateLink(libName: String, linkName: String): Unit = { + val mapLibName = System.mapLibraryName(libName) + loader.loadAndCreateLink(mapLibName, linkName, false) + } + + loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so") + loadAndCreateLink("libicudata.so.70", "libicudata.so") + loadAndCreateLink("libicuuc.so.70", "libicuuc.so") + loadAndCreateLink("libicui18n.so.70", "libicui18n.so") + loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so") + loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so") + loadAndCreateLink("librtmp.so.1", "librtmp.so") + loadAndCreateLink("libssh.so.4", "libssh.so") + loadAndCreateLink("libsasl2.so.2", "libsasl2.so") + loadAndCreateLink("liblber-2.5.so.0", "liblber-2.5.so") + loadAndCreateLink("libldap-2.5.so.0", "libldap-2.5.so") + loadAndCreateLink("libcurl.so.4", "libcurl.so") + loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so") + loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so") + loadAndCreateLink("libgflags.so.2.2", "libgflags.so") + loadAndCreateLink("libunwind.so.8", "libunwind.so") + loadAndCreateLink("libglog.so.0", "libglog.so") + loadAndCreateLink("libidn.so.12", "libidn.so") + loadAndCreateLink("libntlm.so.0", "libntlm.so") + loadAndCreateLink("libgsasl.so.7", "libgsasl.so") + loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so") + loadAndCreateLink("libxml2.so.2", "libxml2.so") + loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so") + loadAndCreateLink("libre2.so.9", "libre2.so") + loadAndCreateLink("libsnappy.so.1", "libsnappy.so") + loadAndCreateLink("libthrift-0.16.0.so", "libthrift.so") } } diff --git a/dev/buildbundle-veloxbe.sh b/dev/buildbundle-veloxbe.sh index eaa82730bb25..8515f7b12db7 100755 --- a/dev/buildbundle-veloxbe.sh +++ b/dev/buildbundle-veloxbe.sh @@ -8,8 +8,21 @@ function build_for_spark { mvn clean package -Pbackends-velox -Pceleborn -Puniffle -Pspark-$spark_version -DskipTests } +function check_supported { + PLATFORM=$(mvn help:evaluate -Dexpression=platform -q -DforceStdout) + ARCH=$(mvn help:evaluate -Dexpression=arch -q -DforceStdout) + if [ "$PLATFORM" == "null object or invalid expression" ] || [ "$ARCH" == "null object or invalid expression" ]; then + OS_NAME=$(mvn help:evaluate -Dexpression=os.name -q -DforceStdout) + OS_ARCH=$(mvn help:evaluate -Dexpression=os.arch -q -DforceStdout) + echo "$OS_NAME-$OS_ARCH is not supported by current Gluten build." + exit 1 + fi +} + cd $GLUTEN_DIR +check_supported + # SPARK_VERSION is defined in builddeps-veloxbe.sh if [ "$SPARK_VERSION" = "ALL" ]; then for spark_version in 3.2 3.3 3.4 3.5 diff --git a/gluten-substrait/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java b/gluten-substrait/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java index 37cd29649c5c..7ca246a8454b 100644 --- a/gluten-substrait/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java +++ b/gluten-substrait/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java @@ -158,44 +158,33 @@ public static void unloadFromPath(String libPath) { } } - public void mapAndLoad(String unmappedLibName, boolean requireUnload) { + public void load(String libPath, boolean requireUnload) { synchronized (loadedLibraries) { try { - final String mappedLibName = System.mapLibraryName(unmappedLibName); - load(mappedLibName, requireUnload); - } catch (Exception e) { - throw new GlutenException(e); - } - } - } - - public void load(String libName, boolean requireUnload) { - synchronized (loadedLibraries) { - try { - if (loadedLibraries.contains(libName)) { - LOG.debug("Library {} has already been loaded, skipping", libName); + if (loadedLibraries.contains(libPath)) { + LOG.debug("Library {} has already been loaded, skipping", libPath); return; } - File file = moveToWorkDir(workDir, libName); + File file = moveToWorkDir(workDir, libPath); loadWithLink(file.getAbsolutePath(), null, requireUnload); - loadedLibraries.add(libName); - LOG.info("Successfully loaded library {}", libName); + loadedLibraries.add(libPath); + LOG.info("Successfully loaded library {}", libPath); } catch (IOException e) { throw new GlutenException(e); } } } - public void loadAndCreateLink(String libName, String linkName, boolean requireUnload) { + public void loadAndCreateLink(String libPath, String linkName, boolean requireUnload) { synchronized (loadedLibraries) { try { - if (loadedLibraries.contains(libName)) { - LOG.debug("Library {} has already been loaded, skipping", libName); + if (loadedLibraries.contains(libPath)) { + LOG.debug("Library {} has already been loaded, skipping", libPath); } - File file = moveToWorkDir(workDir, System.mapLibraryName(libName)); + File file = moveToWorkDir(workDir, libPath); loadWithLink(file.getAbsolutePath(), linkName, requireUnload); - loadedLibraries.add(libName); - LOG.info("Successfully loaded library {}", libName); + loadedLibraries.add(libPath); + LOG.info("Successfully loaded library {}", libPath); } catch (IOException e) { throw new GlutenException(e); } @@ -209,6 +198,9 @@ private File moveToWorkDir(String workDir, String libraryToLoad) throws IOExcept Files.delete(libPath); } final File temp = new File(workDir + "/" + libraryToLoad); + if (!temp.getParentFile().exists()) { + temp.getParentFile().mkdirs(); + } try (InputStream is = JniLibLoader.class.getClassLoader().getResourceAsStream(libraryToLoad)) { if (is == null) { throw new FileNotFoundException(libraryToLoad); diff --git a/pom.xml b/pom.xml index dce60781d096..037ea82c489a 100644 --- a/pom.xml +++ b/pom.xml @@ -578,6 +578,60 @@ + + + + Darwin-x86 + + + mac + x86 + + + + darwin + x86_64 + + + + Darwin-aarch64 + + + mac + aarch64 + + + + darwin + aarch64 + + + + Linux-amd64 + + + Linux + amd64 + + + + linux + amd64 + + + + Linux-aarch64 + + + Linux + aarch64 + + + + linux + aarch64 + +