From 7e7e8e275cbba3893e3cd5a8b9a488dd512397ce Mon Sep 17 00:00:00 2001 From: arnavb Date: Mon, 12 Aug 2024 10:51:15 +0000 Subject: [PATCH 1/4] simplify jnilibloader --- .../utils/SharedLibraryLoaderCentos7.scala | 40 ++- .../utils/SharedLibraryLoaderCentos8.scala | 48 ++-- .../utils/SharedLibraryLoaderCentos9.scala | 48 ++-- .../utils/SharedLibraryLoaderDebian11.scala | 53 ++-- .../utils/SharedLibraryLoaderDebian12.scala | 65 ++--- .../utils/SharedLibraryLoaderUbuntu2004.scala | 85 +++--- .../utils/SharedLibraryLoaderUbuntu2204.scala | 55 ++-- .../gluten/vectorized/JniLibLoader.java | 265 ++++++------------ 8 files changed, 268 insertions(+), 391 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala index 47ed2c47cbb5..14bb1726e322 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala @@ -20,26 +20,24 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderCentos7 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader - .newTransaction() - .loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) - .loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) - .loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) - .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - .loadAndCreateLink("libboost_program_options.so.1.84.0", "libboost_program_options.so", false) - .loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) - .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - .loadAndCreateLink("libdouble-conversion.so.1", "libdouble-conversion.so", false) - .loadAndCreateLink("libevent-2.0.so.5", "libevent-2.0.so", false) - .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - .loadAndCreateLink("libglog.so.0", "libglog.so", false) - .loadAndCreateLink("libntlm.so.0", "libntlm.so", false) - .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) - .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - .loadAndCreateLink("libre2.so.10", "libre2.so", false) - .loadAndCreateLink("libzstd.so.1", "libzstd.so", false) - .loadAndCreateLink("liblz4.so.1", "liblz4.so", false) - .commit() + loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) + loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) + loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) + loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + loader.loadAndCreateLink + ("libboost_program_options.so.1.84.0", "libboost_program_options.so", false) + loader.loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) + loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + loader.loadAndCreateLink("libdouble-conversion.so.1", "libdouble-conversion.so", false) + loader.loadAndCreateLink("libevent-2.0.so.5", "libevent-2.0.so", false) + loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + loader.loadAndCreateLink("libglog.so.0", "libglog.so", false) + loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false) + loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) + loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + loader.loadAndCreateLink("libre2.so.10", "libre2.so", false) + loader.loadAndCreateLink("libzstd.so.1", "libzstd.so", false) + loader.loadAndCreateLink("liblz4.so.1", "liblz4.so", false) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala index c1d3bf2e26cb..fece6aef9087 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala @@ -20,30 +20,28 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderCentos8 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader - .newTransaction() - .loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) - .loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) - .loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) - .loadAndCreateLink("libicudata.so.60", "libicudata.so", false) - .loadAndCreateLink("libicuuc.so.60", "libicuuc.so", false) - .loadAndCreateLink("libicui18n.so.60", "libicui18n.so", false) - .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - .loadAndCreateLink("libboost_program_options.so.1.84.0", "libboost_program_options.so", false) - .loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) - .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - .loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - .loadAndCreateLink("libevent-2.1.so.6", "libevent-2.1.so", false) - .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - .loadAndCreateLink("libglog.so.1", "libglog.so", false) - .loadAndCreateLink("libdwarf.so.1", "libdwarf.so", false) - .loadAndCreateLink("libidn.so.11", "libidn.so", false) - .loadAndCreateLink("libntlm.so.0", "libntlm.so", false) - .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) - .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - .loadAndCreateLink("libre2.so.0", "libre2.so", false) - .loadAndCreateLink("libsodium.so.23", "libsodium.so", false) - .commit() + loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) + loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) + loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) + loader.loadAndCreateLink("libicudata.so.60", "libicudata.so", false) + loader.loadAndCreateLink("libicuuc.so.60", "libicuuc.so", false) + loader.loadAndCreateLink("libicui18n.so.60", "libicui18n.so", false) + loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + loader.loadAndCreateLink + ("libboost_program_options.so.1.84.0", "libboost_program_options.so", false) + loader.loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) + loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) + loader.loadAndCreateLink("libevent-2.1.so.6", "libevent-2.1.so", false) + loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + loader.loadAndCreateLink("libglog.so.1", "libglog.so", false) + loader.loadAndCreateLink("libdwarf.so.1", "libdwarf.so", false) + loader.loadAndCreateLink("libidn.so.11", "libidn.so", false) + loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false) + loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) + loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + loader.loadAndCreateLink("libre2.so.0", "libre2.so", false) + loader.loadAndCreateLink("libsodium.so.23", "libsodium.so", false) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala index 2d9ececb366d..2c7443569d3a 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala @@ -20,30 +20,28 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderCentos9 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader - .newTransaction() - .loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) - .loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) - .loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) - .loadAndCreateLink("libicudata.so.67", "libicudata.so", false) - .loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false) - .loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false) - .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - .loadAndCreateLink("libboost_program_options.so.1.84.0", "libboost_program_options.so", false) - .loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) - .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - .loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) - .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - .loadAndCreateLink("libglog.so.1", "libglog.so", false) - .loadAndCreateLink("libdwarf.so.0", "libdwarf.so", false) - .loadAndCreateLink("libidn.so.12", "libidn.so", false) - .loadAndCreateLink("libntlm.so.0", "libntlm.so", false) - .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) - .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - .loadAndCreateLink("libre2.so.9", "libre2.so", false) - .loadAndCreateLink("libsodium.so.23", "libsodium.so", false) - .commit() + loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) + loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) + loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) + loader.loadAndCreateLink("libicudata.so.67", "libicudata.so", false) + loader.loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false) + loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false) + loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + loader.loadAndCreateLink + ("libboost_program_options.so.1.84.0", "libboost_program_options.so", false) + loader.loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) + loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) + loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) + loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + loader.loadAndCreateLink("libglog.so.1", "libglog.so", false) + loader.loadAndCreateLink("libdwarf.so.0", "libdwarf.so", false) + loader.loadAndCreateLink("libidn.so.12", "libidn.so", false) + loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false) + loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) + loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + loader.loadAndCreateLink("libre2.so.9", "libre2.so", false) + loader.loadAndCreateLink("libsodium.so.23", "libsodium.so", false) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala index ca7d1d22d984..a300cc5b9d05 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala @@ -20,33 +20,30 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderDebian11 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader - .newTransaction() - .loadAndCreateLink("libicudata.so.67", "libicudata.so", false) - .loadAndCreateLink("libre2.so.9", "libre2.so", false) - .loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false) - .loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false) - .loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) - .loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) - .loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false) - .loadAndCreateLink("libunwind.so.8", "libunwind.so", false) - .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - .loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) - .loadAndCreateLink("librtmp.so.1", "librtmp.so", false) - .loadAndCreateLink("libssh2.so.1", "libssh2.so", false) - .loadAndCreateLink("libpsl.so.5", "libpsl.so", false) - .loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false) - .loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) - .loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false) - .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - .loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - .loadAndCreateLink("libglog.so.0", "libglog.so", false) - .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) - .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) - .loadAndCreateLink("libcurl.so.4", "libcurl.so", false) - .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - .commit() + loader.loadAndCreateLink("libicudata.so.67", "libicudata.so", false) + loader.loadAndCreateLink("libre2.so.9", "libre2.so", false) + loader.loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false) + loader.loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false) + loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) + loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) + loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false) + loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false) + loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) + loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false) + loader.loadAndCreateLink("libssh2.so.1", "libssh2.so", false) + loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false) + loader.loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false) + loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) + loader.loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false) + loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) + loader.loadAndCreateLink("libglog.so.0", "libglog.so", false) + loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) + loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) + loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false) + loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala index 128c8eaa2aef..8e24ef0bad52 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala @@ -20,39 +20,36 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderDebian12 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader - .newTransaction() - .loadAndCreateLink("libcrypto.so.3", "libcrypto.so", false) - .loadAndCreateLink("libkrb5support.so.0", "libkrb5support.so", false) - .loadAndCreateLink("libssl.so.3", "libssl.so", false) - .loadAndCreateLink("libicudata.so.72", "libicudata.so", false) - .loadAndCreateLink("libk5crypto.so.3", "libk5crypto.so", false) - .loadAndCreateLink("libkeyutils.so.1", "libkeyutils.so", false) - .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) - .loadAndCreateLink("libthrift-0.17.0.so", "libthrift.so", false) - .loadAndCreateLink("libicuuc.so.72", "libicuuc.so", false) - .loadAndCreateLink("libkrb5.so.3", "libkrb5.so", false) - .loadAndCreateLink("liblber-2.5.so.0", "liblber-2.4.so", false) - .loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) - .loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) - .loadAndCreateLink("libicui18n.so.72", "libicui18n.so", false) - .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - .loadAndCreateLink("libunwind.so.8", "libunwind.so", false) - .loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) - .loadAndCreateLink("librtmp.so.1", "librtmp.so", false) - .loadAndCreateLink("libssh2.so.1", "libssh2.so", false) - .loadAndCreateLink("libpsl.so.5", "libpsl.so", false) - .loadAndCreateLink("libgssapi_krb5.so.2", "libgssapi_krb5.so", false) - .loadAndCreateLink("libldap-2.5.so.0", "libldap_r-2.4.so", false) - .loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) - .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - .loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - .loadAndCreateLink("libglog.so.1", "libglog.so", false) - .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) - .loadAndCreateLink("libcurl.so.4", "libcurl.so", false) - .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - .commit() + loader.loadAndCreateLink("libcrypto.so.3", "libcrypto.so", false) + loader.loadAndCreateLink("libkrb5support.so.0", "libkrb5support.so", false) + loader.loadAndCreateLink("libssl.so.3", "libssl.so", false) + loader.loadAndCreateLink("libicudata.so.72", "libicudata.so", false) + loader.loadAndCreateLink("libk5crypto.so.3", "libk5crypto.so", false) + loader.loadAndCreateLink("libkeyutils.so.1", "libkeyutils.so", false) + loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) + loader.loadAndCreateLink("libthrift-0.17.0.so", "libthrift.so", false) + loader.loadAndCreateLink("libicuuc.so.72", "libicuuc.so", false) + loader.loadAndCreateLink("libkrb5.so.3", "libkrb5.so", false) + loader.loadAndCreateLink("liblber-2.5.so.0", "liblber-2.4.so", false) + loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) + loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) + loader.loadAndCreateLink("libicui18n.so.72", "libicui18n.so", false) + loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false) + loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) + loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false) + loader.loadAndCreateLink("libssh2.so.1", "libssh2.so", false) + loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false) + loader.loadAndCreateLink("libgssapi_krb5.so.2", "libgssapi_krb5.so", false) + loader.loadAndCreateLink("libldap-2.5.so.0", "libldap_r-2.4.so", false) + loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) + loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) + loader.loadAndCreateLink("libglog.so.1", "libglog.so", false) + loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) + loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false) + loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala index 18f2e6cfbeb3..a03a0a7e87c8 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala @@ -20,49 +20,46 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderUbuntu2004 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader - .newTransaction() - .loadAndCreateLink("libroken.so.18", "libroken.so", false) - .loadAndCreateLink("libasn1.so.8", "libasn1.so", false) - .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - .loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) - .loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) - .loadAndCreateLink("libpsl.so.5", "libpsl.so", false) - .loadAndCreateLink("libcrypto.so.1.1", "libcrypto.so", false) - .loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) - .loadAndCreateLink("libnettle.so.7", "libnettle.so", false) - .loadAndCreateLink("libhogweed.so.5", "libhogweed.so", false) - .loadAndCreateLink("librtmp.so.1", "librtmp.so", false) - .loadAndCreateLink("libssh.so.4", "libssh.so", false) - .loadAndCreateLink("libssl.so.1.1", "libssl.so", false) - .loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false) - .loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) - .loadAndCreateLink("libwind.so.0", "libwind.so", false) - .loadAndCreateLink("libheimbase.so.1", "libheimbase.so", false) - .loadAndCreateLink("libhcrypto.so.4", "libhcrypto.so", false) - .loadAndCreateLink("libhx509.so.5", "libhx509.so", false) - .loadAndCreateLink("libkrb5.so.26", "libkrb5.so", false) - .loadAndCreateLink("libheimntlm.so.0", "libheimntlm.so", false) - .loadAndCreateLink("libgssapi.so.3", "libgssapi.so", false) - .loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false) - .loadAndCreateLink("libcurl.so.4", "libcurl.so", false) - .loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) - .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - .loadAndCreateLink("libunwind.so.8", "libunwind.so", false) - .loadAndCreateLink("libglog.so.0", "libglog.so", false) - .loadAndCreateLink("libidn.so.11", "libidn.so", false) - .loadAndCreateLink("libntlm.so.0", "libntlm.so", false) - .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) - .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libicudata.so.66", "libicudata.so", false) - .loadAndCreateLink("libicuuc.so.66", "libicuuc.so", false) - .loadAndCreateLink("libxml2.so.2", "libxml2.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - .loadAndCreateLink("libre2.so.5", "libre2.so", false) - .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) - .loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false) - .commit() + loader.loadAndCreateLink("libroken.so.18", "libroken.so", false) + loader.loadAndCreateLink("libasn1.so.8", "libasn1.so", false) + loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) + loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) + loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false) + loader.loadAndCreateLink("libcrypto.so.1.1", "libcrypto.so", false) + loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) + loader.loadAndCreateLink("libnettle.so.7", "libnettle.so", false) + loader.loadAndCreateLink("libhogweed.so.5", "libhogweed.so", false) + loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false) + loader.loadAndCreateLink("libssh.so.4", "libssh.so", false) + loader.loadAndCreateLink("libssl.so.1.1", "libssl.so", false) + loader.loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false) + loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) + loader.loadAndCreateLink("libwind.so.0", "libwind.so", false) + loader.loadAndCreateLink("libheimbase.so.1", "libheimbase.so", false) + loader.loadAndCreateLink("libhcrypto.so.4", "libhcrypto.so", false) + loader.loadAndCreateLink("libhx509.so.5", "libhx509.so", false) + loader.loadAndCreateLink("libkrb5.so.26", "libkrb5.so", false) + loader.loadAndCreateLink("libheimntlm.so.0", "libheimntlm.so", false) + loader.loadAndCreateLink("libgssapi.so.3", "libgssapi.so", false) + loader.loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false) + loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false) + loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) + loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) + loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false) + loader.loadAndCreateLink("libglog.so.0", "libglog.so", false) + loader.loadAndCreateLink("libidn.so.11", "libidn.so", false) + loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false) + loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) + loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + loader.loadAndCreateLink("libicudata.so.66", "libicudata.so", false) + loader.loadAndCreateLink("libicuuc.so.66", "libicuuc.so", false) + loader.loadAndCreateLink("libxml2.so.2", "libxml2.so", false) + loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + loader.loadAndCreateLink("libre2.so.5", "libre2.so", false) + loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) + loader.loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala index b23105b7dce0..4bb2b59b33ea 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala @@ -20,34 +20,31 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderUbuntu2204 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader - .newTransaction() - .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - .loadAndCreateLink("libicudata.so.70", "libicudata.so", false) - .loadAndCreateLink("libicuuc.so.70", "libicuuc.so", false) - .loadAndCreateLink("libicui18n.so.70", "libicui18n.so", false) - .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - .loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) - .loadAndCreateLink("librtmp.so.1", "librtmp.so", false) - .loadAndCreateLink("libssh.so.4", "libssh.so", false) - .loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) - .loadAndCreateLink("liblber-2.5.so.0", "liblber-2.5.so", false) - .loadAndCreateLink("libldap-2.5.so.0", "libldap-2.5.so", false) - .loadAndCreateLink("libcurl.so.4", "libcurl.so", false) - .loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) - .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - .loadAndCreateLink("libunwind.so.8", "libunwind.so", false) - .loadAndCreateLink("libglog.so.0", "libglog.so", false) - .loadAndCreateLink("libidn.so.12", "libidn.so", false) - .loadAndCreateLink("libntlm.so.0", "libntlm.so", false) - .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) - .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libxml2.so.2", "libxml2.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - .loadAndCreateLink("libre2.so.9", "libre2.so", false) - .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) - .loadAndCreateLink("libthrift-0.16.0.so", "libthrift.so", false) - .commit() + loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + loader.loadAndCreateLink("libicudata.so.70", "libicudata.so", false) + loader.loadAndCreateLink("libicuuc.so.70", "libicuuc.so", false) + loader.loadAndCreateLink("libicui18n.so.70", "libicui18n.so", false) + loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) + loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false) + loader.loadAndCreateLink("libssh.so.4", "libssh.so", false) + loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) + loader.loadAndCreateLink("liblber-2.5.so.0", "liblber-2.5.so", false) + loader.loadAndCreateLink("libldap-2.5.so.0", "libldap-2.5.so", false) + loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false) + loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) + loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) + loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false) + loader.loadAndCreateLink("libglog.so.0", "libglog.so", false) + loader.loadAndCreateLink("libidn.so.12", "libidn.so", false) + loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false) + loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) + loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + loader.loadAndCreateLink("libxml2.so.2", "libxml2.so", false) + loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + loader.loadAndCreateLink("libre2.so.9", "libre2.so", false) + loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) + loader.loadAndCreateLink("libthrift-0.16.0.so", "libthrift.so", false) } } diff --git a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java index b1feb1e5baeb..0da24e149e5b 100644 --- a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java +++ b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java @@ -35,25 +35,15 @@ import java.util.Collections; import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.Vector; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; -import java.util.stream.Stream; import scala.runtime.BoxedUnit; -/** - * LoadXXX methods in the utility prevents reloading of a library internally. It's not necessary for - * caller to manage a loaded library list. - */ public class JniLibLoader { private static final Logger LOG = LoggerFactory.getLogger(JniLibLoader.class); @@ -72,7 +62,7 @@ public class JniLibLoader { private final Set loadedLibraries = new HashSet<>(); private final Lock sync = new ReentrantLock(); - JniLibLoader(String workDir) { + public JniLibLoader(String workDir) { this.workDir = workDir; } @@ -117,25 +107,9 @@ public static void loadFromPath(String libPath, boolean requireUnload) { loadFromPath0(file.getAbsolutePath(), requireUnload); } - public void mapAndLoad(String unmappedLibName, boolean requireUnload) { - newTransaction().mapAndLoad(unmappedLibName, requireUnload).commit(); - } - - public void load(String libName, boolean requireUnload) { - newTransaction().load(libName, requireUnload).commit(); - } - - public void loadAndCreateLink(String libName, String linkName, boolean requireUnload) { - newTransaction().loadAndCreateLink(libName, linkName, requireUnload).commit(); - } - - public JniLoadTransaction newTransaction() { - return new JniLoadTransaction(); - } - public static synchronized void unloadFromPath(String libPath) { if (!LOADED_LIBRARY_PATHS.remove(libPath)) { - LOG.warn("Library {} was not loaded or alreay unloaded:", libPath); + LOG.warn("Library {} was not loaded or already unloaded:", libPath); return; } @@ -147,25 +121,26 @@ public static synchronized void unloadFromPath(String libPath) { Field field = ClassLoader.class.getDeclaredField("nativeLibraries"); field.setAccessible(true); Vector libs = (Vector) field.get(classLoader); - Iterator it = libs.iterator(); - while (it.hasNext()) { - Object object = it.next(); - Field[] fs = object.getClass().getDeclaredFields(); - for (int k = 0; k < fs.length; k++) { - if (fs[k].getName().equals("name")) { - fs[k].setAccessible(true); - - String verbosePath = fs[k].get(object).toString(); - File verboseFile = new File(verbosePath); - String verboseFileName = verboseFile.getName(); - File libFile = new File(libPath); - String libFileName = libFile.getName(); - - if (verboseFileName.equals(libFileName)) { - LOG.info("Finalizing library file: {}", libFileName); - Method finalize = object.getClass().getDeclaredMethod("finalize"); - finalize.setAccessible(true); - finalize.invoke(object); + synchronized (libs) { + Iterator it = libs.iterator(); + while (it.hasNext()) { + Object object = it.next(); + Field[] fs = object.getClass().getDeclaredFields(); + for (int k = 0; k < fs.length; k++) { + if (fs[k].getName().equals("name")) { + fs[k].setAccessible(true); + String verbosePath = fs[k].get(object).toString(); + File verboseFile = new File(verbosePath); + String verboseFileName = verboseFile.getName(); + File libFile = new File(libPath); + String libFileName = libFile.getName(); + + if (verboseFileName.equals(libFileName)) { + LOG.info("Finalizing library file: {}", libFileName); + Method finalize = object.getClass().getDeclaredMethod("finalize"); + finalize.setAccessible(true); + finalize.invoke(object); + } } } } @@ -175,160 +150,80 @@ public static synchronized void unloadFromPath(String libPath) { } } - private static final class LoadRequest { - final String libName; - final String linkName; - final boolean requireUnload; - - private LoadRequest(String libName, String linkName, boolean requireUnload) { - this.libName = libName; - this.linkName = linkName; - this.requireUnload = requireUnload; + public void mapAndLoad(String unmappedLibName, boolean requireUnload) { + sync.lock(); + try { + final String mappedLibName = System.mapLibraryName(unmappedLibName); + load(mappedLibName, requireUnload); + } catch (Exception e) { + throw new GlutenException(e); + } finally { + sync.unlock(); } } - private static final class LoadAction { - final String libName; - final String linkName; - final boolean requireUnload; - final File file; - - private LoadAction(String libName, String linkName, boolean requireUnload, File file) { - this.libName = libName; - this.linkName = linkName; - this.requireUnload = requireUnload; - this.file = file; - } - - public boolean requireLinking() { - return !Objects.isNull(linkName); + public void load(String libName, boolean requireUnload) { + sync.lock(); + try { + if (loadedLibraries.contains(libName)) { + LOG.debug("Library {} has already been loaded, skipping", libName); + return; + } + File file = moveToWorkDir(workDir, System.mapLibraryName(libName)); + loadWithLink(file.getAbsolutePath(), null, requireUnload); + loadedLibraries.add(libName); + LOG.info("Successfully loaded library {}", libName); + } catch (IOException e) { + throw new GlutenException(e); + } finally { + sync.unlock(); } } - public class JniLoadTransaction { - private final AtomicBoolean finished = new AtomicBoolean(false); - private final Map toLoad = new LinkedHashMap<>(); // ordered - - private JniLoadTransaction() { - JniLibLoader.this.sync.lock(); - } - - public JniLoadTransaction mapAndLoad(String unmappedLibName, boolean requireUnload) { - try { - final String mappedLibName = System.mapLibraryName(unmappedLibName); - load(mappedLibName, requireUnload); - return this; - } catch (Exception e) { - abort(); - throw new GlutenException(e); + public void loadAndCreateLink(String libName, String linkName, boolean requireUnload) { + sync.lock(); + try { + if (loadedLibraries.contains(libName)) { + LOG.debug("Library {} has already been loaded, skipping", libName); } + File file = moveToWorkDir(workDir, System.mapLibraryName(libName)); + loadWithLink(file.getAbsolutePath(), linkName, requireUnload); + loadedLibraries.add(libName); + LOG.info("Successfully loaded library {}", libName); + } catch (IOException e) { + throw new GlutenException(e); + } finally { + sync.unlock(); } + } - public JniLoadTransaction load(String libName, boolean requireUnload) { - try { - toLoad.put(libName, new LoadRequest(libName, null, requireUnload)); - return this; - } catch (Exception e) { - abort(); - throw new GlutenException(e); - } + private File moveToWorkDir(String workDir, String libraryToLoad) throws IOException { + // final File temp = File.createTempFile(workDir, libraryToLoad); + final Path libPath = Paths.get(workDir + "/" + libraryToLoad); + if (Files.exists(libPath)) { + Files.delete(libPath); } - - public JniLoadTransaction loadAndCreateLink( - String libName, String linkName, boolean requireUnload) { + final File temp = new File(workDir + "/" + libraryToLoad); + try (InputStream is = JniLibLoader.class.getClassLoader().getResourceAsStream(libraryToLoad)) { + if (is == null) { + throw new FileNotFoundException(libraryToLoad); + } try { - toLoad.put(libName, new LoadRequest(libName, linkName, requireUnload)); - return this; + Files.copy(is, temp.toPath()); } catch (Exception e) { - abort(); throw new GlutenException(e); } } + return temp; + } - public void commit() { - try { - terminate(); - toLoad.entrySet().stream() - .flatMap( - e -> { - try { - final LoadRequest req = e.getValue(); - if (loadedLibraries.contains(req.libName)) { - LOG.debug("Library {} has already been loaded, skipping", req.libName); - return Stream.empty(); - } - // load only libraries not loaded yet - final File file = moveToWorkDir(workDir, req.libName); - return Stream.of( - new LoadAction(req.libName, req.linkName, req.requireUnload, file)); - } catch (IOException ex) { - throw new GlutenException(ex); - } - }) - .collect(Collectors.toList()) - .forEach( - e -> { - try { - LOG.info("Trying to load library {}", e.libName); - loadWithLink(workDir, e); - loadedLibraries.add(e.libName); - LOG.info("Successfully loaded library {}", e.libName); - } catch (Exception ex) { - throw new GlutenException(ex); - } - }); - } finally { - JniLibLoader.this.sync.unlock(); - } - } - - public void abort() { - try { - terminate(); - // do nothing as of now - } finally { - JniLibLoader.this.sync.unlock(); - } - } - - private void terminate() { - if (!finished.compareAndSet(false, true)) { - throw new IllegalStateException(); - } - } - - private File moveToWorkDir(String workDir, String libraryToLoad) throws IOException { - // final File temp = File.createTempFile(workDir, libraryToLoad); - final Path libPath = Paths.get(workDir + "/" + libraryToLoad); - if (Files.exists(libPath)) { - Files.delete(libPath); - } - final File temp = new File(workDir + "/" + libraryToLoad); - try (InputStream is = - JniLibLoader.class.getClassLoader().getResourceAsStream(libraryToLoad)) { - if (is == null) { - throw new FileNotFoundException(libraryToLoad); - } - try { - Files.copy(is, temp.toPath()); - } catch (Exception e) { - throw new GlutenException(e); - } - } - return temp; - } - - private void loadWithLink(String workDir, LoadAction req) throws IOException { - String libPath = req.file.getAbsolutePath(); - loadFromPath0(libPath, req.requireUnload); - LOG.info("Library {} has been loaded", libPath); - if (!req.requireLinking()) { - LOG.debug("Symbolic link not required for library {}, skipping", libPath); - return; - } - // create link - Path target = Paths.get(req.file.getPath()); - Path link = Paths.get(workDir, req.linkName); + private void loadWithLink(String libPath, String linkName, boolean requireUnload) + throws IOException { + loadFromPath0(libPath, requireUnload); + LOG.info("Library {} has been loaded", libPath); + if (linkName != null) { + Path target = Paths.get(libPath); + Path link = Paths.get(workDir, linkName); if (Files.exists(link)) { LOG.info("Symbolic link already exists for library {}, deleting", libPath); Files.delete(link); From 60b02a0a4add8fb823bdc1486537a43cfde25b29 Mon Sep 17 00:00:00 2001 From: arnavb Date: Mon, 12 Aug 2024 13:06:12 +0000 Subject: [PATCH 2/4] Bug fix + Lint --- .../apache/gluten/utils/SharedLibraryLoaderCentos7.scala | 6 ++++-- .../apache/gluten/utils/SharedLibraryLoaderCentos8.scala | 6 ++++-- .../apache/gluten/utils/SharedLibraryLoaderCentos9.scala | 6 ++++-- .../java/org/apache/gluten/vectorized/JniLibLoader.java | 2 +- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala index 14bb1726e322..a7750dcb3e65 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala @@ -24,8 +24,10 @@ class SharedLibraryLoaderCentos7 extends SharedLibraryLoader { loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - loader.loadAndCreateLink - ("libboost_program_options.so.1.84.0", "libboost_program_options.so", false) + loader.loadAndCreateLink( + "libboost_program_options.so.1.84.0", + "libboost_program_options.so", + false) loader.loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) loader.loadAndCreateLink("libdouble-conversion.so.1", "libdouble-conversion.so", false) diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala index fece6aef9087..bd8bf15bec9f 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala @@ -27,8 +27,10 @@ class SharedLibraryLoaderCentos8 extends SharedLibraryLoader { loader.loadAndCreateLink("libicuuc.so.60", "libicuuc.so", false) loader.loadAndCreateLink("libicui18n.so.60", "libicui18n.so", false) loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - loader.loadAndCreateLink - ("libboost_program_options.so.1.84.0", "libboost_program_options.so", false) + loader.loadAndCreateLink( + "libboost_program_options.so.1.84.0", + "libboost_program_options.so", + false) loader.loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala index 2c7443569d3a..06fb25c8b0e4 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala @@ -27,8 +27,10 @@ class SharedLibraryLoaderCentos9 extends SharedLibraryLoader { loader.loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false) loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false) loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - loader.loadAndCreateLink - ("libboost_program_options.so.1.84.0", "libboost_program_options.so", false) + loader.loadAndCreateLink( + "libboost_program_options.so.1.84.0", + "libboost_program_options.so", + false) loader.loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) diff --git a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java index 0da24e149e5b..64d0f5c37d74 100644 --- a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java +++ b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java @@ -169,7 +169,7 @@ public void load(String libName, boolean requireUnload) { LOG.debug("Library {} has already been loaded, skipping", libName); return; } - File file = moveToWorkDir(workDir, System.mapLibraryName(libName)); + File file = moveToWorkDir(workDir, libName); loadWithLink(file.getAbsolutePath(), null, requireUnload); loadedLibraries.add(libName); LOG.info("Successfully loaded library {}", libName); From c929a21afde2eba68ea1e0a7699b969adf86b209 Mon Sep 17 00:00:00 2001 From: arnavb Date: Tue, 13 Aug 2024 10:43:08 +0000 Subject: [PATCH 3/4] update --- .../gluten/vectorized/JniLibLoader.java | 117 +++++++++--------- 1 file changed, 61 insertions(+), 56 deletions(-) diff --git a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java index 64d0f5c37d74..53e8121c7684 100644 --- a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java +++ b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java @@ -47,8 +47,8 @@ public class JniLibLoader { private static final Logger LOG = LoggerFactory.getLogger(JniLibLoader.class); - private static final Set LOADED_LIBRARY_PATHS = new HashSet<>(); - private static final Set REQUIRE_UNLOAD_LIBRARY_PATHS = new LinkedHashSet<>(); + private static final Set LOADED_LIBRARY_PATHS = Collections.synchronizedSet(new HashSet<>()); + private static final Set REQUIRE_UNLOAD_LIBRARY_PATHS = Collections.synchronizedSet(new LinkedHashSet<>()); static { GlutenShutdownManager.addHookForLibUnloading( @@ -59,15 +59,17 @@ public class JniLibLoader { } private final String workDir; - private final Set loadedLibraries = new HashSet<>(); - private final Lock sync = new ReentrantLock(); + private final Set loadedLibraries = Collections.synchronizedSet(new HashSet<>()); - public JniLibLoader(String workDir) { + JniLibLoader(String workDir) { this.workDir = workDir; } - public static synchronized void forceUnloadAll() { - List loaded = new ArrayList<>(REQUIRE_UNLOAD_LIBRARY_PATHS); + public static void forceUnloadAll() { + List loaded; + synchronized (REQUIRE_UNLOAD_LIBRARY_PATHS) { + loaded = new ArrayList<>(REQUIRE_UNLOAD_LIBRARY_PATHS); + } Collections.reverse(loaded); // use reversed order to unload loaded.forEach(JniLibLoader::unloadFromPath); } @@ -85,21 +87,25 @@ private static String toRealPath(String libPath) { } } - private static synchronized void loadFromPath0(String libPath, boolean requireUnload) { + private static void loadFromPath0(String libPath, boolean requireUnload) { libPath = toRealPath(libPath); - if (LOADED_LIBRARY_PATHS.contains(libPath)) { - LOG.debug("Library in path {} has already been loaded, skipping", libPath); - } else { - System.load(libPath); - LOADED_LIBRARY_PATHS.add(libPath); - LOG.info("Library {} has been loaded using path-loading method", libPath); + synchronized (LOADED_LIBRARY_PATHS) { + if (LOADED_LIBRARY_PATHS.contains(libPath)) { + LOG.debug("Library in path {} has already been loaded, skipping", libPath); + } else { + System.load(libPath); + LOADED_LIBRARY_PATHS.add(libPath); + LOG.info("Library {} has been loaded using path-loading method", libPath); + } } if (requireUnload) { - REQUIRE_UNLOAD_LIBRARY_PATHS.add(libPath); + synchronized (REQUIRE_UNLOAD_LIBRARY_PATHS) { + REQUIRE_UNLOAD_LIBRARY_PATHS.add(libPath); + } } } - public static void loadFromPath(String libPath, boolean requireUnload) { + public static synchronized void loadFromPath(String libPath, boolean requireUnload) { final File file = new File(libPath); if (!file.isFile() || !file.exists()) { throw new GlutenException("library at path: " + libPath + " is not a file or does not exist"); @@ -107,15 +113,17 @@ public static void loadFromPath(String libPath, boolean requireUnload) { loadFromPath0(file.getAbsolutePath(), requireUnload); } - public static synchronized void unloadFromPath(String libPath) { - if (!LOADED_LIBRARY_PATHS.remove(libPath)) { - LOG.warn("Library {} was not loaded or already unloaded:", libPath); - return; + public static void unloadFromPath(String libPath) { + synchronized (LOADED_LIBRARY_PATHS) { + if (!LOADED_LIBRARY_PATHS.remove(libPath)) { + LOG.warn("Library {} was not loaded or already unloaded:", libPath); + return; + } } - LOG.info("Starting unload library path: {} ", libPath); - REQUIRE_UNLOAD_LIBRARY_PATHS.remove(libPath); - + synchronized (REQUIRE_UNLOAD_LIBRARY_PATHS) { + REQUIRE_UNLOAD_LIBRARY_PATHS.remove(libPath); + } try { ClassLoader classLoader = JniLibLoader.class.getClassLoader(); Field field = ClassLoader.class.getDeclaredField("nativeLibraries"); @@ -151,49 +159,46 @@ public static synchronized void unloadFromPath(String libPath) { } public void mapAndLoad(String unmappedLibName, boolean requireUnload) { - sync.lock(); - try { - final String mappedLibName = System.mapLibraryName(unmappedLibName); - load(mappedLibName, requireUnload); - } catch (Exception e) { - throw new GlutenException(e); - } finally { - sync.unlock(); + synchronized (loadedLibraries) { + try { + final String mappedLibName = System.mapLibraryName(unmappedLibName); + load(mappedLibName, requireUnload); + } catch (Exception e) { + throw new GlutenException(e); + } } } public void load(String libName, boolean requireUnload) { - sync.lock(); - try { - if (loadedLibraries.contains(libName)) { - LOG.debug("Library {} has already been loaded, skipping", libName); - return; + synchronized (loadedLibraries) { + try { + if (loadedLibraries.contains(libName)) { + LOG.debug("Library {} has already been loaded, skipping", libName); + return; + } + File file = moveToWorkDir(workDir, libName); + loadWithLink(file.getAbsolutePath(), null, requireUnload); + loadedLibraries.add(libName); + LOG.info("Successfully loaded library {}", libName); + } catch (IOException e) { + throw new GlutenException(e); } - File file = moveToWorkDir(workDir, libName); - loadWithLink(file.getAbsolutePath(), null, requireUnload); - loadedLibraries.add(libName); - LOG.info("Successfully loaded library {}", libName); - } catch (IOException e) { - throw new GlutenException(e); - } finally { - sync.unlock(); } } public void loadAndCreateLink(String libName, String linkName, boolean requireUnload) { - sync.lock(); - try { - if (loadedLibraries.contains(libName)) { - LOG.debug("Library {} has already been loaded, skipping", libName); + synchronized (loadedLibraries) { + try { + if (loadedLibraries.contains(libName)) { + LOG.debug("Library {} has already been loaded, skipping", libName); + } + File file = moveToWorkDir(workDir, System.mapLibraryName(libName)); + loadWithLink(file.getAbsolutePath(), linkName, requireUnload); + loadedLibraries.add(libName); + LOG.info("Successfully loaded library {}", libName); + } catch (IOException e) { + throw new GlutenException(e); } - File file = moveToWorkDir(workDir, System.mapLibraryName(libName)); - loadWithLink(file.getAbsolutePath(), linkName, requireUnload); - loadedLibraries.add(libName); - LOG.info("Successfully loaded library {}", libName); - } catch (IOException e) { - throw new GlutenException(e); - } finally { - sync.unlock(); } } From a95e023f94479312a65d9c83df8602f84b5ce231 Mon Sep 17 00:00:00 2001 From: arnavb Date: Tue, 13 Aug 2024 15:15:13 +0000 Subject: [PATCH 4/4] update --- .../java/org/apache/gluten/vectorized/JniLibLoader.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java index 53e8121c7684..37cd29649c5c 100644 --- a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java +++ b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java @@ -39,16 +39,16 @@ import java.util.List; import java.util.Set; import java.util.Vector; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; import scala.runtime.BoxedUnit; public class JniLibLoader { private static final Logger LOG = LoggerFactory.getLogger(JniLibLoader.class); - private static final Set LOADED_LIBRARY_PATHS = Collections.synchronizedSet(new HashSet<>()); - private static final Set REQUIRE_UNLOAD_LIBRARY_PATHS = Collections.synchronizedSet(new LinkedHashSet<>()); + private static final Set LOADED_LIBRARY_PATHS = + Collections.synchronizedSet(new HashSet<>()); + private static final Set REQUIRE_UNLOAD_LIBRARY_PATHS = + Collections.synchronizedSet(new LinkedHashSet<>()); static { GlutenShutdownManager.addHookForLibUnloading(