From ef5f01cba5c6c7ddf57734ca685a458174fc9064 Mon Sep 17 00:00:00 2001 From: Arnav Balyan <60175178+ArnavBalyan@users.noreply.github.com> Date: Wed, 14 Aug 2024 06:15:49 +0530 Subject: [PATCH] [GLUTEN-6148][CORE] Simplify JniLibLoader loading mechanism for native libraries (#6791) Closes #6148 --- .../utils/SharedLibraryLoaderCentos7.scala | 42 +-- .../utils/SharedLibraryLoaderCentos8.scala | 50 +-- .../utils/SharedLibraryLoaderCentos9.scala | 50 +-- .../utils/SharedLibraryLoaderDebian11.scala | 53 ++-- .../utils/SharedLibraryLoaderDebian12.scala | 65 ++-- .../utils/SharedLibraryLoaderUbuntu2004.scala | 85 +++-- .../utils/SharedLibraryLoaderUbuntu2204.scala | 55 ++-- .../gluten/vectorized/JniLibLoader.java | 300 ++++++------------ 8 files changed, 294 insertions(+), 406 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala index 47ed2c47cbb5..a7750dcb3e65 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos7.scala @@ -20,26 +20,26 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderCentos7 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader - .newTransaction() - .loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) - .loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) - .loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) - .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - .loadAndCreateLink("libboost_program_options.so.1.84.0", "libboost_program_options.so", false) - .loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) - .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - .loadAndCreateLink("libdouble-conversion.so.1", "libdouble-conversion.so", false) - .loadAndCreateLink("libevent-2.0.so.5", "libevent-2.0.so", false) - .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - .loadAndCreateLink("libglog.so.0", "libglog.so", false) - .loadAndCreateLink("libntlm.so.0", "libntlm.so", false) - .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) - .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - .loadAndCreateLink("libre2.so.10", "libre2.so", false) - .loadAndCreateLink("libzstd.so.1", "libzstd.so", false) - .loadAndCreateLink("liblz4.so.1", "liblz4.so", false) - .commit() + loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) + loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) + loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) + loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + loader.loadAndCreateLink( + "libboost_program_options.so.1.84.0", + "libboost_program_options.so", + false) + loader.loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) + loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + loader.loadAndCreateLink("libdouble-conversion.so.1", "libdouble-conversion.so", false) + loader.loadAndCreateLink("libevent-2.0.so.5", "libevent-2.0.so", false) + loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + loader.loadAndCreateLink("libglog.so.0", "libglog.so", false) + loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false) + loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) + loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + loader.loadAndCreateLink("libre2.so.10", "libre2.so", false) + loader.loadAndCreateLink("libzstd.so.1", "libzstd.so", false) + loader.loadAndCreateLink("liblz4.so.1", "liblz4.so", false) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala index c1d3bf2e26cb..bd8bf15bec9f 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos8.scala @@ -20,30 +20,30 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderCentos8 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader - .newTransaction() - .loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) - .loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) - .loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) - .loadAndCreateLink("libicudata.so.60", "libicudata.so", false) - .loadAndCreateLink("libicuuc.so.60", "libicuuc.so", false) - .loadAndCreateLink("libicui18n.so.60", "libicui18n.so", false) - .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - .loadAndCreateLink("libboost_program_options.so.1.84.0", "libboost_program_options.so", false) - .loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) - .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - .loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - .loadAndCreateLink("libevent-2.1.so.6", "libevent-2.1.so", false) - .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - .loadAndCreateLink("libglog.so.1", "libglog.so", false) - .loadAndCreateLink("libdwarf.so.1", "libdwarf.so", false) - .loadAndCreateLink("libidn.so.11", "libidn.so", false) - .loadAndCreateLink("libntlm.so.0", "libntlm.so", false) - .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) - .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - .loadAndCreateLink("libre2.so.0", "libre2.so", false) - .loadAndCreateLink("libsodium.so.23", "libsodium.so", false) - .commit() + loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) + loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) + loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) + loader.loadAndCreateLink("libicudata.so.60", "libicudata.so", false) + loader.loadAndCreateLink("libicuuc.so.60", "libicuuc.so", false) + loader.loadAndCreateLink("libicui18n.so.60", "libicui18n.so", false) + loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + loader.loadAndCreateLink( + "libboost_program_options.so.1.84.0", + "libboost_program_options.so", + false) + loader.loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) + loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) + loader.loadAndCreateLink("libevent-2.1.so.6", "libevent-2.1.so", false) + loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + loader.loadAndCreateLink("libglog.so.1", "libglog.so", false) + loader.loadAndCreateLink("libdwarf.so.1", "libdwarf.so", false) + loader.loadAndCreateLink("libidn.so.11", "libidn.so", false) + loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false) + loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) + loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + loader.loadAndCreateLink("libre2.so.0", "libre2.so", false) + loader.loadAndCreateLink("libsodium.so.23", "libsodium.so", false) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala index 2d9ececb366d..06fb25c8b0e4 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderCentos9.scala @@ -20,30 +20,30 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderCentos9 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader - .newTransaction() - .loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) - .loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) - .loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) - .loadAndCreateLink("libicudata.so.67", "libicudata.so", false) - .loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false) - .loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false) - .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - .loadAndCreateLink("libboost_program_options.so.1.84.0", "libboost_program_options.so", false) - .loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) - .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - .loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) - .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - .loadAndCreateLink("libglog.so.1", "libglog.so", false) - .loadAndCreateLink("libdwarf.so.0", "libdwarf.so", false) - .loadAndCreateLink("libidn.so.12", "libidn.so", false) - .loadAndCreateLink("libntlm.so.0", "libntlm.so", false) - .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) - .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - .loadAndCreateLink("libre2.so.9", "libre2.so", false) - .loadAndCreateLink("libsodium.so.23", "libsodium.so", false) - .commit() + loader.loadAndCreateLink("libboost_atomic.so.1.84.0", "libboost_atomic.so", false) + loader.loadAndCreateLink("libboost_thread.so.1.84.0", "libboost_thread.so", false) + loader.loadAndCreateLink("libboost_system.so.1.84.0", "libboost_system.so", false) + loader.loadAndCreateLink("libicudata.so.67", "libicudata.so", false) + loader.loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false) + loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false) + loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + loader.loadAndCreateLink( + "libboost_program_options.so.1.84.0", + "libboost_program_options.so", + false) + loader.loadAndCreateLink("libboost_filesystem.so.1.84.0", "libboost_filesystem.so", false) + loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) + loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) + loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + loader.loadAndCreateLink("libglog.so.1", "libglog.so", false) + loader.loadAndCreateLink("libdwarf.so.0", "libdwarf.so", false) + loader.loadAndCreateLink("libidn.so.12", "libidn.so", false) + loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false) + loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) + loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + loader.loadAndCreateLink("libre2.so.9", "libre2.so", false) + loader.loadAndCreateLink("libsodium.so.23", "libsodium.so", false) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala index ca7d1d22d984..a300cc5b9d05 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian11.scala @@ -20,33 +20,30 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderDebian11 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader - .newTransaction() - .loadAndCreateLink("libicudata.so.67", "libicudata.so", false) - .loadAndCreateLink("libre2.so.9", "libre2.so", false) - .loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false) - .loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false) - .loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) - .loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) - .loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false) - .loadAndCreateLink("libunwind.so.8", "libunwind.so", false) - .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - .loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) - .loadAndCreateLink("librtmp.so.1", "librtmp.so", false) - .loadAndCreateLink("libssh2.so.1", "libssh2.so", false) - .loadAndCreateLink("libpsl.so.5", "libpsl.so", false) - .loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false) - .loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) - .loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false) - .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - .loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - .loadAndCreateLink("libglog.so.0", "libglog.so", false) - .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) - .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) - .loadAndCreateLink("libcurl.so.4", "libcurl.so", false) - .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - .commit() + loader.loadAndCreateLink("libicudata.so.67", "libicudata.so", false) + loader.loadAndCreateLink("libre2.so.9", "libre2.so", false) + loader.loadAndCreateLink("libicuuc.so.67", "libicuuc.so", false) + loader.loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false) + loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) + loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) + loader.loadAndCreateLink("libicui18n.so.67", "libicui18n.so", false) + loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false) + loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) + loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false) + loader.loadAndCreateLink("libssh2.so.1", "libssh2.so", false) + loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false) + loader.loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false) + loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) + loader.loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false) + loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) + loader.loadAndCreateLink("libglog.so.0", "libglog.so", false) + loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) + loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) + loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false) + loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala index 128c8eaa2aef..8e24ef0bad52 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderDebian12.scala @@ -20,39 +20,36 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderDebian12 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader - .newTransaction() - .loadAndCreateLink("libcrypto.so.3", "libcrypto.so", false) - .loadAndCreateLink("libkrb5support.so.0", "libkrb5support.so", false) - .loadAndCreateLink("libssl.so.3", "libssl.so", false) - .loadAndCreateLink("libicudata.so.72", "libicudata.so", false) - .loadAndCreateLink("libk5crypto.so.3", "libk5crypto.so", false) - .loadAndCreateLink("libkeyutils.so.1", "libkeyutils.so", false) - .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) - .loadAndCreateLink("libthrift-0.17.0.so", "libthrift.so", false) - .loadAndCreateLink("libicuuc.so.72", "libicuuc.so", false) - .loadAndCreateLink("libkrb5.so.3", "libkrb5.so", false) - .loadAndCreateLink("liblber-2.5.so.0", "liblber-2.4.so", false) - .loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) - .loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) - .loadAndCreateLink("libicui18n.so.72", "libicui18n.so", false) - .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - .loadAndCreateLink("libunwind.so.8", "libunwind.so", false) - .loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) - .loadAndCreateLink("librtmp.so.1", "librtmp.so", false) - .loadAndCreateLink("libssh2.so.1", "libssh2.so", false) - .loadAndCreateLink("libpsl.so.5", "libpsl.so", false) - .loadAndCreateLink("libgssapi_krb5.so.2", "libgssapi_krb5.so", false) - .loadAndCreateLink("libldap-2.5.so.0", "libldap_r-2.4.so", false) - .loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) - .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - .loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - .loadAndCreateLink("libglog.so.1", "libglog.so", false) - .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) - .loadAndCreateLink("libcurl.so.4", "libcurl.so", false) - .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - .commit() + loader.loadAndCreateLink("libcrypto.so.3", "libcrypto.so", false) + loader.loadAndCreateLink("libkrb5support.so.0", "libkrb5support.so", false) + loader.loadAndCreateLink("libssl.so.3", "libssl.so", false) + loader.loadAndCreateLink("libicudata.so.72", "libicudata.so", false) + loader.loadAndCreateLink("libk5crypto.so.3", "libk5crypto.so", false) + loader.loadAndCreateLink("libkeyutils.so.1", "libkeyutils.so", false) + loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) + loader.loadAndCreateLink("libthrift-0.17.0.so", "libthrift.so", false) + loader.loadAndCreateLink("libicuuc.so.72", "libicuuc.so", false) + loader.loadAndCreateLink("libkrb5.so.3", "libkrb5.so", false) + loader.loadAndCreateLink("liblber-2.5.so.0", "liblber-2.4.so", false) + loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) + loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) + loader.loadAndCreateLink("libicui18n.so.72", "libicui18n.so", false) + loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false) + loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) + loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false) + loader.loadAndCreateLink("libssh2.so.1", "libssh2.so", false) + loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false) + loader.loadAndCreateLink("libgssapi_krb5.so.2", "libgssapi_krb5.so", false) + loader.loadAndCreateLink("libldap-2.5.so.0", "libldap_r-2.4.so", false) + loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) + loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) + loader.loadAndCreateLink("libglog.so.1", "libglog.so", false) + loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) + loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false) + loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala index 18f2e6cfbeb3..a03a0a7e87c8 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2004.scala @@ -20,49 +20,46 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderUbuntu2004 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader - .newTransaction() - .loadAndCreateLink("libroken.so.18", "libroken.so", false) - .loadAndCreateLink("libasn1.so.8", "libasn1.so", false) - .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - .loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) - .loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) - .loadAndCreateLink("libpsl.so.5", "libpsl.so", false) - .loadAndCreateLink("libcrypto.so.1.1", "libcrypto.so", false) - .loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) - .loadAndCreateLink("libnettle.so.7", "libnettle.so", false) - .loadAndCreateLink("libhogweed.so.5", "libhogweed.so", false) - .loadAndCreateLink("librtmp.so.1", "librtmp.so", false) - .loadAndCreateLink("libssh.so.4", "libssh.so", false) - .loadAndCreateLink("libssl.so.1.1", "libssl.so", false) - .loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false) - .loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) - .loadAndCreateLink("libwind.so.0", "libwind.so", false) - .loadAndCreateLink("libheimbase.so.1", "libheimbase.so", false) - .loadAndCreateLink("libhcrypto.so.4", "libhcrypto.so", false) - .loadAndCreateLink("libhx509.so.5", "libhx509.so", false) - .loadAndCreateLink("libkrb5.so.26", "libkrb5.so", false) - .loadAndCreateLink("libheimntlm.so.0", "libheimntlm.so", false) - .loadAndCreateLink("libgssapi.so.3", "libgssapi.so", false) - .loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false) - .loadAndCreateLink("libcurl.so.4", "libcurl.so", false) - .loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) - .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - .loadAndCreateLink("libunwind.so.8", "libunwind.so", false) - .loadAndCreateLink("libglog.so.0", "libglog.so", false) - .loadAndCreateLink("libidn.so.11", "libidn.so", false) - .loadAndCreateLink("libntlm.so.0", "libntlm.so", false) - .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) - .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libicudata.so.66", "libicudata.so", false) - .loadAndCreateLink("libicuuc.so.66", "libicuuc.so", false) - .loadAndCreateLink("libxml2.so.2", "libxml2.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - .loadAndCreateLink("libre2.so.5", "libre2.so", false) - .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) - .loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false) - .commit() + loader.loadAndCreateLink("libroken.so.18", "libroken.so", false) + loader.loadAndCreateLink("libasn1.so.8", "libasn1.so", false) + loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + loader.loadAndCreateLink("libbrotlicommon.so.1", "libbrotlicommon.so", false) + loader.loadAndCreateLink("libbrotlidec.so.1", "libbrotlidec.so", false) + loader.loadAndCreateLink("libpsl.so.5", "libpsl.so", false) + loader.loadAndCreateLink("libcrypto.so.1.1", "libcrypto.so", false) + loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) + loader.loadAndCreateLink("libnettle.so.7", "libnettle.so", false) + loader.loadAndCreateLink("libhogweed.so.5", "libhogweed.so", false) + loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false) + loader.loadAndCreateLink("libssh.so.4", "libssh.so", false) + loader.loadAndCreateLink("libssl.so.1.1", "libssl.so", false) + loader.loadAndCreateLink("liblber-2.4.so.2", "liblber-2.4.so", false) + loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) + loader.loadAndCreateLink("libwind.so.0", "libwind.so", false) + loader.loadAndCreateLink("libheimbase.so.1", "libheimbase.so", false) + loader.loadAndCreateLink("libhcrypto.so.4", "libhcrypto.so", false) + loader.loadAndCreateLink("libhx509.so.5", "libhx509.so", false) + loader.loadAndCreateLink("libkrb5.so.26", "libkrb5.so", false) + loader.loadAndCreateLink("libheimntlm.so.0", "libheimntlm.so", false) + loader.loadAndCreateLink("libgssapi.so.3", "libgssapi.so", false) + loader.loadAndCreateLink("libldap_r-2.4.so.2", "libldap_r-2.4.so", false) + loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false) + loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) + loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) + loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false) + loader.loadAndCreateLink("libglog.so.0", "libglog.so", false) + loader.loadAndCreateLink("libidn.so.11", "libidn.so", false) + loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false) + loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) + loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + loader.loadAndCreateLink("libicudata.so.66", "libicudata.so", false) + loader.loadAndCreateLink("libicuuc.so.66", "libicuuc.so", false) + loader.loadAndCreateLink("libxml2.so.2", "libxml2.so", false) + loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + loader.loadAndCreateLink("libre2.so.5", "libre2.so", false) + loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) + loader.loadAndCreateLink("libthrift-0.13.0.so", "libthrift.so", false) } } diff --git a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala index b23105b7dce0..4bb2b59b33ea 100755 --- a/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/utils/SharedLibraryLoaderUbuntu2204.scala @@ -20,34 +20,31 @@ import org.apache.gluten.vectorized.JniLibLoader class SharedLibraryLoaderUbuntu2204 extends SharedLibraryLoader { override def loadLib(loader: JniLibLoader): Unit = { - loader - .newTransaction() - .loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) - .loadAndCreateLink("libicudata.so.70", "libicudata.so", false) - .loadAndCreateLink("libicuuc.so.70", "libicuuc.so", false) - .loadAndCreateLink("libicui18n.so.70", "libicui18n.so", false) - .loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) - .loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) - .loadAndCreateLink("librtmp.so.1", "librtmp.so", false) - .loadAndCreateLink("libssh.so.4", "libssh.so", false) - .loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) - .loadAndCreateLink("liblber-2.5.so.0", "liblber-2.5.so", false) - .loadAndCreateLink("libldap-2.5.so.0", "libldap-2.5.so", false) - .loadAndCreateLink("libcurl.so.4", "libcurl.so", false) - .loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) - .loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) - .loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) - .loadAndCreateLink("libunwind.so.8", "libunwind.so", false) - .loadAndCreateLink("libglog.so.0", "libglog.so", false) - .loadAndCreateLink("libidn.so.12", "libidn.so", false) - .loadAndCreateLink("libntlm.so.0", "libntlm.so", false) - .loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) - .loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) - .loadAndCreateLink("libxml2.so.2", "libxml2.so", false) - .loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) - .loadAndCreateLink("libre2.so.9", "libre2.so", false) - .loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) - .loadAndCreateLink("libthrift-0.16.0.so", "libthrift.so", false) - .commit() + loader.loadAndCreateLink("libboost_context.so.1.84.0", "libboost_context.so", false) + loader.loadAndCreateLink("libicudata.so.70", "libicudata.so", false) + loader.loadAndCreateLink("libicuuc.so.70", "libicuuc.so", false) + loader.loadAndCreateLink("libicui18n.so.70", "libicui18n.so", false) + loader.loadAndCreateLink("libboost_regex.so.1.84.0", "libboost_regex.so", false) + loader.loadAndCreateLink("libnghttp2.so.14", "libnghttp2.so", false) + loader.loadAndCreateLink("librtmp.so.1", "librtmp.so", false) + loader.loadAndCreateLink("libssh.so.4", "libssh.so", false) + loader.loadAndCreateLink("libsasl2.so.2", "libsasl2.so", false) + loader.loadAndCreateLink("liblber-2.5.so.0", "liblber-2.5.so", false) + loader.loadAndCreateLink("libldap-2.5.so.0", "libldap-2.5.so", false) + loader.loadAndCreateLink("libcurl.so.4", "libcurl.so", false) + loader.loadAndCreateLink("libdouble-conversion.so.3", "libdouble-conversion.so", false) + loader.loadAndCreateLink("libevent-2.1.so.7", "libevent-2.1.so", false) + loader.loadAndCreateLink("libgflags.so.2.2", "libgflags.so", false) + loader.loadAndCreateLink("libunwind.so.8", "libunwind.so", false) + loader.loadAndCreateLink("libglog.so.0", "libglog.so", false) + loader.loadAndCreateLink("libidn.so.12", "libidn.so", false) + loader.loadAndCreateLink("libntlm.so.0", "libntlm.so", false) + loader.loadAndCreateLink("libgsasl.so.7", "libgsasl.so", false) + loader.loadAndCreateLink("libprotobuf.so.32", "libprotobuf.so", false) + loader.loadAndCreateLink("libxml2.so.2", "libxml2.so", false) + loader.loadAndCreateLink("libhdfs3.so.1", "libhdfs3.so", false) + loader.loadAndCreateLink("libre2.so.9", "libre2.so", false) + loader.loadAndCreateLink("libsnappy.so.1", "libsnappy.so", false) + loader.loadAndCreateLink("libthrift-0.16.0.so", "libthrift.so", false) } } diff --git a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java index b1feb1e5baeb..37cd29649c5c 100644 --- a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java +++ b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java @@ -35,30 +35,20 @@ import java.util.Collections; import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.Set; import java.util.Vector; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.stream.Collectors; -import java.util.stream.Stream; import scala.runtime.BoxedUnit; -/** - * LoadXXX methods in the utility prevents reloading of a library internally. It's not necessary for - * caller to manage a loaded library list. - */ public class JniLibLoader { private static final Logger LOG = LoggerFactory.getLogger(JniLibLoader.class); - private static final Set LOADED_LIBRARY_PATHS = new HashSet<>(); - private static final Set REQUIRE_UNLOAD_LIBRARY_PATHS = new LinkedHashSet<>(); + private static final Set LOADED_LIBRARY_PATHS = + Collections.synchronizedSet(new HashSet<>()); + private static final Set REQUIRE_UNLOAD_LIBRARY_PATHS = + Collections.synchronizedSet(new LinkedHashSet<>()); static { GlutenShutdownManager.addHookForLibUnloading( @@ -69,15 +59,17 @@ public class JniLibLoader { } private final String workDir; - private final Set loadedLibraries = new HashSet<>(); - private final Lock sync = new ReentrantLock(); + private final Set loadedLibraries = Collections.synchronizedSet(new HashSet<>()); JniLibLoader(String workDir) { this.workDir = workDir; } - public static synchronized void forceUnloadAll() { - List loaded = new ArrayList<>(REQUIRE_UNLOAD_LIBRARY_PATHS); + public static void forceUnloadAll() { + List loaded; + synchronized (REQUIRE_UNLOAD_LIBRARY_PATHS) { + loaded = new ArrayList<>(REQUIRE_UNLOAD_LIBRARY_PATHS); + } Collections.reverse(loaded); // use reversed order to unload loaded.forEach(JniLibLoader::unloadFromPath); } @@ -95,21 +87,25 @@ private static String toRealPath(String libPath) { } } - private static synchronized void loadFromPath0(String libPath, boolean requireUnload) { + private static void loadFromPath0(String libPath, boolean requireUnload) { libPath = toRealPath(libPath); - if (LOADED_LIBRARY_PATHS.contains(libPath)) { - LOG.debug("Library in path {} has already been loaded, skipping", libPath); - } else { - System.load(libPath); - LOADED_LIBRARY_PATHS.add(libPath); - LOG.info("Library {} has been loaded using path-loading method", libPath); + synchronized (LOADED_LIBRARY_PATHS) { + if (LOADED_LIBRARY_PATHS.contains(libPath)) { + LOG.debug("Library in path {} has already been loaded, skipping", libPath); + } else { + System.load(libPath); + LOADED_LIBRARY_PATHS.add(libPath); + LOG.info("Library {} has been loaded using path-loading method", libPath); + } } if (requireUnload) { - REQUIRE_UNLOAD_LIBRARY_PATHS.add(libPath); + synchronized (REQUIRE_UNLOAD_LIBRARY_PATHS) { + REQUIRE_UNLOAD_LIBRARY_PATHS.add(libPath); + } } } - public static void loadFromPath(String libPath, boolean requireUnload) { + public static synchronized void loadFromPath(String libPath, boolean requireUnload) { final File file = new File(libPath); if (!file.isFile() || !file.exists()) { throw new GlutenException("library at path: " + libPath + " is not a file or does not exist"); @@ -117,55 +113,42 @@ public static void loadFromPath(String libPath, boolean requireUnload) { loadFromPath0(file.getAbsolutePath(), requireUnload); } - public void mapAndLoad(String unmappedLibName, boolean requireUnload) { - newTransaction().mapAndLoad(unmappedLibName, requireUnload).commit(); - } - - public void load(String libName, boolean requireUnload) { - newTransaction().load(libName, requireUnload).commit(); - } - - public void loadAndCreateLink(String libName, String linkName, boolean requireUnload) { - newTransaction().loadAndCreateLink(libName, linkName, requireUnload).commit(); - } - - public JniLoadTransaction newTransaction() { - return new JniLoadTransaction(); - } - - public static synchronized void unloadFromPath(String libPath) { - if (!LOADED_LIBRARY_PATHS.remove(libPath)) { - LOG.warn("Library {} was not loaded or alreay unloaded:", libPath); - return; + public static void unloadFromPath(String libPath) { + synchronized (LOADED_LIBRARY_PATHS) { + if (!LOADED_LIBRARY_PATHS.remove(libPath)) { + LOG.warn("Library {} was not loaded or already unloaded:", libPath); + return; + } } - LOG.info("Starting unload library path: {} ", libPath); - REQUIRE_UNLOAD_LIBRARY_PATHS.remove(libPath); - + synchronized (REQUIRE_UNLOAD_LIBRARY_PATHS) { + REQUIRE_UNLOAD_LIBRARY_PATHS.remove(libPath); + } try { ClassLoader classLoader = JniLibLoader.class.getClassLoader(); Field field = ClassLoader.class.getDeclaredField("nativeLibraries"); field.setAccessible(true); Vector libs = (Vector) field.get(classLoader); - Iterator it = libs.iterator(); - while (it.hasNext()) { - Object object = it.next(); - Field[] fs = object.getClass().getDeclaredFields(); - for (int k = 0; k < fs.length; k++) { - if (fs[k].getName().equals("name")) { - fs[k].setAccessible(true); - - String verbosePath = fs[k].get(object).toString(); - File verboseFile = new File(verbosePath); - String verboseFileName = verboseFile.getName(); - File libFile = new File(libPath); - String libFileName = libFile.getName(); - - if (verboseFileName.equals(libFileName)) { - LOG.info("Finalizing library file: {}", libFileName); - Method finalize = object.getClass().getDeclaredMethod("finalize"); - finalize.setAccessible(true); - finalize.invoke(object); + synchronized (libs) { + Iterator it = libs.iterator(); + while (it.hasNext()) { + Object object = it.next(); + Field[] fs = object.getClass().getDeclaredFields(); + for (int k = 0; k < fs.length; k++) { + if (fs[k].getName().equals("name")) { + fs[k].setAccessible(true); + String verbosePath = fs[k].get(object).toString(); + File verboseFile = new File(verbosePath); + String verboseFileName = verboseFile.getName(); + File libFile = new File(libPath); + String libFileName = libFile.getName(); + + if (verboseFileName.equals(libFileName)) { + LOG.info("Finalizing library file: {}", libFileName); + Method finalize = object.getClass().getDeclaredMethod("finalize"); + finalize.setAccessible(true); + finalize.invoke(object); + } } } } @@ -175,160 +158,77 @@ public static synchronized void unloadFromPath(String libPath) { } } - private static final class LoadRequest { - final String libName; - final String linkName; - final boolean requireUnload; - - private LoadRequest(String libName, String linkName, boolean requireUnload) { - this.libName = libName; - this.linkName = linkName; - this.requireUnload = requireUnload; - } - } - - private static final class LoadAction { - final String libName; - final String linkName; - final boolean requireUnload; - final File file; - - private LoadAction(String libName, String linkName, boolean requireUnload, File file) { - this.libName = libName; - this.linkName = linkName; - this.requireUnload = requireUnload; - this.file = file; - } - - public boolean requireLinking() { - return !Objects.isNull(linkName); - } - } - - public class JniLoadTransaction { - private final AtomicBoolean finished = new AtomicBoolean(false); - private final Map toLoad = new LinkedHashMap<>(); // ordered - - private JniLoadTransaction() { - JniLibLoader.this.sync.lock(); - } - - public JniLoadTransaction mapAndLoad(String unmappedLibName, boolean requireUnload) { + public void mapAndLoad(String unmappedLibName, boolean requireUnload) { + synchronized (loadedLibraries) { try { final String mappedLibName = System.mapLibraryName(unmappedLibName); load(mappedLibName, requireUnload); - return this; } catch (Exception e) { - abort(); throw new GlutenException(e); } } + } - public JniLoadTransaction load(String libName, boolean requireUnload) { + public void load(String libName, boolean requireUnload) { + synchronized (loadedLibraries) { try { - toLoad.put(libName, new LoadRequest(libName, null, requireUnload)); - return this; - } catch (Exception e) { - abort(); + if (loadedLibraries.contains(libName)) { + LOG.debug("Library {} has already been loaded, skipping", libName); + return; + } + File file = moveToWorkDir(workDir, libName); + loadWithLink(file.getAbsolutePath(), null, requireUnload); + loadedLibraries.add(libName); + LOG.info("Successfully loaded library {}", libName); + } catch (IOException e) { throw new GlutenException(e); } } + } - public JniLoadTransaction loadAndCreateLink( - String libName, String linkName, boolean requireUnload) { + public void loadAndCreateLink(String libName, String linkName, boolean requireUnload) { + synchronized (loadedLibraries) { try { - toLoad.put(libName, new LoadRequest(libName, linkName, requireUnload)); - return this; - } catch (Exception e) { - abort(); + if (loadedLibraries.contains(libName)) { + LOG.debug("Library {} has already been loaded, skipping", libName); + } + File file = moveToWorkDir(workDir, System.mapLibraryName(libName)); + loadWithLink(file.getAbsolutePath(), linkName, requireUnload); + loadedLibraries.add(libName); + LOG.info("Successfully loaded library {}", libName); + } catch (IOException e) { throw new GlutenException(e); } } + } - public void commit() { - try { - terminate(); - toLoad.entrySet().stream() - .flatMap( - e -> { - try { - final LoadRequest req = e.getValue(); - if (loadedLibraries.contains(req.libName)) { - LOG.debug("Library {} has already been loaded, skipping", req.libName); - return Stream.empty(); - } - // load only libraries not loaded yet - final File file = moveToWorkDir(workDir, req.libName); - return Stream.of( - new LoadAction(req.libName, req.linkName, req.requireUnload, file)); - } catch (IOException ex) { - throw new GlutenException(ex); - } - }) - .collect(Collectors.toList()) - .forEach( - e -> { - try { - LOG.info("Trying to load library {}", e.libName); - loadWithLink(workDir, e); - loadedLibraries.add(e.libName); - LOG.info("Successfully loaded library {}", e.libName); - } catch (Exception ex) { - throw new GlutenException(ex); - } - }); - } finally { - JniLibLoader.this.sync.unlock(); - } - } - - public void abort() { - try { - terminate(); - // do nothing as of now - } finally { - JniLibLoader.this.sync.unlock(); - } - } - - private void terminate() { - if (!finished.compareAndSet(false, true)) { - throw new IllegalStateException(); - } + private File moveToWorkDir(String workDir, String libraryToLoad) throws IOException { + // final File temp = File.createTempFile(workDir, libraryToLoad); + final Path libPath = Paths.get(workDir + "/" + libraryToLoad); + if (Files.exists(libPath)) { + Files.delete(libPath); } - - private File moveToWorkDir(String workDir, String libraryToLoad) throws IOException { - // final File temp = File.createTempFile(workDir, libraryToLoad); - final Path libPath = Paths.get(workDir + "/" + libraryToLoad); - if (Files.exists(libPath)) { - Files.delete(libPath); + final File temp = new File(workDir + "/" + libraryToLoad); + try (InputStream is = JniLibLoader.class.getClassLoader().getResourceAsStream(libraryToLoad)) { + if (is == null) { + throw new FileNotFoundException(libraryToLoad); } - final File temp = new File(workDir + "/" + libraryToLoad); - try (InputStream is = - JniLibLoader.class.getClassLoader().getResourceAsStream(libraryToLoad)) { - if (is == null) { - throw new FileNotFoundException(libraryToLoad); - } - try { - Files.copy(is, temp.toPath()); - } catch (Exception e) { - throw new GlutenException(e); - } + try { + Files.copy(is, temp.toPath()); + } catch (Exception e) { + throw new GlutenException(e); } - return temp; } + return temp; + } - private void loadWithLink(String workDir, LoadAction req) throws IOException { - String libPath = req.file.getAbsolutePath(); - loadFromPath0(libPath, req.requireUnload); - LOG.info("Library {} has been loaded", libPath); - if (!req.requireLinking()) { - LOG.debug("Symbolic link not required for library {}, skipping", libPath); - return; - } - // create link - Path target = Paths.get(req.file.getPath()); - Path link = Paths.get(workDir, req.linkName); + private void loadWithLink(String libPath, String linkName, boolean requireUnload) + throws IOException { + loadFromPath0(libPath, requireUnload); + LOG.info("Library {} has been loaded", libPath); + if (linkName != null) { + Path target = Paths.get(libPath); + Path link = Paths.get(workDir, linkName); if (Files.exists(link)) { LOG.info("Symbolic link already exists for library {}, deleting", libPath); Files.delete(link);