diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml
index 0fe8f5f6fd8e3..e24f5f62c8421 100755
--- a/backends-velox/pom.xml
+++ b/backends-velox/pom.xml
@@ -186,6 +186,7 @@
     <resources>
       <resource>
         <directory>${cpp.releases.dir}</directory>
+        <targetPath>org/apache/gluten/${platform}/${arch}</targetPath>
       </resource>
       <resource>
         <directory>${resource.dir}</directory>
diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
index 2cfc4e9a9099d..6b42919f0b003 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxListenerApi.scala
@@ -116,8 +116,8 @@ class VeloxListenerApi extends ListenerApi with Logging {
       JniLibLoader.loadFromPath(libPath, false)
     } else {
       val baseLibName = conf.get(GlutenConfig.GLUTEN_LIB_NAME, "gluten")
-      loader.mapAndLoad(baseLibName, false)
-      loader.mapAndLoad(VeloxBackend.BACKEND_NAME, false)
+      loader.mapAndLoad(java.util.Optional.of(baseLibPackage), baseLibName, false)
+      loader.mapAndLoad(java.util.Optional.of(baseLibPackage), VeloxBackend.BACKEND_NAME, false)
     }
 
     // Initial native backend with configurations.
@@ -130,6 +130,18 @@ class VeloxListenerApi extends ListenerApi with Logging {
     GlutenRowSplitter.setInstance(new VeloxRowSplitter())
   }
 
+  private lazy val baseLibPackage: String = {
+    val osName = System.getProperty("os.name") match {
+      case n if n.contains("Linux") => "linux"
+      case n if n.contains("Mac") => "darwin"
+      case _ =>
+        // Default to linux
+        "linux"
+    }
+    val arch = System.getProperty("os.arch")
+    s"org/apache/gluten/$osName/$arch"
+  }
+
   private def shutdown(): Unit = {
     // TODO shutdown implementation in velox to release resources
   }
diff --git a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java
index 37cd29649c5c1..9fa9f603e397e 100644
--- a/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java
+++ b/gluten-core/src/main/java/org/apache/gluten/vectorized/JniLibLoader.java
@@ -37,6 +37,7 @@
 import java.util.Iterator;
 import java.util.LinkedHashSet;
 import java.util.List;
+import java.util.Optional;
 import java.util.Set;
 import java.util.Vector;
 
@@ -159,10 +160,16 @@ public static void unloadFromPath(String libPath) {
   }
 
   public void mapAndLoad(String unmappedLibName, boolean requireUnload) {
+    mapAndLoad(Optional.empty(), unmappedLibName, requireUnload);
+  }
+
+  public void mapAndLoad(
+      Optional<String> packagePath, String unmappedLibName, boolean requireUnload) {
     synchronized (loadedLibraries) {
       try {
         final String mappedLibName = System.mapLibraryName(unmappedLibName);
-        load(mappedLibName, requireUnload);
+        String resourceName = packagePath.map(p -> p + "/" + mappedLibName).orElse(mappedLibName);
+        load(resourceName, requireUnload);
       } catch (Exception e) {
         throw new GlutenException(e);
       }
diff --git a/pom.xml b/pom.xml
index 4fb327f9f7ddd..261de2e25b560 100644
--- a/pom.xml
+++ b/pom.xml
@@ -482,6 +482,60 @@
 	</plugins>
       </build>
     </profile>
+
+    <!-- Profiles for different platforms -->
+    <profile>
+      <id>Darwin-x86</id>
+      <activation>
+        <os>
+          <family>mac</family>
+          <arch>x86</arch>
+        </os>
+      </activation>
+      <properties>
+        <platform>darwin</platform>
+        <arch>x86_64</arch>
+      </properties>
+    </profile>
+    <profile>
+      <id>Darwin-aarch64</id>
+      <activation>
+        <os>
+          <family>mac</family>
+          <arch>aarch64</arch>
+        </os>
+      </activation>
+      <properties>
+        <platform>darwin</platform>
+        <arch>aarch64</arch>
+      </properties>
+    </profile>
+    <profile>
+      <id>Linux-amd64</id>
+      <activation>
+        <os>
+          <family>Linux</family>
+          <arch>amd64</arch>
+        </os>
+      </activation>
+      <properties>
+        <platform>linux</platform>
+        <arch>amd64</arch>
+      </properties>
+    </profile>
+    <profile>
+      <id>Linux-aarch64</id>
+      <activation>
+        <os>
+          <family>Linux</family>
+          <arch>aarch64</arch>
+        </os>
+      </activation>
+      <properties>
+        <platform>linux</platform>
+        <arch>aarch64</arch>
+      </properties>
+    </profile>
   </profiles>
 
   <dependencyManagement>