diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml
index 9349c3c0923c..240b2218641c 100755
--- a/backends-velox/pom.xml
+++ b/backends-velox/pom.xml
@@ -57,6 +57,25 @@
         </dependency>
       </dependencies>
     </profile>
+    <profile>
+      <id>uniffle</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <dependencies>
+        <dependency>
+          <groupId>org.apache.gluten</groupId>
+          <artifactId>gluten-uniffle</artifactId>
+          <version>${project.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>org.apache.uniffle</groupId>
+          <artifactId>rss-client-spark${spark.major.version}-shaded</artifactId>
+          <version>${uniffle.version}</version>
+          <scope>provided</scope>
+        </dependency>
+      </dependencies>
+    </profile>
     <profile>
       <id>iceberg</id>
       <activation>
diff --git a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/gluten/uniffle/UniffleShuffleManager.java b/backends-velox/src-uniffle/main/java/org/apache/spark/shuffle/gluten/uniffle/UniffleShuffleManager.java
similarity index 100%
rename from gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/gluten/uniffle/UniffleShuffleManager.java
rename to backends-velox/src-uniffle/main/java/org/apache/spark/shuffle/gluten/uniffle/UniffleShuffleManager.java
diff --git a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java b/backends-velox/src-uniffle/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java
similarity index 100%
rename from gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java
rename to backends-velox/src-uniffle/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java
diff --git a/gluten-uniffle/velox/src/main/scala/org/apache/spark/shuffle/writer/PartitionPusher.scala b/backends-velox/src-uniffle/main/scala/org/apache/spark/shuffle/writer/PartitionPusher.scala
similarity index 100%
rename from gluten-uniffle/velox/src/main/scala/org/apache/spark/shuffle/writer/PartitionPusher.scala
rename to backends-velox/src-uniffle/main/scala/org/apache/spark/shuffle/writer/PartitionPusher.scala
diff --git a/gluten-uniffle/.gitkeep b/gluten-uniffle/.gitkeep
new file mode 100644
index 000000000000..f2d1254d2735
--- /dev/null
+++ b/gluten-uniffle/.gitkeep
@@ -0,0 +1 @@
+The module is kept for adding common code shared by backends for Uniffle support in Gluten.
diff --git a/gluten-uniffle/package/pom.xml b/gluten-uniffle/package/pom.xml
deleted file mode 100644
index e49748e7c8e9..000000000000
--- a/gluten-uniffle/package/pom.xml
+++ /dev/null
@@ -1,29 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <parent>
-    <artifactId>gluten-uniffle</artifactId>
-    <groupId>org.apache.gluten</groupId>
-    <version>1.3.0-SNAPSHOT</version>
-    <relativePath>../pom.xml</relativePath>
-  </parent>
-  <modelVersion>4.0.0</modelVersion>
-
-  <artifactId>gluten-uniffle-package</artifactId>
-  <packaging>jar</packaging>
-  <name>Gluten Uniffle Package</name>
-
-  <profiles>
-    <profile>
-      <id>backends-velox</id>
-      <dependencies>
-        <dependency>
-          <groupId>org.apache.gluten</groupId>
-          <artifactId>gluten-uniffle-velox</artifactId>
-          <version>${project.version}</version>
-        </dependency>
-      </dependencies>
-    </profile>
-  </profiles>
-</project>
diff --git a/gluten-uniffle/pom.xml b/gluten-uniffle/pom.xml
index b7fe4c2e4268..efc8ce6555c5 100644
--- a/gluten-uniffle/pom.xml
+++ b/gluten-uniffle/pom.xml
@@ -11,7 +11,7 @@
   <modelVersion>4.0.0</modelVersion>
 
   <artifactId>gluten-uniffle</artifactId>
-  <packaging>pom</packaging>
+  <packaging>jar</packaging>
   <name>Gluten Uniffle</name>
 
   <dependencies>
@@ -75,15 +75,4 @@
       </plugin>
     </plugins>
   </build>
-  <profiles>
-    <profile>
-      <id>backends-velox</id>
-      <properties>
-      </properties>
-      <modules>
-        <module>velox</module>
-        <module>package</module>
-      </modules>
-    </profile>
-  </profiles>
 </project>
diff --git a/gluten-uniffle/velox/pom.xml b/gluten-uniffle/velox/pom.xml
deleted file mode 100755
index ab730674fbb3..000000000000
--- a/gluten-uniffle/velox/pom.xml
+++ /dev/null
@@ -1,62 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xmlns="http://maven.apache.org/POM/4.0.0"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-  <parent>
-    <artifactId>gluten-uniffle</artifactId>
-    <groupId>org.apache.gluten</groupId>
-    <version>1.3.0-SNAPSHOT</version>
-    <relativePath>../pom.xml</relativePath>
-  </parent>
-  <modelVersion>4.0.0</modelVersion>
-
-  <artifactId>gluten-uniffle-velox</artifactId>
-  <packaging>jar</packaging>
-  <name>Gluten Uniffle Velox</name>
-
-  <dependencies>
-    <dependency>
-      <groupId>org.apache.gluten</groupId>
-      <artifactId>backends-velox</artifactId>
-      <version>${project.version}</version>
-      <scope>provided</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.apache.gluten</groupId>
-      <artifactId>gluten-arrow</artifactId>
-      <version>${project.version}</version>
-      <scope>provided</scope>
-    </dependency>
-  </dependencies>
-
-  <build>
-    <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
-    <testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
-    <plugins>
-      <plugin>
-        <groupId>net.alchim31.maven</groupId>
-        <artifactId>scala-maven-plugin</artifactId>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-compiler-plugin</artifactId>
-      </plugin>
-      <plugin>
-        <groupId>org.scalastyle</groupId>
-        <artifactId>scalastyle-maven-plugin</artifactId>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-checkstyle-plugin</artifactId>
-      </plugin>
-      <plugin>
-        <groupId>com.diffplug.spotless</groupId>
-        <artifactId>spotless-maven-plugin</artifactId>
-      </plugin>
-      <plugin>
-        <groupId>org.apache.maven.plugins</groupId>
-        <artifactId>maven-jar-plugin</artifactId>
-      </plugin>
-    </plugins>
-  </build>
-</project>
diff --git a/package/pom.xml b/package/pom.xml
index 230f79d0942a..b9c114181bcd 100644
--- a/package/pom.xml
+++ b/package/pom.xml
@@ -78,7 +78,7 @@
       <dependencies>
         <dependency>
           <groupId>org.apache.gluten</groupId>
-          <artifactId>gluten-uniffle-package</artifactId>
+          <artifactId>gluten-uniffle</artifactId>
           <version>${project.version}</version>
         </dependency>
       </dependencies>
diff --git a/pom.xml b/pom.xml
index e6f3709c4cfe..4d704dc9b448 100644
--- a/pom.xml
+++ b/pom.xml
@@ -495,6 +495,70 @@
       <modules>
         <module>gluten-uniffle</module>
       </modules>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.codehaus.mojo</groupId>
+            <artifactId>build-helper-maven-plugin</artifactId>
+            <executions>
+              <execution>
+                <id>add-uniffle-sources</id>
+                <phase>generate-sources</phase>
+                <goals>
+                  <goal>add-source</goal>
+                </goals>
+                <configuration>
+                  <sources>
+                    <source>${project.basedir}/src-uniffle/main/scala</source>
+                    <source>${project.basedir}/src-uniffle/main/java</source>
+                  </sources>
+                </configuration>
+              </execution>
+              <execution>
+                <id>add-uniffle-resources</id>
+                <phase>generate-resources</phase>
+                <goals>
+                  <goal>add-resource</goal>
+                </goals>
+                <configuration>
+                  <resources>
+                    <resource>
+                      <directory>${project.basedir}/src-uniffle/main/resources</directory>
+                    </resource>
+                  </resources>
+                </configuration>
+              </execution>
+              <execution>
+                <id>add-uniffle-test-sources</id>
+                <phase>generate-test-sources</phase>
+                <goals>
+                  <goal>add-test-source</goal>
+                </goals>
+                <configuration>
+                  <sources>
+                    <source>${project.basedir}/src-uniffle/test/scala</source>
+                    <source>${project.basedir}/src-uniffle/test/java</source>
+                  </sources>
+                </configuration>
+              </execution>
+              <execution>
+                <id>add-uniffle-test-resources</id>
+                <phase>generate-test-resources</phase>
+                <goals>
+                  <goal>add-test-resource</goal>
+                </goals>
+                <configuration>
+                  <resources>
+                    <resource>
+                      <directory>${project.basedir}/src-uniffle/test/resources</directory>
+                    </resource>
+                  </resources>
+                </configuration>
+              </execution>
+            </executions>
+          </plugin>
+        </plugins>
+      </build>
     </profile>
     <profile>
       <id>delta</id>