Skip to content

Commit

Permalink
[CORE] Change domain name to org.apache.gluten (apache#5185)
Browse files Browse the repository at this point in the history
Change gluten domain name to org.apache.gluten. User needs to note the plugin configuration is affected.

--conf spark.plugins=org.apache.gluten.GlutenPlugin
  • Loading branch information
yma11 authored Apr 1, 2024
1 parent 8905dc1 commit cf252bc
Show file tree
Hide file tree
Showing 910 changed files with 2,668 additions and 2,650 deletions.
12 changes: 6 additions & 6 deletions .github/workflows/velox_be.yml.deprecated
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,8 @@ jobs:
run: |
$PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh '
cd /opt/gluten && \
mvn clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss -DargLine="-Dspark.test.home=/opt/spark322" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \
mvn test -Pspark-3.2 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest'
mvn clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss -DargLine="-Dspark.test.home=/opt/spark322" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \
mvn test -Pspark-3.2 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest'
# Cpp micro benchmarks will use generated files from unit test in backends-velox module.
- name: Run micro benchmarks
run: |
Expand Down Expand Up @@ -221,8 +221,8 @@ jobs:
- name: Build and Run unit test for Spark 3.3.1 (other tests)
run: |
$PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten && \
mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark331" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \
mvn test -Pspark-3.3 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest'
mvn clean install -Pspark-3.3 -Pbackends-velox -Prss -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark331" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \
mvn test -Pspark-3.3 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest'
- name: Copy golden files from container to host
if: failure()
run: |
Expand Down Expand Up @@ -304,8 +304,8 @@ jobs:
- name: Build and Run unit test for Spark 3.4.2 (other tests)
run: |
$PATH_TO_GLUTEN_TE/$OS_IMAGE_NAME/gha/gha-checkout/exec.sh 'cd /opt/gluten && \
mvn clean install -Pspark-3.4 -Pbackends-velox -Prss -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark342" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \
mvn test -Pspark-3.4 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest'
mvn clean install -Pspark-3.4 -Pbackends-velox -Prss -Piceberg -Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/spark342" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \
mvn test -Pspark-3.4 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest'
- name: Copy golden files from container to host
if: failure()
run: |
Expand Down
12 changes: 6 additions & 6 deletions .github/workflows/velox_docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -393,8 +393,8 @@ jobs:
export SPARK_SCALA_VERSION=2.12
export MAVEN_HOME=/usr/lib/maven
export PATH=${PATH}:${MAVEN_HOME}/bin
mvn -ntp clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \
mvn -ntp test -Pspark-3.2 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest
mvn -ntp clean install -Pspark-3.2 -Pspark-ut -Pbackends-velox -Prss -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark32/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \
mvn -ntp test -Pspark-3.2 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest
- name: Upload golden files
if: failure()
uses: actions/upload-artifact@v4
Expand Down Expand Up @@ -480,8 +480,8 @@ jobs:
export SPARK_SCALA_VERSION=2.12 && \
export MAVEN_HOME=/usr/lib/maven
export PATH=${PATH}:${MAVEN_HOME}/bin
mvn -ntp clean install -Pspark-3.3 -Pbackends-velox -Prss -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \
mvn -ntp test -Pspark-3.3 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest
mvn -ntp clean install -Pspark-3.3 -Pbackends-velox -Prss -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark33/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \
mvn -ntp test -Pspark-3.3 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest
- name: Upload golden files
if: failure()
uses: actions/upload-artifact@v4
Expand Down Expand Up @@ -568,8 +568,8 @@ jobs:
export SPARK_SCALA_VERSION=2.12 && \
export MAVEN_HOME=/usr/lib/maven
export PATH=${PATH}:${MAVEN_HOME}/bin
mvn -ntp clean install -Pspark-3.4 -Pbackends-velox -Prss -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,io.glutenproject.tags.UDFTest,io.glutenproject.tags.SkipTestTags && \
mvn -ntp test -Pspark-3.4 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=io.glutenproject.tags.UDFTest
mvn -ntp clean install -Pspark-3.4 -Pbackends-velox -Prss -Pspark-ut -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark34/spark_home/" -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags && \
mvn -ntp test -Pspark-3.4 -Pbackends-velox -DtagsToExclude=None -DtagsToInclude=org.apache.gluten.tags.UDFTest
- name: Upload golden files
if: failure()
uses: actions/upload-artifact@v4
Expand Down
2 changes: 1 addition & 1 deletion .scalafmt.conf
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ rewrite.rules = [AvoidInfix, Imports, RedundantBraces, SortModifiers]
# Imports
rewrite.imports.sort = scalastyle
rewrite.imports.groups = [
["io.glutenproject\\..*"],
["org.apache.gluten\\..*"],
["io.substrait.spark\\..*"],
["org.apache.spark\\..*"],
[".*"],
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ One way is to use released jar. Here is a simple example. Currently, only centos
```
spark-shell \
--master yarn --deploy-mode client \
--conf spark.plugins=io.glutenproject.GlutenPlugin \
--conf spark.plugins=org.apache.gluten.GlutenPlugin \
--conf spark.memory.offHeap.enabled=true \
--conf spark.memory.offHeap.size=20g \
--conf spark.shuffle.manager=org.apache.spark.shuffle.sort.ColumnarShuffleManager \
Expand All @@ -85,7 +85,7 @@ Alternatively, you can build gluten from source, then do some configurations to
export gluten_jar = /PATH/TO/GLUTEN/backends-velox/target/<gluten-jar>
spark-shell
--master yarn --deploy-mode client \
--conf spark.plugins=io.glutenproject.GlutenPlugin \
--conf spark.plugins=org.apache.gluten.GlutenPlugin \
--conf spark.memory.offHeap.enabled=true \
--conf spark.memory.offHeap.size=20g \
--conf spark.driver.extraClassPath=${gluten_jar} \
Expand Down
6 changes: 3 additions & 3 deletions backends-clickhouse/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>gluten-parent</artifactId>
<groupId>io.glutenproject</groupId>
<groupId>org.apache.gluten</groupId>
<version>1.2.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
Expand All @@ -15,7 +15,7 @@

<dependencies>
<dependency>
<groupId>io.glutenproject</groupId>
<groupId>org.apache.gluten</groupId>
<artifactId>gluten-core</artifactId>
<version>${project.version}</version>
<scope>compile</scope>
Expand All @@ -32,7 +32,7 @@
<version>${custom.protobuf.version}</version>
</dependency>
<dependency>
<groupId>io.glutenproject</groupId>
<groupId>org.apache.gluten</groupId>
<artifactId>gluten-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
*/
package org.apache.spark.sql.delta.commands

import io.glutenproject.expression.ConverterUtils
import org.apache.gluten.expression.ConverterUtils

import org.apache.spark.{TaskContext, TaskOutputFileAlreadyExistException}
import org.apache.spark.internal.Logging
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
*/
package org.apache.spark.sql.delta.commands

import io.glutenproject.expression.ConverterUtils
import org.apache.gluten.expression.ConverterUtils

import org.apache.spark.{TaskContext, TaskOutputFileAlreadyExistException}
import org.apache.spark.internal.Logging
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.execution;
package org.apache.gluten.execution;

import io.glutenproject.vectorized.CHColumnVector;
import io.glutenproject.vectorized.GeneralInIterator;
import org.apache.gluten.vectorized.CHColumnVector;
import org.apache.gluten.vectorized.GeneralInIterator;

import org.apache.spark.sql.vectorized.ColumnarBatch;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.execution;
package org.apache.gluten.execution;

import java.nio.ByteBuffer;
import java.nio.ByteOrder;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.memory.alloc;
package org.apache.gluten.memory.alloc;

import io.glutenproject.GlutenConfig;
import io.glutenproject.memory.SimpleMemoryUsageRecorder;
import io.glutenproject.memory.memtarget.MemoryTarget;
import org.apache.gluten.GlutenConfig;
import org.apache.gluten.memory.SimpleMemoryUsageRecorder;
import org.apache.gluten.memory.memtarget.MemoryTarget;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.memory.alloc;
package org.apache.gluten.memory.alloc;

/**
* Like {@link io.glutenproject.vectorized.NativePlanEvaluator}, this along with {@link
* Like {@link org.apache.gluten.vectorized.NativePlanEvaluator}, this along with {@link
* CHNativeMemoryAllocators}, as built-in toolkit for managing native memory allocations.
*/
public class CHNativeMemoryAllocator {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.memory.alloc;
package org.apache.gluten.memory.alloc;

import org.apache.spark.util.TaskResource;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.memory.alloc;
package org.apache.gluten.memory.alloc;

public class CHNativeMemoryAllocatorManagerImpl implements CHNativeMemoryAllocatorManager {
private final CHNativeMemoryAllocator managed;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.memory.alloc;
package org.apache.gluten.memory.alloc;

import io.glutenproject.memory.SimpleMemoryUsageRecorder;
import io.glutenproject.memory.memtarget.MemoryTargets;
import io.glutenproject.memory.memtarget.Spiller;
import org.apache.gluten.memory.SimpleMemoryUsageRecorder;
import org.apache.gluten.memory.memtarget.MemoryTargets;
import org.apache.gluten.memory.memtarget.Spiller;

import org.apache.spark.memory.TaskMemoryManager;
import org.apache.spark.util.TaskResources;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.memory.alloc;
package org.apache.gluten.memory.alloc;

public interface CHReservationListener {
CHReservationListener NOOP =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.metrics;
package org.apache.gluten.metrics;

import com.fasterxml.jackson.annotation.JsonProperty;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.metrics;
package org.apache.gluten.metrics;

import com.fasterxml.jackson.annotation.JsonProperty;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.metrics;
package org.apache.gluten.metrics;

import com.fasterxml.jackson.annotation.JsonProperty;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.metrics;
package org.apache.gluten.metrics;

import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.metrics;
package org.apache.gluten.metrics;

import io.glutenproject.substrait.AggregationParams;
import io.glutenproject.substrait.JoinParams;
import org.apache.gluten.substrait.AggregationParams;
import org.apache.gluten.substrait.JoinParams;

import java.util.List;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.utils;
package org.apache.gluten.utils;

import io.glutenproject.exception.GlutenException;
import org.apache.gluten.exception.GlutenException;

import org.apache.spark.SparkEnv;
import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.vectorized;
package org.apache.gluten.vectorized;

import io.glutenproject.metrics.IMetrics;
import org.apache.gluten.metrics.IMetrics;

import org.apache.spark.sql.execution.utils.CHExecUtil;
import org.apache.spark.sql.vectorized.ColumnVector;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.vectorized;
package org.apache.gluten.vectorized;

import org.apache.spark.sql.execution.metric.SQLMetric;
import org.apache.spark.sql.vectorized.ColumnarBatch;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.vectorized;
package org.apache.gluten.vectorized;

import org.apache.spark.sql.vectorized.ColumnarBatch;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.vectorized;
package org.apache.gluten.vectorized;

import io.glutenproject.execution.SparkRowIterator;
import io.glutenproject.row.SparkRowInfo;
import org.apache.gluten.execution.SparkRowIterator;
import org.apache.gluten.row.SparkRowInfo;

public class CHBlockConverterJniWrapper {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.vectorized;
package org.apache.gluten.vectorized;

import org.apache.spark.sql.vectorized.ColumnarBatch;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.vectorized;
package org.apache.gluten.vectorized;

import org.apache.spark.sql.types.DataType;
import org.apache.spark.sql.types.Decimal;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.vectorized;
package org.apache.gluten.vectorized;

import io.glutenproject.exception.GlutenException;
import org.apache.gluten.exception.GlutenException;

import org.apache.spark.sql.execution.utils.CHExecUtil;
import org.apache.spark.sql.vectorized.ColumnVector;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.glutenproject.vectorized;

import io.glutenproject.GlutenConfig;
import io.glutenproject.backendsapi.BackendsApiManager;
import io.glutenproject.memory.alloc.CHNativeMemoryAllocators;
import io.glutenproject.substrait.expression.ExpressionBuilder;
import io.glutenproject.substrait.expression.StringMapNode;
import io.glutenproject.substrait.extensions.AdvancedExtensionNode;
import io.glutenproject.substrait.extensions.ExtensionBuilder;
import io.glutenproject.substrait.plan.PlanBuilder;
package org.apache.gluten.vectorized;

import org.apache.gluten.GlutenConfig;
import org.apache.gluten.backendsapi.BackendsApiManager;
import org.apache.gluten.memory.alloc.CHNativeMemoryAllocators;
import org.apache.gluten.substrait.expression.ExpressionBuilder;
import org.apache.gluten.substrait.expression.StringMapNode;
import org.apache.gluten.substrait.extensions.AdvancedExtensionNode;
import org.apache.gluten.substrait.extensions.ExtensionBuilder;
import org.apache.gluten.substrait.plan.PlanBuilder;

import org.apache.spark.SparkConf;
import org.apache.spark.sql.internal.SQLConf;
Expand Down
Loading

0 comments on commit cf252bc

Please sign in to comment.