Skip to content

Commit

Permalink
Merge pull request #1734 from cdapio/CDAP-20274-fix-spark-test-issues
Browse files Browse the repository at this point in the history 
CDAP-20274 fix spark issues in unit tests
  • Loading branch information
@albertshau
albertshau authored Jan 14, 2023
2 parents 1c3491a + f3f1ddb commit 60cc36a
Show file tree
Hide file tree
Showing 4 changed files with 76 additions and 29 deletions.
35 changes: 34 additions & 1 deletion core-plugins/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,23 @@
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>hydrator-test</artifactId>
<exclusions>
<exclusion>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-unit-test</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-unit-test-spark3_2.12</artifactId>
</dependency>
<!-- Spark3.1.1 uses constant JavaVersion.JAVA_9 that is only there in newer versions of commons-lang3 -->
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
Expand Down Expand Up @@ -70,7 +87,11 @@
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-data-pipeline2_2.11</artifactId>
<artifactId>cdap-data-pipeline3_2.12</artifactId>
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-data-streams3_2.12</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
Expand All @@ -80,6 +101,11 @@
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
</exclusion>
<!-- this is incompatible with the jackson-module-scala_2.12 used by Spark3 -->
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
Expand Down Expand Up @@ -221,6 +247,13 @@
<artifactId>format-orc</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<exclusions>
<!-- conflicts with kryo from Spark3 -->
<exclusion>
<groupId>com.esotericsoftware</groupId>
<artifactId>kryo-shaded</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.cdap.plugin</groupId>
Expand Down
8 changes: 2 additions & 6 deletions hydrator-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,6 @@
<name>Hydrator Common</name>
<packaging>jar</packaging>

<properties>
<spark.version>1.6.1</spark.version>
</properties>

<dependencies>
<dependency>
<groupId>io.cdap.cdap</groupId>
Expand All @@ -56,7 +52,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
<artifactId>spark-core_2.12</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
<exclusions>
Expand Down Expand Up @@ -104,7 +100,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_2.10</artifactId>
<artifactId>spark-mllib_2.12</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
Expand Down
22 changes: 20 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@
<json.version>20160212</json.version>
<netty.version>4.1.75.Final</netty.version>
<netty-http.version>1.3.0</netty-http.version>
<spark1.version>1.6.3</spark1.version>
<spark.version>3.1.1</spark.version>
<testSourceLocation>${project.basedir}/src/test/java/</testSourceLocation>
</properties>

Expand Down Expand Up @@ -176,7 +176,7 @@
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-api-spark2_2.11</artifactId>
<artifactId>cdap-api-spark3_2.12</artifactId>
<version>${cdap.version}</version>
<scope>provided</scope>
</dependency>
Expand Down Expand Up @@ -215,12 +215,24 @@
<version>${cdap.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-data-pipeline3_2.12</artifactId>
<version>${cdap.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-data-streams2_2.11</artifactId>
<version>${cdap.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-data-streams3_2.12</artifactId>
<version>${cdap.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>hydrator-test</artifactId>
Expand Down Expand Up @@ -265,6 +277,12 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-unit-test-spark3_2.12</artifactId>
<version>${cdap.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
Expand Down
40 changes: 20 additions & 20 deletions spark-plugins/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,25 +23,14 @@
<version>2.11.0-SNAPSHOT</version>
</parent>

<properties>
<spark.version>2.1.3</spark.version>
</properties>

<name>Spark Hydrator Plugins</name>
<artifactId>spark-plugins</artifactId>
<modelVersion>4.0.0</modelVersion>

<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>

<dependencies>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-api-spark2_2.11</artifactId>
<artifactId>cdap-api-spark3_2.12</artifactId>
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
Expand All @@ -53,11 +42,11 @@
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-data-pipeline2_2.11</artifactId>
<artifactId>cdap-data-pipeline3_2.12</artifactId>
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-data-streams2_2.11</artifactId>
<artifactId>cdap-data-streams3_2.12</artifactId>
</dependency>
<dependency>
<groupId>io.netty</groupId>
Expand All @@ -70,6 +59,17 @@
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>hydrator-test</artifactId>
<exclusions>
<!-- this is excluded in favor of a direct test dependency on cdap-unit-test-spark3_2.12 -->
<exclusion>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-unit-test</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>io.cdap.cdap</groupId>
<artifactId>cdap-unit-test-spark3_2.12</artifactId>
</dependency>
<dependency>
<groupId>io.cdap.plugin</groupId>
Expand All @@ -86,7 +86,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.11</artifactId>
<artifactId>spark-core_2.12</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
<exclusions>
Expand Down Expand Up @@ -143,19 +143,19 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_2.11</artifactId>
<artifactId>spark-mllib_2.12</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.11</artifactId>
<artifactId>spark-streaming_2.12</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-repl_2.11</artifactId>
<artifactId>spark-repl_2.12</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
Expand All @@ -173,8 +173,8 @@
</dependency>
<dependency>
<groupId>org.apache.bahir</groupId>
<artifactId>spark-streaming-twitter_2.11</artifactId>
<version>${spark.version}</version>
<artifactId>spark-streaming-twitter_2.12</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
Expand Down

0 comments on commit 60cc36a

Please sign in to comment.