Fixes to allow running in Nautilus SDK Desktop.

- Added Pravega credentials dependency to all projects - Flink JAR can now be published to Nautilus Maven repo. - Hadoop examples can now be run without Hadoop installed. - Spark examples can now be run without Spark installed. Signed-off-by: Claudio Fahey <[email protected]>
pravega · Apr 23, 2019 · 35f1363 · 35f1363
1 parent e0d152c
commit 35f1363
Show file tree

Hide file tree

Showing 9 changed files with 54 additions and 5 deletions.
diff --git a/flink-connector-examples/build.gradle b/flink-connector-examples/build.gradle
@@ -17,6 +17,7 @@ apply plugin: "scala"
 apply plugin: "distribution"
 apply plugin: 'idea'
 apply plugin: 'eclipse'
+apply plugin: 'maven-publish'
 
 sourceCompatibility = "1.8"
 archivesBaseName = 'pravega-flink-examples'
@@ -31,13 +32,18 @@ dependencies {
     compile "org.apache.flink:flink-streaming-java_2.11:${flinkVersion}"
     compile "org.apache.flink:flink-streaming-scala_2.11:${flinkVersion}"
     compile "org.slf4j:slf4j-log4j12:1.7.25"
+    if (includePravegaCredentials.toBoolean()) {
+        compile "io.pravega:pravega-keycloak-credentials:${pravegaCredentialsVersion}"
+    }
 }
 
 shadowJar {
     dependencies {
         include dependency("org.scala-lang.modules:scala-java8-compat_2.11")
         include dependency("io.pravega:pravega-connectors-flink_2.11")
     }
+    classifier = ""
+    zip64 true
 }
 
 task scriptWordCountWriter(type: CreateStartScripts) {
@@ -111,3 +117,21 @@ distributions {
         }
     }
 }
+
+publishing {
+    repositories {
+        maven {
+            credentials {
+                username "nautilus"
+                password "password"
+            }
+            url = "http://repo/maven2"
+        }
+    }
+
+    publications {
+        shadow(MavenPublication) { publication ->
+            project.shadow.component(publication)
+        }
+    }
+}
diff --git a/gradle.properties b/gradle.properties
@@ -10,6 +10,8 @@
 ### dependencies
 pravegaVersion=0.4.0
 flinkConnectorVersion=0.4.0
+includePravegaCredentials=false
+pravegaCredentialsVersion=0.4.0-2030.d99411b-0.0.1-020.26736d2
 
 ### Pravega-samples output library
 samplesVersion=0.5.0-SNAPSHOT

diff --git a/hadoop-connector-examples/build.gradle b/hadoop-connector-examples/build.gradle
@@ -35,10 +35,13 @@ repositories {
 }
 
 dependencies {
-    compile     "io.pravega:pravega-connectors-hadoop:${hadoopConnectorVersion}"
-    compileOnly "org.apache.hadoop:hadoop-common:${hadoopVersion}"
-    compileOnly "org.apache.hadoop:hadoop-mapreduce-client-core:${hadoopVersion}"
-    compileOnly "org.apache.spark:spark-core_2.11:${sparkVersion}"
+    compile "io.pravega:pravega-connectors-hadoop:${hadoopConnectorVersion}"
+    compile "org.apache.hadoop:hadoop-common:${hadoopVersion}"
+    compile "org.apache.hadoop:hadoop-mapreduce-client-core:${hadoopVersion}"
+    compile "org.apache.spark:spark-core_2.11:${sparkVersion}"
+    if (includePravegaCredentials.toBoolean()) {
+        compile "io.pravega:pravega-keycloak-credentials:${pravegaCredentialsVersion}"
+    }
 }
 
 shadowJar {

diff --git a/hadoop-connector-examples/src/main/java/io/pravega/example/spark/wordcount/WordCount.java b/hadoop-connector-examples/src/main/java/io/pravega/example/spark/wordcount/WordCount.java
@@ -18,6 +18,7 @@
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.util.GenericOptionsParser;
 import org.apache.spark.SparkConf;
+import org.apache.spark.SparkContext;
 import org.apache.spark.api.java.JavaPairRDD;
 import org.apache.spark.api.java.JavaRDD;
 import org.apache.spark.api.java.JavaSparkContext;
@@ -46,7 +47,8 @@ public static void main(String[] args) throws Exception {
         conf.setStrings(PravegaConfig.INPUT_STREAM_NAME, remainingArgs[2]);
         conf.setStrings(PravegaConfig.INPUT_DESERIALIZER, TextSerializer.class.getName());
 
-        JavaSparkContext sc = new JavaSparkContext(new SparkConf());
+        SparkConf sparkConf = new SparkConf().setAppName("wordcount").setMaster("local[1]");
+        JavaSparkContext sc = new JavaSparkContext(SparkContext.getOrCreate(sparkConf));
 
         JavaPairRDD<EventKey, Text> lines = sc.newAPIHadoopRDD(conf, PravegaInputFormat.class, EventKey.class, Text.class);
         JavaRDD<String> words = lines.map(x -> x._2).flatMap(s -> Arrays.asList(SPACE.split(s.toString())).iterator());

diff --git a/pravega-client-examples/build.gradle b/pravega-client-examples/build.gradle
@@ -33,6 +33,10 @@ dependencies {
 
     compile "org.slf4j:slf4j-api:1.7.14"
     compile "ch.qos.logback:logback-classic:1.1.7"
+
+    if (includePravegaCredentials.toBoolean()) {
+        compile "io.pravega:pravega-keycloak-credentials:${pravegaCredentialsVersion}"
+    }
 }
 
 // Build examples

diff --git a/scenarios/anomaly-detection/build.gradle b/scenarios/anomaly-detection/build.gradle
@@ -27,6 +27,9 @@ dependencies {
     compile "org.apache.flink:flink-streaming-java_2.11:${flinkVersion}"
     compile "org.apache.flink:flink-connector-elasticsearch5_2.11:${flinkVersion}"
     compile "ch.qos.logback:logback-classic:1.1.7"
+    if (includePravegaCredentials.toBoolean()) {
+        compile "io.pravega:pravega-keycloak-credentials:${pravegaCredentialsVersion}"
+    }
 }
 
 shadowJar {

diff --git a/scenarios/pravega-flink-connector-sql-samples/build.gradle b/scenarios/pravega-flink-connector-sql-samples/build.gradle
@@ -32,6 +32,10 @@ dependencies {
 
     compile "joda-time:joda-time:2.7"
     compile "org.projectlombok:lombok:1.16.18"
+
+    if (includePravegaCredentials.toBoolean()) {
+        compile "io.pravega:pravega-keycloak-credentials:${pravegaCredentialsVersion}"
+    }
 }
 
 task scriptConnectorTableApiSamples(type: CreateStartScripts) {

diff --git a/scenarios/turbine-heat-processor/build.gradle b/scenarios/turbine-heat-processor/build.gradle
@@ -31,6 +31,9 @@ dependencies {
     compile "org.apache.flink:flink-streaming-java_2.11:${flinkVersion}"
     compile "org.apache.flink:flink-streaming-scala_2.11:${flinkVersion}"
     compile "org.slf4j:slf4j-log4j12:1.7.14"
+    if (includePravegaCredentials.toBoolean()) {
+        compile "io.pravega:pravega-keycloak-credentials:${pravegaCredentialsVersion}"
+    }
 }
 
 shadowJar {

diff --git a/scenarios/turbine-heat-sensor/build.gradle b/scenarios/turbine-heat-sensor/build.gradle
@@ -33,6 +33,10 @@ dependencies {
 
     compile "org.slf4j:slf4j-api:1.7.14"
     compile "ch.qos.logback:logback-classic:1.1.7"
+
+    if (includePravegaCredentials.toBoolean()) {
+        compile "io.pravega:pravega-keycloak-credentials:${pravegaCredentialsVersion}"
+    }
 }
 
 task scriptTurbineSensor(type: CreateStartScripts) {