forked from rajivushah/xap-kafka-ie
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathSpark Streaming Kafka Access Fixed.json
1 lines (1 loc) · 4.7 KB
/
Spark Streaming Kafka Access Fixed.json
1
{"paragraphs":[{"title":"Required Dependencies","text":"%dep\n\nz.load(\"org.apache.spark:spark-streaming-kafka_2.10:jar:1.6.0\")\nz.load(\"org.apache.kafka:kafka_2.10:jar:0.10.2.1\")\n\nz.load(\"C:/WorkspaceMain/xap-kafka-ie/parent/common/target/common-1.0.0-SNAPSHOT.jar\")\nz.load(\"C:/WorkspaceMain/xap-kafka-ie/parent/decoder/target/decoder-1.0.0-SNAPSHOT.jar\")\n//z.load(\"/code/xap-kafka-ie/parent/common/target/common-1.0.0-SNAPSHOT.jar\")\n//z.load(\"/code/xap-kafka-ie/parent/decoder/target/decoder-1.0.0-SNAPSHOT.jar\")\n","dateUpdated":"2017-06-30T13:11:16-0400","config":{"colWidth":12,"editorMode":"ace/mode/scala","title":true,"graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1498837564756_538332920","id":"20160830-145728_362255206","result":{"code":"SUCCESS","type":"TEXT","msg":"res0: org.apache.zeppelin.dep.Dependency = org.apache.zeppelin.dep.Dependency@2e3fac9\n"},"dateCreated":"2017-06-30T11:46:04-0400","dateStarted":"2017-06-30T13:11:16-0400","dateFinished":"2017-06-30T13:11:22-0400","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:359","focus":true},{"text":"%spark\n\nimport kafka.serializer.StringDecoder\n\nimport org.apache.spark.streaming.{Seconds, StreamingContext}\nimport org.apache.spark.streaming.kafka.KafkaUtils\n\nimport org.common.PriceFeed\nimport org.kafka.decoder.PriceFeedStreamDecoder\n\n\nval ssc = new StreamingContext(sc, Seconds(10))\n//ssc.checkpoint(\"c:/ie-training/tmp\")\nssc.checkpoint(\"/tmp/checkpoint\")\n\nvar topics = Set(\"priceFeed\") \nval kafkaParams = Map(\n \"metadata.broker.list\" -> \"localhost:9092\",\n \"auto.offset.reset\" -> \"smallest\"\n)\n\nval messages = KafkaUtils.createDirectStream[String, PriceFeed, StringDecoder, PriceFeedStreamDecoder](ssc, kafkaParams, topics) \n\nmessages.foreachRDD(rdd => \n if (!rdd.isEmpty) {\n println(\"------------------------\")\n val count = rdd.count.toInt\n println(\"count received \" + count) // displays count received 3966 \n rdd.take(1).foreach(println) // this will print all RDD such as (null,PriceFeed [id=A1^1497450614721^14, symbol=A0, price=1.0]) \n rdd.values.saveToGrid()\n }\n)\n\nssc.start ","dateUpdated":"2017-06-30T13:11:24-0400","config":{"colWidth":12,"editorMode":"ace/mode/scala","graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1498837564760_536793924","id":"20170614-111940_344094011","result":{"code":"SUCCESS","type":"TEXT","msg":"import kafka.serializer.StringDecoder\nimport org.apache.spark.streaming.{Seconds, StreamingContext}\nimport org.apache.spark.streaming.kafka.KafkaUtils\nimport org.common.PriceFeed\nimport org.kafka.decoder.PriceFeedStreamDecoder\nssc: org.apache.spark.streaming.StreamingContext = org.apache.spark.streaming.StreamingContext@32a41073\ntopics: scala.collection.immutable.Set[String] = Set(priceFeed)\nkafkaParams: scala.collection.immutable.Map[String,String] = Map(metadata.broker.list -> localhost:9092, auto.offset.reset -> smallest)\nmessages: org.apache.spark.streaming.dstream.InputDStream[(String, org.common.PriceFeed)] = org.apache.spark.streaming.kafka.DirectKafkaInputDStream@2971a52d\n"},"dateCreated":"2017-06-30T11:46:04-0400","dateStarted":"2017-06-30T13:11:24-0400","dateFinished":"2017-06-30T13:11:36-0400","status":"FINISHED","progressUpdateIntervalMs":500,"$$hashKey":"object:360","focus":true},{"text":"%spark\n\nssc.stop(stopSparkContext=false, stopGracefully=true)\n","dateUpdated":"2017-06-30T11:46:04-0400","config":{"colWidth":12,"editorMode":"ace/mode/scala","graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1498837564760_536793924","id":"20160830-170845_1145510546","dateCreated":"2017-06-30T11:46:04-0400","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:361"},{"dateUpdated":"2017-06-30T11:46:04-0400","config":{"colWidth":12,"editorMode":"ace/mode/scala","graph":{"mode":"table","height":300,"optionOpen":false,"keys":[],"values":[],"groups":[],"scatter":{}},"enabled":true},"settings":{"params":{},"forms":{}},"jobName":"paragraph_1498837564760_536793924","id":"20160830-165856_475845132","dateCreated":"2017-06-30T11:46:04-0400","status":"READY","errorMessage":"","progressUpdateIntervalMs":500,"$$hashKey":"object:362"}],"name":"Spark Streaming Kafka Access Fixed","id":"2CMNS9SFP","angularObjects":{"2BY3CWM32:shared_process":[],"2C1697SXS:shared_process":[]},"config":{"looknfeel":"default"},"info":{}}