add ersources

jixuan1989 · Nov 21, 2018 · 295f48f · 295f48f
1 parent 80ee3b7
commit 295f48f
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 2 deletions.
diff --git a/src/main/resources/core-site.xml b/src/main/resources/core-site.xml
@@ -0,0 +1,6 @@
+<configuration>
+    <property>
+        <name>fs.default.name</name>
+        <value>hdfs://node1:9000</value>
+    </property>
+</configuration>
diff --git a/src/main/resources/hdfs-site.xml b/src/main/resources/hdfs-site.xml
@@ -0,0 +1,14 @@
+<configuration>
+   <property>
+        <name>dfs.replication</name>
+        <value>2</value>
+   </property>
+    <property>
+        <name>dfs.datanode.data.dir</name>
+        <value>/home/hxd/hadoop_data</value>
+    </property>
+    <!--<property>
+        <name>dfs.namenode.secondary.http-address</name>
+        <value>MASTERIP:50090</value>
+    </property>-->
+</configuration>
diff --git a/src/main/scala/cn/edu/tsinghua/spark/WordCount.scala b/src/main/scala/cn/edu/tsinghua/spark/WordCount.scala
@@ -8,11 +8,11 @@ object WordCount {
 
     // init spark context
     val conf = new SparkConf().setAppName("wordcount")//.setMaster("spark://node1:7077")
-      .setMaster("local")
+   //   .setMaster("local")
     val sc = new SparkContext(conf)
 
     // read file
-    val textRdd = sc.textFile("pom.xml")
+    val textRdd = sc.textFile("/test/hadoop")
 
     // map reduce
     val result = textRdd.map(word => (word, 1)).reduceByKey(_+_)