Pull request for recommender system framework over GraphChi for review

pom.xml

@@ -14,15 +14,18 @@
         <name>Sonatype Nexus Snapshots</name>
         <url>https://oss.sonatype.org/content/repositories/snapshots/</url>
         <snapshots><enabled>true</enabled></snapshots>
-
     </repository>
-  <repository>  
+  	<repository>  
       <id>scala-tools.org</id>  
       <name>Scala-tools Maven2 Repository</name>  
       <url>http://scala-tools.org/repo-releases</url>  
     </repository>
+    <repository>  
+      <id>ApacheReleases</id>  
+      <name>Apache release repository</name>  
+      <url>https://repository.apache.org/content/repositories/releases</url>  
+    </repository>
   </repositories>
-
     <dependencies>
         <dependency>
         <groupId>com.yammer.metrics</groupId>
@@ -32,35 +35,35 @@
 
 	    <!-- Scala version is very important. Luckily the plugin warns you if you don't specify:   
 	        [WARNING] you don't define org.scala-lang:scala-library as a dependency of the project -->  
-	    <dependency>  
-	      <groupId>org.scala-lang</groupId>  
-	      <artifactId>scala-library</artifactId>  
-	      <version>2.9.0-1</version>  
-	    </dependency>
-        <dependency>
-        	<groupId>mysql</groupId>
-        	<artifactId>mysql-connector-java</artifactId>
-        	<version>5.1.6</version>
-        </dependency>
-	 <dependency>
-	      <groupId>junit</groupId>
-	      <artifactId>junit</artifactId>
-	      <version>4.10</version>
-	      <type>jar</type>
-	      <scope>test</scope>
-	      <optional>true</optional>
-	    </dependency>
- <dependency>
-	<groupId>org.apache.pig</groupId>
-	<artifactId>pig</artifactId>
-	<scope>compile</scope>
-	<version>0.10.0</version>
-</dependency>
-        <dependency>
+    <dependency>  
+	   <groupId>org.scala-lang</groupId>  
+	   <artifactId>scala-library</artifactId>  
+	   <version>2.9.0-1</version>  
+    </dependency>
+    <dependency>
+        <groupId>mysql</groupId>
+        <artifactId>mysql-connector-java</artifactId>
+        <version>5.1.6</version>
+    </dependency>
+    <dependency>
+        <groupId>junit</groupId>
+        <artifactId>junit</artifactId>
+        <version>4.10</version>
+        <type>jar</type>
+        <scope>test</scope>
+        <optional>true</optional>
+    </dependency>
+	    <dependency>
             <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-core</artifactId>
-            <version>0.20.2</version>
+            <artifactId>hadoop-client</artifactId>
+            <version>2.2.0</version>
         </dependency>
+ 	<dependency>
+		<groupId>org.apache.pig</groupId>
+		<artifactId>pig</artifactId>
+		<scope>compile</scope>
+		<version>0.12.0</version>
+	</dependency>
         <dependency>
 	        <groupId>org.apache.commons</groupId>
 	        <artifactId>commons-math</artifactId>
@@ -76,6 +79,21 @@
             <artifactId>commons-cli</artifactId>
             <version>1.2</version>
         </dependency>
+        <dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-math3</artifactId>
+			<version>3.1</version>
+		</dependency>
+		<dependency>
+			<groupId>gov.sandia.foundry</groupId>
+			<artifactId>gov-sandia-cognition-learning-core</artifactId>
+			<version>3.3.3</version>
+		</dependency>
+		<dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk</artifactId>
+            <version>1.6.7</version>
+        </dependency>
     </dependencies>
 
   	<build>

sample_data/Movielens/conversion_scripts.sh

@@ -0,0 +1,3 @@
+python movielens_userfeatures.py /media/Data1/Capstone/Movielens/ml-100k/working_dir/u.user;
+python movielens_item_features.py /media/Data1/Capstone/Movielens/ml-100k/working_dir/u.item;
+python convert_to_mm.py -g /media/Data1/Capstone/Movielens/ml-100k/working_dir/u.data -e 100000 -u '{"file_name":"/media/Data1/Capstone/Movielens/ml-100k/working_dir/u.user.processed", "num":943}' -i '{"file_name":"/media/Data1/Capstone/Movielens/ml-100k/working_dir/u.item.processed", "num":1682}';

sample_data/Movielens/ml-100k/README

@@ -0,0 +1,145 @@
+SUMMARY & USAGE LICENSE
+=============================================
+
+MovieLens data sets were collected by the GroupLens Research Project
+at the University of Minnesota.
+
+This data set consists of:
+	* 100,000 ratings (1-5) from 943 users on 1682 movies. 
+	* Each user has rated at least 20 movies. 
+        * Simple demographic info for the users (age, gender, occupation, zip)
+
+The data was collected through the MovieLens web site
+(movielens.umn.edu) during the seven-month period from September 19th, 
+1997 through April 22nd, 1998. This data has been cleaned up - users
+who had less than 20 ratings or did not have complete demographic
+information were removed from this data set. Detailed descriptions of
+the data file can be found at the end of this file.
+
+Neither the University of Minnesota nor any of the researchers
+involved can guarantee the correctness of the data, its suitability
+for any particular purpose, or the validity of results based on the
+use of the data set.  The data set may be used for any research
+purposes under the following conditions:
+
+     * The user may not state or imply any endorsement from the
+       University of Minnesota or the GroupLens Research Group.
+
+     * The user must acknowledge the use of the data set in
+       publications resulting from the use of the data set, and must
+       send us an electronic or paper copy of those publications.
+
+     * The user may not redistribute the data without separate
+       permission.
+
+     * The user may not use this information for any commercial or
+       revenue-bearing purposes without first obtaining permission
+       from a faculty member of the GroupLens Research Project at the
+       University of Minnesota.
+
+If you have any further questions or comments, please contact Jon Herlocker
+<[email protected]>. 
+
+ACKNOWLEDGEMENTS
+==============================================
+
+Thanks to Al Borchers for cleaning up this data and writing the
+accompanying scripts.
+
+PUBLISHED WORK THAT HAS USED THIS DATASET
+==============================================
+
+Herlocker, J., Konstan, J., Borchers, A., Riedl, J.. An Algorithmic
+Framework for Performing Collaborative Filtering. Proceedings of the
+1999 Conference on Research and Development in Information
+Retrieval. Aug. 1999.
+
+FURTHER INFORMATION ABOUT THE GROUPLENS RESEARCH PROJECT
+==============================================
+
+The GroupLens Research Project is a research group in the Department
+of Computer Science and Engineering at the University of Minnesota.
+Members of the GroupLens Research Project are involved in many
+research projects related to the fields of information filtering,
+collaborative filtering, and recommender systems. The project is lead
+by professors John Riedl and Joseph Konstan. The project began to
+explore automated collaborative filtering in 1992, but is most well
+known for its world wide trial of an automated collaborative filtering
+system for Usenet news in 1996.  The technology developed in the
+Usenet trial formed the base for the formation of Net Perceptions,
+Inc., which was founded by members of GroupLens Research. Since then
+the project has expanded its scope to research overall information
+filtering solutions, integrating in content-based methods as well as
+improving current collaborative filtering technology.
+
+Further information on the GroupLens Research project, including
+research publications, can be found at the following web site:
+
+        http://www.grouplens.org/
+
+GroupLens Research currently operates a movie recommender based on
+collaborative filtering:
+
+        http://www.movielens.org/
+
+DETAILED DESCRIPTIONS OF DATA FILES
+==============================================
+
+Here are brief descriptions of the data.
+
+ml-data.tar.gz   -- Compressed tar file.  To rebuild the u data files do this:
+                gunzip ml-data.tar.gz
+                tar xvf ml-data.tar
+                mku.sh
+
+u.data     -- The full u data set, 100000 ratings by 943 users on 1682 items.
+              Each user has rated at least 20 movies.  Users and items are
+              numbered consecutively from 1.  The data is randomly
+              ordered. This is a tab separated list of 
+	         user id | item id | rating | timestamp. 
+              The time stamps are unix seconds since 1/1/1970 UTC   
+
+u.info     -- The number of users, items, and ratings in the u data set.
+
+u.item     -- Information about the items (movies); this is a tab separated
+              list of
+              movie id | movie title | release date | video release date |
+              IMDb URL | unknown | Action | Adventure | Animation |
+              Children's | Comedy | Crime | Documentary | Drama | Fantasy |
+              Film-Noir | Horror | Musical | Mystery | Romance | Sci-Fi |
+              Thriller | War | Western |
+              The last 19 fields are the genres, a 1 indicates the movie
+              is of that genre, a 0 indicates it is not; movies can be in
+              several genres at once.
+              The movie ids are the ones used in the u.data data set.
+
+u.genre    -- A list of the genres.
+
+u.user     -- Demographic information about the users; this is a tab
+              separated list of
+              user id | age | gender | occupation | zip code
+              The user ids are the ones used in the u.data data set.
+
+u.occupation -- A list of the occupations.
+
+u1.base    -- The data sets u1.base and u1.test through u5.base and u5.test
+u1.test       are 80%/20% splits of the u data into training and test data.
+u2.base       Each of u1, ..., u5 have disjoint test sets; this if for
+u2.test       5 fold cross validation (where you repeat your experiment
+u3.base       with each training and test set and average the results).
+u3.test       These data sets can be generated from u.data by mku.sh.
+u4.base
+u4.test
+u5.base
+u5.test
+
+ua.base    -- The data sets ua.base, ua.test, ub.base, and ub.test
+ua.test       split the u data into a training set and a test set with
+ub.base       exactly 10 ratings per user in the test set.  The sets
+ub.test       ua.test and ub.test are disjoint.  These data sets can
+              be generated from u.data by mku.sh.
+
+allbut.pl  -- The script that generates training and test sets where
+              all but n of a users ratings are in the training data.
+
+mku.sh     -- A shell script to generate all the u data sets from u.data.

sample_data/Movielens/ml-100k/all_tests.json

@@ -0,0 +1,52 @@
+[
+    {
+        "serializedFile":"/tmp/BIAS_SGD_003_1_10_001_100",
+        "outputFile":"/tmp/out.txt",
+        "errorMeasure": "RMSE"
+    },
+    {
+        "serializedFile":"/tmp/BIAS_SGD_003_1_10_001_100",
+        "outputFile":"/tmp/out.txt",
+        "errorMeasure": "MAE"
+    },
+    {
+        "serializedFile":"/tmp/SVDPP_10_01_1_007_005_50",
+        "outputFile":"/tmp/out.txt",
+        "errorMeasure": "RMSE"
+    },
+    {
+        "serializedFile":"/tmp/SVDPP_10_01_1_007_005_50",
+        "outputFile":"/tmp/out.txt",
+        "errorMeasure": "MAE"
+    },
+    {
+        "serializedFile":"/tmp/ALS_065",
+        "outputFile":"/tmp/out.txt",
+        "errorMeasure": "RMSE"
+    },
+    {
+        "serializedFile":"/tmp/ALS_065",
+        "outputFile":"/tmp/out.txt",
+        "errorMeasure": "MAE"
+    },
+    {
+        "serializedFile": "/tmp/LibFM_1__20_002_50",
+        "outputFile":"/tmp/out.txt",
+        "errorMeasure": "RMSE"
+    },
+    {
+        "serializedFile": "/tmp/LibFM_1__20_002_50",
+        "outputFile":"/tmp/out.txt",
+        "errorMeasure": "MAE"
+    },
+    {
+        "serializedFile":"/tmp/PMF_065_10_5b",
+        "outputFile":"/tmp/out.txt",
+        "errorMeasure": "RMSE"
+    },
+    {
+        "serializedFile":"/tmp/PMF_065_10_5b",
+        "outputFile":"/tmp/out.txt",
+        "errorMeasure": "MAE"
+    }
+]

sample_data/Movielens/ml-100k/allbut.pl

@@ -0,0 +1,34 @@
+#!/usr/local/bin/perl
+
+# get args
+if (@ARGV < 3) {
+	print STDERR "Usage: $0 base_name start stop max_test [ratings ...]\n";
+	exit 1;
+}
+$basename = shift;
+$start = shift;
+$stop = shift;
+$maxtest = shift;
+
+# open files
+open( TESTFILE, ">$basename.test" ) or die "Cannot open $basename.test for writing\n";
+open( BASEFILE, ">$basename.base" ) or die "Cannot open $basename.base for writing\n";
+
+# init variables
+$testcnt = 0;
+
+while (<>) {
+	($user) = split;
+	if (! defined $ratingcnt{$user}) {
+		$ratingcnt{$user} = 0;
+	}
+	++$ratingcnt{$user};
+	if (($testcnt < $maxtest || $maxtest <= 0)
+	&& $ratingcnt{$user} >= $start && $ratingcnt{$user} <= $stop) {
+		++$testcnt;
+		print TESTFILE;
+	}
+	else {
+		print BASEFILE;
+	}
+}

sample_data/Movielens/ml-100k/file_ml-100k_desc.json

@@ -0,0 +1,15 @@
+{
+    "ratingsUrl": "file://./sample_data/Movielens/ml-100k/working_dir2/u.data_tr1.mm",
+    "userFeaturesUrl": "file://./sample_data/Movielens/ml-100k/working_dir2/u.user.processed.converted",
+    "itemFeaturesUrl": "file://./sample_data/Movielens/ml-100k/working_dir2/u.item.processed.converted",
+    "validationUrl": "file://./sample_data/Movielens/ml-100k/working_dir2/u.data_val1.mm",
+    "numUsers": "943",
+    "numItems": "1682",
+    "numRatings": "80000",
+    "meanRatings": "0",
+    "numUserFeatures": "38",
+    "numItemFeatures": "20",
+    "numRatingFeatures": "0",
+    "minval": "1",
+    "maxval": "5"
+}

sample_data/Movielens/ml-100k/file_test_ml-100k_desc.json

@@ -0,0 +1,14 @@
+{
+    "ratingsUrl": "file://./sample_data/Movielens/ml-100k/working_dir2/u.data_val1.mm",
+    "userFeaturesUrl": "file://./sample_data/Movielens/ml-100k/working_dir2/u.user.processed.converted",
+    "itemFeaturesUrl": "file://./sample_data/Movielens/ml-100k/working_dir2/u.item.processed.converted",
+    "numUsers": "943",
+    "numItems": "1682",
+    "numRatings": "20000",
+    "meanRatings": "0",
+    "numUserFeatures": "38",
+    "numItemFeatures": "20",
+    "numRatingFeatures": "0",
+    "minval": "1",
+    "maxval": "5"
+}

sample_data/Movielens/ml-100k/hdfs_ml-100k_desc.json

@@ -0,0 +1,15 @@
+{
+    "ratingsUrl": "hdfs://localhost:9000/user/hdfs/Movielens/ml-100k/working_dir2/u.data_tr1.mm",
+    "userFeaturesUrl": "hdfs://localhost:9000/user/hdfs/Movielens/ml-100k/working_dir2/u.user.processed.converted",
+    "itemFeaturesUrl": "hdfs://localhost:9000/user/hdfs/Movielens/ml-100k/working_dir2/u.item.processed.converted",
+    "validationUrl": "hdfs://localhost:9000/user/hdfs/Movielens/ml-100k/working_dir2/u.data_val1.mm",
+    "numUsers": "943",
+    "numItems": "1682",
+    "numRatings": "80000",
+    "meanRatings": "0",
+    "numUserFeatures": "38",
+    "numItemFeatures": "20",
+    "numRatingFeatures": "0",
+    "minval": "1",
+    "maxval": "5"
+}