Merge branch 'store-the-tables' into adjacent-plus-plumelib

Makefile

@@ -38,6 +38,9 @@ fix-java-style:
 check-java-style:
 	./gradlew -q spotlessCheck javadoc requireJavadoc -g ../.gradle/
 
+update-small-results:
+	rm -rf test/small-goal-files/
+	rsync -av --exclude='*.pdf' --exclude='*.png' --exclude='*.pgf' results/small/ test/small-goal-files/
 
 # This target deletes files that are not committed to version control.
 clean:

README.md

@@ -119,7 +119,10 @@ If you make a change to the mergers that changes merge results, you need to
 update the goal files or else reproducibility checks will fail.
 Copy certain files from `results/small/` to `test/small-goal-files/`.
 
-To update the reproducibility tests, run `make run-all`.
+To update the reproducibility tests, run `make run-all`
+(this takes a long time!) and commit the results.
+This will run merges in parallel.  If the load on your machine becomes very low
+(like no parallelism is happening), then terminate the process and restart it.
 
 ### Load the stored cache
 

test/check_equal_csv.py

@@ -34,12 +34,31 @@ def remove_run_time(df):
     goal_folder = Path(args.goal_folder)
     actual_folder = Path(args.actual_folder)
 
+    for goal_file in goal_folder.glob("**/*.tex"):
+        goal_file = goal_file.relative_to(goal_folder)
+        print(f"Checking {goal_file}")
+        actual_file = actual_folder / goal_file
+        assert actual_file.exists(), f"{actual_file} does not exist"
+        with open(goal_folder / goal_file, "r") as goal_f:
+            goal_lines = goal_f.readlines()
+        with open(actual_file, "r") as actual_f:
+            actual_lines = actual_f.readlines()
+        if goal_lines != actual_lines:
+            print(f"{goal_folder/goal_file} and {actual_file} are not equal")
+            raise ValueError(f"{goal_folder/goal_file} and {actual_file} are not equal")
+
     for goal_file in goal_folder.glob("**/*.csv"):
         goal_file = goal_file.relative_to(goal_folder)
         print(f"Checking {goal_file}")
         actual_file = actual_folder / goal_file
         assert actual_file.exists(), f"{actual_file} does not exist"
-        goal_df = pd.read_csv(goal_folder / goal_file, header=0, index_col="idx")
+        try:
+            goal_df = pd.read_csv(goal_folder / goal_file, header=0, index_col="idx")
+        except Exception:
+            goal_df = pd.read_csv(goal_folder / goal_file)
+            actual_df = pd.read_csv(actual_file)
+            assert goal_df.equals(actual_df)
+            continue
         actual_df = pd.read_csv(actual_file, header=0, index_col="idx")
         goal_df = remove_run_time(goal_df)
         actual_df = remove_run_time(actual_df)

test/small-goal-files/all_repos_head_test_results.csv

@@ -0,0 +1,5 @@
+idx,repository,language,architecture,community,continuous_integration,documentation,history,issues,license,size,unit_test,stars,scorebased_org,randomforest_org,scorebased_utl,randomforest_utl,head hash,head tree fingerprint,head test result
+0,benedikt-schesch/git-hires-merge-example,Java,1.0,1,0,0.433617,10.5,0.0,1,34146,0.448443,54,1,0,1,1,4ae40d33c3b959e3a15e458eb9b0676251e36a41,1cebc99cd5f8a1b210b66b6985f8aa5c38c4af194a32ad23dc3c6261f43d2968,Tests_passed
+1,mangstadt/ez-vcard,Java,1.0,1,0,0.433617,10.5,0.0,1,34146,0.448443,54,1,0,1,1,95298de036f1df38fd7ca7e3225c4fd5ad5c44e6,134a29559bbdf9bcea93d2153b59e65ef205410c0c295f63b6d82a5e9dc828f9,Tests_passed
+2,pedrovgs/algorithms,Java,0.932927,1,1,0.390942,89.0,0.75,1,6616,0.556371,1377,1,1,1,1,c108e3a4b4fa2556b8c88d7ab6803084b01a4364,3c7ee13d6f66c42510e5edb8cff2102d8c61ed625f7cc871397eaa2b1e529895,Tests_passed
+4,tntim96/jscover,Java,0.985816,1,0,0.490264,25.0,6.448276,1,54612,0.627527,298,1,1,1,1,fff7a84ce12a037518e3c58e1d941a4d3dabb7e2,db8ac3637a22e488a08c15d4ddae3d115f42359e9fd0f42e2b2246aa8767d43e,Tests_passed

test/small-goal-files/defs.tex

@@ -0,0 +1,30 @@
+\def\smallReposInitial{4\xspace}
+\def\smallReposValid{4\xspace}
+\def\smallMergesInitial{63\xspace}
+\def\smallMergesPer{50\xspace}
+\def\smallMergesNonTrivial{33\xspace}
+\def\smallReposNonTrivial{3\xspace}
+\def\smallMergesJavaDiff{10\xspace}
+\def\smallReposJavaDiff{4\xspace}
+\def\smallMergesJavaDiffAndParentsPass{6\xspace}
+\def\smallReposJavaDiffAndParentsPass{4\xspace}
+\def\smallReposSampled{4\xspace}
+\def\smallMergesSampled{6\xspace}
+\def\smallReposYieldedFull{0\xspace}
+\def\smallReposTotal{4\xspace}
+\def\smallMergesTotal{6\xspace}
+
+% Results
+\def\smallAverageTriesUntilPass{1.0\xspace}
+\def\smallNumberofMergesWith1TriesUntilPass{54\xspace}
+\def\smallSporkOverOrtCorrect{3\xspace}
+\def\smallSporkOverOrtIncorrect{0\xspace}
+\def\smallMainBranchMerges{4\xspace}
+\def\smallMainBranchMergesPercent{67\xspace}
+\def\smallOtherBranchMerges{2\xspace}
+\def\smallOtherBranchMergesPercent{33\xspace}
+\def\smallReposJava{4\xspace}
+
+% Timeout
+\def\smallParentTestTimeout{30\xspace}
+\def\smallMergeTestTimeout{45\xspace}

test/small-goal-files/merges_analyzed/benedikt-schesch/git-hires-merge-example.csv

@@ -1,3 +1,3 @@
-idx,branch_name,merge,left,right,notes,num_diff_files,union_diff_files,num_intersecting_files,num_diff_lines,num_diff_hunks,imports_involved,non_java_involved,diff contains java file,left_tree_fingerprint,left parent test result,right_tree_fingerprint,right parent test result,parents pass,test merge,sampled for testing
-1,refs/heads/main,4ae40d33c3b959e3a15e458eb9b0676251e36a41,48199306f02a82abdeff0c01fab1ce112126d727,488122ce6c91e157d1f49b88b79feaa083fdee5b,,2,2,2,4,1,False,True,True,2252a9465319d00b974abd931cc98e1f435b4fe232b4dd61a828ad40139ec20c,Tests_passed,3782e47be673266f23cba016dc13973231429bb34e4c08a314c5b2f6c37d238f,Tests_passed,True,True,True
-2,refs/remotes/origin/import_1,0209c47c118fcaf978f6ebc5bad8a334b541de0b,b4331092d8c5d1b03e6a9807ab135ed263b52663,8204af9984280ccc7f7793f903fc0807b626bf9c,,2,2,2,5,2,True,True,True,f48eaaf883e49f3e3a1f3e1ddff64beb42c9d9bacec7d273f748baa63093bd9b,Tests_passed,052542ef8f16db238683e23556af730ee0d7c65c3ac0554d01e7ee800ca9b7c6,Tests_passed,True,True,True
+idx,branch_name,merge,left,right,notes,diff contains java file,imports_involved,left parent test result,left_tree_fingerprint,non_java_involved,num_diff_files,num_diff_hunks,num_diff_lines,num_intersecting_files,parents pass,right parent test result,right_tree_fingerprint,test merge,union_diff_files,sampled for testing
+1,refs/heads/main,4ae40d33c3b959e3a15e458eb9b0676251e36a41,48199306f02a82abdeff0c01fab1ce112126d727,488122ce6c91e157d1f49b88b79feaa083fdee5b,,True,False,Tests_passed,2252a9465319d00b974abd931cc98e1f435b4fe232b4dd61a828ad40139ec20c,True,2,1,4,2,True,Tests_passed,3782e47be673266f23cba016dc13973231429bb34e4c08a314c5b2f6c37d238f,True,2,True
+2,refs/remotes/origin/import_1,0209c47c118fcaf978f6ebc5bad8a334b541de0b,b4331092d8c5d1b03e6a9807ab135ed263b52663,8204af9984280ccc7f7793f903fc0807b626bf9c,,True,True,Tests_passed,f48eaaf883e49f3e3a1f3e1ddff64beb42c9d9bacec7d273f748baa63093bd9b,True,2,2,5,2,True,Tests_passed,052542ef8f16db238683e23556af730ee0d7c65c3ac0554d01e7ee800ca9b7c6,True,2,True

test/small-goal-files/merges_analyzed/mangstadt/ez-vcard.csv

@@ -1,14 +1,14 @@
-idx,branch_name,merge,left,right,notes,num_diff_files,union_diff_files,num_intersecting_files,num_diff_lines,num_diff_hunks,imports_involved,non_java_involved,diff contains java file,test merge,left_tree_fingerprint,left parent test result,right_tree_fingerprint,right parent test result,parents pass,sampled for testing
-1,refs/heads/master,8f26d88838bfe671cef10e6a832c0c6a57b42fcd,814aae2b1a8ba6e42074d282b2ff9d1af632aef3,f308a61574b9182d151aa9d0cf5fbe5818a48fb8,,5,5,1,299,30,True,True,False,False,,,,,,False
-2,refs/heads/master,0637f5fc19cf6ad79cd2c58981facc3678d5bbff,b8b0b0c8e1dff237975942022a42c6c25b9d9741,3c10bf587f097be3e2a77b0d2d556e6f9efd97b1,,7,7,1,43,12,True,True,False,False,,,,,,False
-3,refs/heads/master,633e9c677f26c1cae476d9ce43331cc30bd8f9e8,af63ebf1a441d0165bab7f6250caf0e2793b9829,22631a4817aa29b231919cb5d8c05d935386b763,a parent is the base,2,2,0,2,1,False,True,False,False,,,,,,False
-4,refs/heads/master,a927decebd50d02bff3f8c56ade565b98d0ecd4c,2380144cf35bc2c211874ac53db2d846fccffb97,6621980643544df2261bd874717839dd24b28324,,7,7,1,38,8,True,True,False,False,,,,,,False
-5,refs/heads/master,d6a31c5fece9a634b7829df08bd0afa0ef06b69a,4639361f062ee6455b88e31996743e300240e71e,4e959f00b9e3a80ea99486834b0ab164be663fdc,a parent is the base,2,2,0,2,1,False,True,False,False,,,,,,False
-6,refs/heads/master,8b26b8c56130e60347f8f20c17c3eca14b601cc1,e454bdf3e3b7d02bb95ff8a58952e70cb97dd0f9,522254f7ceb5e390cbb0904d698cb6294e081ce2,a parent is the base,4,4,0,18,3,True,True,False,False,,,,,,False
-7,refs/heads/master,63ab6b841a7aa31372315a902b98fc5b2cc3b661,ea6026ee62cc184db68d841d50d58474fcdf4862,ab2032ca9769d452d4906f51cf56ca7d983a27c4,,342,342,10,16131,2526,True,True,True,True,aa82b49dd87edb07fbe37b7c2d6801fce8db7c437d1afe8a7b2723c81806286e,Tests_passed,66055b6c4498a64239ae0e49781e086ed06ec3e58be7d9651c23c53f8c6bab6a,Tests_passed,True,True
-8,refs/heads/master,5e11fc8e2c1f5763ff6e6b37d3e750e369af7c94,b42390b2d6279bf7c23580f0f4ac517cba448cd5,dee07e7edfb332efd5d5207660248f71e96a1a0e,,51,51,1,104,50,True,True,False,False,,,,,,False
-9,refs/heads/master,bce54f56a1da7332cabdec21b26e82cd899ce82b,16b6d6cd734326e55af96146ac89594c8ed1b233,ce422afbecd67144c1e8caceaa151f80d4f8a0cc,,35,35,1,1940,161,True,True,False,False,,,,,,False
-10,refs/heads/master,f238de6e66d3e5d03715aa88dccc660d0ab89318,03bd6bf2c9873b23344b31ca7c39524ad9ee5756,c824e9fabecc54d431d4c860b95a2ad13794556c,a parent is the base,3,3,0,38,2,False,True,False,False,,,,,,False
-11,refs/heads/master,a2a632bba203ee71191e71e11058726f2f030b00,345934e15e82c9cbffb5b297aab51ad105b9ad82,541f59d09b5232b68cda87ee2abba7c62d33ee32,,4,4,1,77,4,True,True,False,False,,,,,,False
-12,refs/remotes/origin/pull/49,72e37df9ec88656844758c7303e8e1f9234295c3,24476c206b8df9f5458e2ec57fa16d3ebe23f898,64164540fa269cd1ed14f04236a88b8a3ac22dcb,,122,122,1,9540,291,True,True,False,False,,,,,,False
-13,refs/remotes/origin/pull/49,073c34351663673f4b53142bacf75033706e067f,64847e218c67593c77b0ebdefc93b5e7b4c82406,e80bea970d0b1a4ee340641f01b30b09f816eaa2,,116,116,1,10364,669,True,True,False,False,,,,,,False
+idx,branch_name,merge,left,right,notes,diff contains java file,imports_involved,non_java_involved,num_diff_files,num_diff_hunks,num_diff_lines,num_intersecting_files,test merge,union_diff_files,left parent test result,left_tree_fingerprint,parents pass,right parent test result,right_tree_fingerprint,sampled for testing
+1,refs/heads/master,8f26d88838bfe671cef10e6a832c0c6a57b42fcd,814aae2b1a8ba6e42074d282b2ff9d1af632aef3,f308a61574b9182d151aa9d0cf5fbe5818a48fb8,,False,True,True,5,30,299,1,False,5,,,,,,False
+2,refs/heads/master,0637f5fc19cf6ad79cd2c58981facc3678d5bbff,b8b0b0c8e1dff237975942022a42c6c25b9d9741,3c10bf587f097be3e2a77b0d2d556e6f9efd97b1,,False,True,True,7,12,43,1,False,7,,,,,,False
+3,refs/heads/master,633e9c677f26c1cae476d9ce43331cc30bd8f9e8,af63ebf1a441d0165bab7f6250caf0e2793b9829,22631a4817aa29b231919cb5d8c05d935386b763,a parent is the base,False,False,True,2,1,2,0,False,2,,,,,,False
+4,refs/heads/master,a927decebd50d02bff3f8c56ade565b98d0ecd4c,2380144cf35bc2c211874ac53db2d846fccffb97,6621980643544df2261bd874717839dd24b28324,,False,True,True,7,8,38,1,False,7,,,,,,False
+5,refs/heads/master,d6a31c5fece9a634b7829df08bd0afa0ef06b69a,4639361f062ee6455b88e31996743e300240e71e,4e959f00b9e3a80ea99486834b0ab164be663fdc,a parent is the base,False,False,True,2,1,2,0,False,2,,,,,,False
+6,refs/heads/master,8b26b8c56130e60347f8f20c17c3eca14b601cc1,e454bdf3e3b7d02bb95ff8a58952e70cb97dd0f9,522254f7ceb5e390cbb0904d698cb6294e081ce2,a parent is the base,False,True,True,4,3,18,0,False,4,,,,,,False
+7,refs/heads/master,63ab6b841a7aa31372315a902b98fc5b2cc3b661,ea6026ee62cc184db68d841d50d58474fcdf4862,ab2032ca9769d452d4906f51cf56ca7d983a27c4,,True,True,True,342,2526,16131,10,True,342,Tests_passed,aa82b49dd87edb07fbe37b7c2d6801fce8db7c437d1afe8a7b2723c81806286e,True,Tests_passed,66055b6c4498a64239ae0e49781e086ed06ec3e58be7d9651c23c53f8c6bab6a,True
+8,refs/heads/master,5e11fc8e2c1f5763ff6e6b37d3e750e369af7c94,b42390b2d6279bf7c23580f0f4ac517cba448cd5,dee07e7edfb332efd5d5207660248f71e96a1a0e,,False,True,True,51,50,104,1,False,51,,,,,,False
+9,refs/heads/master,bce54f56a1da7332cabdec21b26e82cd899ce82b,16b6d6cd734326e55af96146ac89594c8ed1b233,ce422afbecd67144c1e8caceaa151f80d4f8a0cc,,False,True,True,35,161,1940,1,False,35,,,,,,False
+10,refs/heads/master,f238de6e66d3e5d03715aa88dccc660d0ab89318,03bd6bf2c9873b23344b31ca7c39524ad9ee5756,c824e9fabecc54d431d4c860b95a2ad13794556c,a parent is the base,False,False,True,3,2,38,0,False,3,,,,,,False
+11,refs/heads/master,a2a632bba203ee71191e71e11058726f2f030b00,345934e15e82c9cbffb5b297aab51ad105b9ad82,541f59d09b5232b68cda87ee2abba7c62d33ee32,,False,True,True,4,4,77,1,False,4,,,,,,False
+12,refs/remotes/origin/pull/49,72e37df9ec88656844758c7303e8e1f9234295c3,24476c206b8df9f5458e2ec57fa16d3ebe23f898,64164540fa269cd1ed14f04236a88b8a3ac22dcb,,False,True,True,122,291,9540,1,False,122,,,,,,False
+13,refs/remotes/origin/pull/49,073c34351663673f4b53142bacf75033706e067f,64847e218c67593c77b0ebdefc93b5e7b4c82406,e80bea970d0b1a4ee340641f01b30b09f816eaa2,,False,True,True,116,669,10364,1,False,116,,,,,,False