Store tables (#376)

Makefile

@@ -38,6 +38,9 @@ fix-java-style:
 check-java-style:
 	./gradlew -q spotlessCheck javadoc requireJavadoc -g ../.gradle/
 
+update-small-results:
+	rm -rf test/small-goal-files/
+	rsync -av --exclude='*.pdf' --exclude='*.png' --exclude='*unhandled_and_failed_merges_without_intellimerge*' --exclude='*.pgf' results/small/ test/small-goal-files/
 
 # This target deletes files that are not committed to version control.
 clean:

test/check_equal_csv.py

@@ -39,7 +39,21 @@ def remove_run_time(df):
         print(f"Checking {goal_file}")
         actual_file = actual_folder / goal_file
         assert actual_file.exists(), f"{actual_file} does not exist"
-        goal_df = pd.read_csv(goal_folder / goal_file, header=0, index_col="idx")
+        try:
+            goal_df = pd.read_csv(goal_folder / goal_file, header=0, index_col="idx")
+        except Exception:
+            goal_df = pd.read_csv(goal_folder / goal_file)
+            actual_df = pd.read_csv(actual_file)
+            if not goal_df.equals(actual_df):
+                print(f"{goal_folder/goal_file} and {actual_file} are not equal")
+                print("Goal file:")
+                print(goal_df)
+                print("Actual file:")
+                print(actual_df)
+                raise ValueError(
+                    f"{goal_folder/goal_file} and {actual_file} are not equal"
+                )
+            continue
         actual_df = pd.read_csv(actual_file, header=0, index_col="idx")
         goal_df = remove_run_time(goal_df)
         actual_df = remove_run_time(actual_df)

test/small-goal-files/all_repos_head_test_results.csv

@@ -0,0 +1,5 @@
+idx,repository,language,architecture,community,continuous_integration,documentation,history,issues,license,size,unit_test,stars,scorebased_org,randomforest_org,scorebased_utl,randomforest_utl,head hash,head tree fingerprint,head test result
+0,benedikt-schesch/git-hires-merge-example,Java,1.0,1,0,0.433617,10.5,0.0,1,34146,0.448443,54,1,0,1,1,4ae40d33c3b959e3a15e458eb9b0676251e36a41,1cebc99cd5f8a1b210b66b6985f8aa5c38c4af194a32ad23dc3c6261f43d2968,Tests_passed
+1,mangstadt/ez-vcard,Java,1.0,1,0,0.433617,10.5,0.0,1,34146,0.448443,54,1,0,1,1,95298de036f1df38fd7ca7e3225c4fd5ad5c44e6,134a29559bbdf9bcea93d2153b59e65ef205410c0c295f63b6d82a5e9dc828f9,Tests_passed
+2,pedrovgs/algorithms,Java,0.932927,1,1,0.390942,89.0,0.75,1,6616,0.556371,1377,1,1,1,1,c108e3a4b4fa2556b8c88d7ab6803084b01a4364,3c7ee13d6f66c42510e5edb8cff2102d8c61ed625f7cc871397eaa2b1e529895,Tests_passed
+4,tntim96/jscover,Java,0.985816,1,0,0.490264,25.0,6.448276,1,54612,0.627527,298,1,1,1,1,fff7a84ce12a037518e3c58e1d941a4d3dabb7e2,db8ac3637a22e488a08c15d4ddae3d115f42359e9fd0f42e2b2246aa8767d43e,Tests_passed

test/small-goal-files/defs.tex

@@ -0,0 +1,30 @@
+\def\smallReposInitial{4\xspace}
+\def\smallReposValid{4\xspace}
+\def\smallMergesInitial{63\xspace}
+\def\smallMergesPer{50\xspace}
+\def\smallMergesNonTrivial{33\xspace}
+\def\smallReposNonTrivial{3\xspace}
+\def\smallMergesJavaDiff{10\xspace}
+\def\smallReposJavaDiff{4\xspace}
+\def\smallMergesJavaDiffAndParentsPass{6\xspace}
+\def\smallReposJavaDiffAndParentsPass{4\xspace}
+\def\smallReposSampled{4\xspace}
+\def\smallMergesSampled{6\xspace}
+\def\smallReposYieldedFull{0\xspace}
+\def\smallReposTotal{4\xspace}
+\def\smallMergesTotal{6\xspace}
+
+% Results
+\def\smallAverageTriesUntilPass{1.0\xspace}
+\def\smallNumberofMergesWith1TriesUntilPass{54\xspace}
+\def\smallSporkOverOrtCorrect{3\xspace}
+\def\smallSporkOverOrtIncorrect{0\xspace}
+\def\smallMainBranchMerges{4\xspace}
+\def\smallMainBranchMergesPercent{67\xspace}
+\def\smallOtherBranchMerges{2\xspace}
+\def\smallOtherBranchMergesPercent{33\xspace}
+\def\smallReposJava{4\xspace}
+
+% Timeout
+\def\smallParentTestTimeout{30\xspace}
+\def\smallMergeTestTimeout{45\xspace}

test/small-goal-files/merges_tested/benedikt-schesch/git-hires-merge-example.csv

@@ -1,3 +1,3 @@
-idx,branch_name,merge,left,right,notes,num_diff_files,union_diff_files,num_intersecting_files,num_diff_lines,num_diff_hunks,imports_involved,non_java_involved,diff contains java file,left_tree_fingerprint,left parent test result,right_tree_fingerprint,right parent test result,parents pass,test merge,sampled for testing,gitmerge_ort,gitmerge_ort_merge_fingerprint,gitmerge_ort_ignorespace,gitmerge_ort_ignorespace_merge_fingerprint,gitmerge_recursive_histogram,gitmerge_recursive_histogram_merge_fingerprint,gitmerge_recursive_ignorespace,gitmerge_recursive_ignorespace_merge_fingerprint,gitmerge_recursive_minimal,gitmerge_recursive_minimal_merge_fingerprint,gitmerge_recursive_myers,gitmerge_recursive_myers_merge_fingerprint,gitmerge_recursive_patience,gitmerge_recursive_patience_merge_fingerprint,gitmerge_resolve,gitmerge_resolve_merge_fingerprint,git_hires_merge,git_hires_merge_merge_fingerprint,spork,spork_merge_fingerprint,intellimerge,intellimerge_merge_fingerprint,git_hires_merge_plus,git_hires_merge_plus_merge_fingerprint,intellimerge_plus,intellimerge_plus_merge_fingerprint,gitmerge_recursive_histogram_plus,gitmerge_recursive_histogram_plus_merge_fingerprint,gitmerge_recursive_ignorespace_plus,gitmerge_recursive_ignorespace_plus_merge_fingerprint,gitmerge_recursive_minimal_plus,gitmerge_recursive_minimal_plus_merge_fingerprint,gitmerge_recursive_myers_plus,gitmerge_recursive_myers_plus_merge_fingerprint,gitmerge_recursive_patience_plus,gitmerge_recursive_patience_plus_merge_fingerprint,gitmerge_resolve_plus,gitmerge_resolve_plus_merge_fingerprint,spork_plus,spork_plus_merge_fingerprint,ivn,ivn_merge_fingerprint,ivn_ignorespace,ivn_ignorespace_merge_fingerprint,adjacent,adjacent_merge_fingerprint,imports,imports_merge_fingerprint,version_numbers,version_numbers_merge_fingerprint
-1,refs/heads/main,4ae40d33c3b959e3a15e458eb9b0676251e36a41,48199306f02a82abdeff0c01fab1ce112126d727,488122ce6c91e157d1f49b88b79feaa083fdee5b,,2,2,2,4,1,False,True,True,2252a9465319d00b974abd931cc98e1f435b4fe232b4dd61a828ad40139ec20c,Tests_passed,3782e47be673266f23cba016dc13973231429bb34e4c08a314c5b2f6c37d238f,Tests_passed,True,True,True,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,54749277742f9010b6666756eb77020e2b251b0c575b9ed96020fe650e660c3b,Tests_passed,1cebc99cd5f8a1b210b66b6985f8aa5c38c4af194a32ad23dc3c6261f43d2968,Tests_passed,d04c699fb9748d42ffbbd2a94ae59e522247862c60f3dc29d7c3ed974fc47cb8,Tests_passed,37fdd5a8db81b80bee8cc5b121b9b2f70019498749193c47168e6aa4b86a02e9,Tests_passed,1cebc99cd5f8a1b210b66b6985f8aa5c38c4af194a32ad23dc3c6261f43d2968,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,4abab825dba02ae94f7cc664a590c7c4a57687e2b79fb78d2b92a5d190661e0f,Tests_passed,d04c699fb9748d42ffbbd2a94ae59e522247862c60f3dc29d7c3ed974fc47cb8,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19
-2,refs/remotes/origin/import_1,0209c47c118fcaf978f6ebc5bad8a334b541de0b,b4331092d8c5d1b03e6a9807ab135ed263b52663,8204af9984280ccc7f7793f903fc0807b626bf9c,,2,2,2,5,2,True,True,True,f48eaaf883e49f3e3a1f3e1ddff64beb42c9d9bacec7d273f748baa63093bd9b,Tests_passed,052542ef8f16db238683e23556af730ee0d7c65c3ac0554d01e7ee800ca9b7c6,Tests_passed,True,True,True,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,4c1e23118bc3dc9db98dd8a5d322792c1987f738893ce43bf3032e71d37615a4,Merge_failed,4e0fa6edde284105a006148092586bd2bbce067c56588f4e5c02783786563c3f,Tests_passed,371b1c88034153afe8c3292ba4698b8bea2aa4aea4495b090a68f8f0555290d5,Tests_passed,6f65953e612e19453986bb2b697a79cbab808d3dc01cd3014db987083989a175,Merge_failed,4e0fa6edde284105a006148092586bd2bbce067c56588f4e5c02783786563c3f,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,d5f2cdfd3ff568d6b5f1200f5176590d41ab3ad38f72f5e4f630e3aeb8dc83dd,Tests_passed,371b1c88034153afe8c3292ba4698b8bea2aa4aea4495b090a68f8f0555290d5,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d
+idx,branch_name,merge,left,right,notes,num_diff_files,union_diff_files,num_intersecting_files,num_diff_lines,num_diff_hunks,imports_involved,non_java_involved,diff contains java file,left_tree_fingerprint,left parent test result,right_tree_fingerprint,right parent test result,parents pass,test merge,sampled for testing,gitmerge_ort,gitmerge_ort_merge_fingerprint,gitmerge_ort_ignorespace,gitmerge_ort_ignorespace_merge_fingerprint,gitmerge_recursive_histogram,gitmerge_recursive_histogram_merge_fingerprint,gitmerge_recursive_ignorespace,gitmerge_recursive_ignorespace_merge_fingerprint,gitmerge_recursive_minimal,gitmerge_recursive_minimal_merge_fingerprint,gitmerge_recursive_myers,gitmerge_recursive_myers_merge_fingerprint,gitmerge_recursive_patience,gitmerge_recursive_patience_merge_fingerprint,gitmerge_resolve,gitmerge_resolve_merge_fingerprint,git_hires_merge,git_hires_merge_merge_fingerprint,spork,spork_merge_fingerprint,intellimerge,intellimerge_merge_fingerprint,adjacent,adjacent_merge_fingerprint,imports,imports_merge_fingerprint,version_numbers,version_numbers_merge_fingerprint,git_hires_merge_plus,git_hires_merge_plus_merge_fingerprint,intellimerge_plus,intellimerge_plus_merge_fingerprint,gitmerge_recursive_histogram_plus,gitmerge_recursive_histogram_plus_merge_fingerprint,gitmerge_recursive_ignorespace_plus,gitmerge_recursive_ignorespace_plus_merge_fingerprint,gitmerge_recursive_minimal_plus,gitmerge_recursive_minimal_plus_merge_fingerprint,gitmerge_recursive_myers_plus,gitmerge_recursive_myers_plus_merge_fingerprint,gitmerge_recursive_patience_plus,gitmerge_recursive_patience_plus_merge_fingerprint,gitmerge_resolve_plus,gitmerge_resolve_plus_merge_fingerprint,spork_plus,spork_plus_merge_fingerprint,ivn,ivn_merge_fingerprint,ivn_ignorespace,ivn_ignorespace_merge_fingerprint
+1,refs/heads/main,4ae40d33c3b959e3a15e458eb9b0676251e36a41,48199306f02a82abdeff0c01fab1ce112126d727,488122ce6c91e157d1f49b88b79feaa083fdee5b,,2,2,2,4,1,False,True,True,2252a9465319d00b974abd931cc98e1f435b4fe232b4dd61a828ad40139ec20c,Tests_passed,3782e47be673266f23cba016dc13973231429bb34e4c08a314c5b2f6c37d238f,Tests_passed,True,True,True,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,54749277742f9010b6666756eb77020e2b251b0c575b9ed96020fe650e660c3b,Tests_passed,1cebc99cd5f8a1b210b66b6985f8aa5c38c4af194a32ad23dc3c6261f43d2968,Tests_passed,d04c699fb9748d42ffbbd2a94ae59e522247862c60f3dc29d7c3ed974fc47cb8,Tests_passed,37fdd5a8db81b80bee8cc5b121b9b2f70019498749193c47168e6aa4b86a02e9,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Tests_passed,1cebc99cd5f8a1b210b66b6985f8aa5c38c4af194a32ad23dc3c6261f43d2968,Tests_passed,37fdd5a8db81b80bee8cc5b121b9b2f70019498749193c47168e6aa4b86a02e9,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,322ae7a63a66cfd25604bb69a31df5243952a4a71df5dc68d8a63f49947fcb45,Tests_passed,d04c699fb9748d42ffbbd2a94ae59e522247862c60f3dc29d7c3ed974fc47cb8,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19,Merge_failed,66544c265579fd37b14384c25b569fb765721d9b8046fd6e4574d4b04ce68d19
+2,refs/remotes/origin/import_1,0209c47c118fcaf978f6ebc5bad8a334b541de0b,b4331092d8c5d1b03e6a9807ab135ed263b52663,8204af9984280ccc7f7793f903fc0807b626bf9c,,2,2,2,5,2,True,True,True,f48eaaf883e49f3e3a1f3e1ddff64beb42c9d9bacec7d273f748baa63093bd9b,Tests_passed,052542ef8f16db238683e23556af730ee0d7c65c3ac0554d01e7ee800ca9b7c6,Tests_passed,True,True,True,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,4c1e23118bc3dc9db98dd8a5d322792c1987f738893ce43bf3032e71d37615a4,Merge_failed,4e0fa6edde284105a006148092586bd2bbce067c56588f4e5c02783786563c3f,Tests_passed,371b1c88034153afe8c3292ba4698b8bea2aa4aea4495b090a68f8f0555290d5,Tests_passed,6f65953e612e19453986bb2b697a79cbab808d3dc01cd3014db987083989a175,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,4e0fa6edde284105a006148092586bd2bbce067c56588f4e5c02783786563c3f,Tests_passed,6f65953e612e19453986bb2b697a79cbab808d3dc01cd3014db987083989a175,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,b252f03fe9869202f0a8b7a8f7cb3b6584372def1d29b948201cd1e24769802d,Tests_passed,371b1c88034153afe8c3292ba4698b8bea2aa4aea4495b090a68f8f0555290d5,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d,Merge_failed,69e9fb68947b80201eb2a238ed6e463c457e27d75b3a7e02a1c27b7157d8ed5d