Reclassify Git unhandled as failures (#333)

Makefile

@@ -187,4 +187,4 @@ tags:
 	etags ${SH_SCRIPTS} ${BASH_SCRIPTS} ${PYTHON_FILES}
 
 run:
-	nice -n 5 sh run_full.sh | tee output.txt
+	nice -n 5 sh run_combined.sh | tee output.txt

README.md

@@ -6,6 +6,10 @@
 
 ## Requirements
 
+### Download the cached data
+
+Download the compressed cached data [here](https://zenodo.org/records/13366866) and put it in the root directory of the project. Be aware the the uncompressed cache size is 84GB as of 09-23-2024.
+
 ### Python
 
 To install all the Python requirements, create a conda or mamba environment:
@@ -82,13 +86,13 @@ The output data appears in `results/small/`.
 To run the stack on all repos:
 
 ```bash
-./run_full.sh
+./run_combined.sh
 ```
 
 To run the stack on all repos and also diff the merges' outputs:
 
 ```bash
-./run_full.sh -d
+./run_combined.sh -d
 ```
 
 This will run the entire code on all the repos and automatically decompress the cache if `cache/` does not exist.
@@ -97,7 +101,7 @@ The final result is found in `results/result.csv`.
 Directory `results/merges` contains all the merges for each repo.
 Directory `results/merges_tested` contains all the merges that have been tested.
 
-To execute `run_full.sh` on multiple machines in parallel create a machine address list in `machines.txt` and run:
+To execute `run_combined.sh` on multiple machines in parallel create a machine address list in `machines.txt` and run:
 
 ```bash
 ./src/scripts/utils/run_multiple_machines.sh main machines.txt <project_path_on_machine>
@@ -109,10 +113,9 @@ If `make small-test` fails in a branch that you wish to merge into the main
 branch, run `make small-test` in the main branch (which should succeed) and also
 in your branch, and investigate the differences.
 
-
 ### Load the stored cache
 
-To decompress the cache run `make decompress-cache`. This is done automatically in `run_full.sh` if `cache/` does not exist.
+To decompress the cache run `make decompress-cache`. This is done automatically in `run_combined.sh` if `cache/` does not exist.
 
 ### Store the cache
 
@@ -144,7 +147,13 @@ To run style checking run `make style`.
 
 * run_small.sh -> This file executes the stack on two repositories.
 
-* run_full.sh -> This file executes the stack on all the repositories.
+* run_combined.sh -> This file executes the stack on all the repositories.
+
+* run_greatest_hits.sh -> This file executes the stack on the greatest hits repositories.
+
+* run_reaper.sh -> This file executes the stack on the reaper repositories.
+
+* run_1000.sh -> This file executes the stack on the 1000 repositories.
 
 * src/ -> contains the following scripts:
 
@@ -228,3 +237,7 @@ To investigate differences between two mergers:
 * run `src/python/utils/select_from_results.py` to create a .csv database containing only the differences.
 * Set `DELETE_WORKDIRS` to `false` in `src/python/variables.py`.
 * run `src/python/replay_merge.py --idx INDEX` (maybe add `-test`) for the index of the merge you are interested in.
+
+## Overwriting results manually
+
+In some cases it might be worth to overwrite the computed results. To do that you should modify the `results/manual_override.csv` file. In that file for the merge you want to overwrite a result of you should include at least the information `repository,merge,left,right` and a new column for the result you want to overwrite. You can overwrite anything you want but if there is a column you don't want to overwrite either do not include that column or leave the entry blanck i.e. `,,`. See the file for an example.

results/combined/tables/all/tool_comparison_table.csv

@@ -0,0 +1,17 @@
+,gitmerge_ort,gitmerge_ort_ignorespace,gitmerge_recursive_histogram,gitmerge_recursive_ignorespace,gitmerge_recursive_minimal,gitmerge_recursive_myers,gitmerge_recursive_patience,gitmerge_resolve,git_hires_merge,spork,intellimerge,plumelib_ort,plumelib_ort_ignorespace,plumelib_ort_adjacent,plumelib_ort_imports,plumelib_ort_version_number
+gitmerge_ort,-,1,0,1,0,0,0,2,0,3,2,0,1,0,0,0
+gitmerge_ort_ignorespace,18,-,18,0,18,18,18,20,14,14,13,15,0,11,18,18
+gitmerge_recursive_histogram,0,1,-,1,0,0,0,2,0,3,2,0,1,0,0,0
+gitmerge_recursive_ignorespace,19,1,19,-,18,18,18,21,14,15,14,15,0,12,18,19
+gitmerge_recursive_minimal,1,2,1,1,-,0,0,2,0,3,3,0,1,1,0,1
+gitmerge_recursive_myers,1,2,1,1,0,-,0,2,0,3,3,0,1,1,0,1
+gitmerge_recursive_patience,1,2,1,1,0,0,-,2,0,3,3,0,1,1,0,1
+gitmerge_resolve,2,3,2,2,1,1,1,-,1,4,4,1,2,2,1,2
+git_hires_merge,56,53,56,52,55,55,55,57,-,25,28,49,48,32,52,56
+spork,289,272,289,271,288,288,286,294,243,-,158,233,211,227,243,289
+intellimerge,926,865,926,867,927,927,925,944,775,451,-,788,745,757,830,926
+plumelib_ort,35,32,35,33,36,36,35,41,30,17,23,-,0,23,12,34
+plumelib_ort_ignorespace,61,43,61,44,62,62,61,67,54,33,39,26,-,43,38,60
+plumelib_ort_adjacent,73,67,73,67,73,73,72,75,49,34,40,53,48,-,60,73
+plumelib_ort_imports,23,23,23,24,24,24,23,29,21,12,17,0,0,16,-,23
+plumelib_ort_version_number,1,2,1,2,1,1,1,3,1,4,3,0,1,1,1,-

results/combined/tables/all/tool_comparison_table.tex

@@ -0,0 +1,29 @@
+% Do not edit. This file is automatically generated.
+\begin{table}[h]
+\centering
+\caption{Comparison of Merge Tool Results: Incorrect vs Unhandled}
+\label{tab:tool-comparison}
+\small
+\begin{tabular}{lrrrrrrrrrrrrrrrr}
+\toprule
+& Gitmerge-ort & Gitmerge-ort-ignorespace & Gitmerge-recursive-histogram & Gitmerge-recursive-ignorespace & Gitmerge-recursive-minimal & Gitmerge-recursive-myers & Gitmerge-recursive-patience & Gitmerge-resolve & Hires-Merge & Spork & IntelliMerge & IVn & IVn-ignorespace & Adjacent & Imports & Version Numbers \\
+\midrule
+Gitmerge-ort & - & 1 & 0 & 1 & 0 & 0 & 0 & 2 & 0 & 3 & 2 & 0 & 1 & 0 & 0 & 0 \\
+Gitmerge-ort-ignorespace & 18 & - & 18 & 0 & 18 & 18 & 18 & 20 & 14 & 14 & 13 & 15 & 0 & 11 & 18 & 18 \\
+Gitmerge-recursive-histogram & 0 & 1 & - & 1 & 0 & 0 & 0 & 2 & 0 & 3 & 2 & 0 & 1 & 0 & 0 & 0 \\
+Gitmerge-recursive-ignorespace & 19 & 1 & 19 & - & 18 & 18 & 18 & 21 & 14 & 15 & 14 & 15 & 0 & 12 & 18 & 19 \\
+Gitmerge-recursive-minimal & 1 & 2 & 1 & 1 & - & 0 & 0 & 2 & 0 & 3 & 3 & 0 & 1 & 1 & 0 & 1 \\
+Gitmerge-recursive-myers & 1 & 2 & 1 & 1 & 0 & - & 0 & 2 & 0 & 3 & 3 & 0 & 1 & 1 & 0 & 1 \\
+Gitmerge-recursive-patience & 1 & 2 & 1 & 1 & 0 & 0 & - & 2 & 0 & 3 & 3 & 0 & 1 & 1 & 0 & 1 \\
+Gitmerge-resolve & 2 & 3 & 2 & 2 & 1 & 1 & 1 & - & 1 & 4 & 4 & 1 & 2 & 2 & 1 & 2 \\
+Hires-Merge & 56 & 53 & 56 & 52 & 55 & 55 & 55 & 57 & - & 25 & 28 & 49 & 48 & 32 & 52 & 56 \\
+Spork & 289 & 272 & 289 & 271 & 288 & 288 & 286 & 294 & 243 & - & 158 & 233 & 211 & 227 & 243 & 289 \\
+IntelliMerge & 926 & 865 & 926 & 867 & 927 & 927 & 925 & 944 & 775 & 451 & - & 788 & 745 & 757 & 830 & 926 \\
+IVn & 35 & 32 & 35 & 33 & 36 & 36 & 35 & 41 & 30 & 17 & 23 & - & 0 & 23 & 12 & 34 \\
+IVn-ignorespace & 61 & 43 & 61 & 44 & 62 & 62 & 61 & 67 & 54 & 33 & 39 & 26 & - & 43 & 38 & 60 \\
+Adjacent & 73 & 67 & 73 & 67 & 73 & 73 & 72 & 75 & 49 & 34 & 40 & 53 & 48 & - & 60 & 73 \\
+Imports & 23 & 23 & 23 & 24 & 24 & 24 & 23 & 29 & 21 & 12 & 17 & 0 & 0 & 16 & - & 23 \\
+Version Numbers & 1 & 2 & 1 & 2 & 1 & 1 & 1 & 3 & 1 & 4 & 3 & 0 & 1 & 1 & 1 & - \\
+\bottomrule
+\end{tabular}
+\end{table}

results/combined/tables/git/tool_comparison_table.csv

@@ -0,0 +1,8 @@
+,gitmerge_ort,gitmerge_ort_ignorespace,gitmerge_recursive_histogram,gitmerge_recursive_minimal,gitmerge_recursive_myers,gitmerge_recursive_patience,gitmerge_resolve
+gitmerge_ort,-,1,0,0,0,0,2
+gitmerge_ort_ignorespace,18,-,18,18,18,18,20
+gitmerge_recursive_histogram,0,1,-,0,0,0,2
+gitmerge_recursive_minimal,1,2,1,-,0,0,2
+gitmerge_recursive_myers,1,2,1,0,-,0,2
+gitmerge_recursive_patience,1,2,1,0,0,-,2
+gitmerge_resolve,2,3,2,1,1,1,-

results/combined/tables/git/tool_comparison_table.tex

@@ -0,0 +1,20 @@
+% Do not edit. This file is automatically generated.
+\begin{table}[h]
+\centering
+\caption{Comparison of Merge Tool Results: Incorrect vs Unhandled}
+\label{tab:tool-comparison}
+\small
+\begin{tabular}{lrrrrrrr}
+\toprule
+& Gitmerge-ort & Gitmerge-ort-ignorespace & Gitmerge-recursive-histogram & Gitmerge-recursive-minimal & Gitmerge-recursive-myers & Gitmerge-recursive-patience & Gitmerge-resolve \\
+\midrule
+Gitmerge-ort & - & 1 & 0 & 0 & 0 & 0 & 2 \\
+Gitmerge-ort-ignorespace & 18 & - & 18 & 18 & 18 & 18 & 20 \\
+Gitmerge-recursive-histogram & 0 & 1 & - & 0 & 0 & 0 & 2 \\
+Gitmerge-recursive-minimal & 1 & 2 & 1 & - & 0 & 0 & 2 \\
+Gitmerge-recursive-myers & 1 & 2 & 1 & 0 & - & 0 & 2 \\
+Gitmerge-recursive-patience & 1 & 2 & 1 & 0 & 0 & - & 2 \\
+Gitmerge-resolve & 2 & 3 & 2 & 1 & 1 & 1 & - \\
+\bottomrule
+\end{tabular}
+\end{table}

results/combined/tables/tools/tool_comparison_table.csv

@@ -0,0 +1,11 @@
+,gitmerge_ort,gitmerge_ort_ignorespace,git_hires_merge,spork,intellimerge,plumelib_ort_adjacent,plumelib_ort_imports,plumelib_ort_version_number,plumelib_ort,plumelib_ort_ignorespace
+gitmerge_ort,-,1,0,3,2,0,0,0,0,1
+gitmerge_ort_ignorespace,18,-,14,14,13,11,18,18,15,0
+git_hires_merge,56,53,-,25,28,32,52,56,49,48
+spork,289,272,243,-,158,227,243,289,233,211
+intellimerge,926,865,775,451,-,757,830,926,788,745
+plumelib_ort_adjacent,73,67,49,34,40,-,60,73,53,48
+plumelib_ort_imports,23,23,21,12,17,16,-,23,0,0
+plumelib_ort_version_number,1,2,1,4,3,1,1,-,0,1
+plumelib_ort,35,32,30,17,23,23,12,34,-,0
+plumelib_ort_ignorespace,61,43,54,33,39,43,38,60,26,-

results/combined/tables/tools/tool_comparison_table.tex

@@ -0,0 +1,23 @@
+% Do not edit. This file is automatically generated.
+\begin{table}[h]
+\centering
+\caption{Comparison of Merge Tool Results: Incorrect vs Unhandled}
+\label{tab:tool-comparison}
+\small
+\begin{tabular}{lrrrrrrrrrr}
+\toprule
+& Gitmerge-ort & Gitmerge-ort-ignorespace & Hires-Merge & Spork & IntelliMerge & Adjacent & Imports & Version Numbers & IVn & IVn-ignorespace \\
+\midrule
+Gitmerge-ort & - & 1 & 0 & 3 & 2 & 0 & 0 & 0 & 0 & 1 \\
+Gitmerge-ort-ignorespace & 18 & - & 14 & 14 & 13 & 11 & 18 & 18 & 15 & 0 \\
+Hires-Merge & 56 & 53 & - & 25 & 28 & 32 & 52 & 56 & 49 & 48 \\
+Spork & 289 & 272 & 243 & - & 158 & 227 & 243 & 289 & 233 & 211 \\
+IntelliMerge & 926 & 865 & 775 & 451 & - & 757 & 830 & 926 & 788 & 745 \\
+Adjacent & 73 & 67 & 49 & 34 & 40 & - & 60 & 73 & 53 & 48 \\
+Imports & 23 & 23 & 21 & 12 & 17 & 16 & - & 23 & 0 & 0 \\
+Version Numbers & 1 & 2 & 1 & 4 & 3 & 1 & 1 & - & 0 & 1 \\
+IVn & 35 & 32 & 30 & 17 & 23 & 23 & 12 & 34 & - & 0 \\
+IVn-ignorespace & 61 & 43 & 54 & 33 & 39 & 43 & 38 & 60 & 26 & - \\
+\bottomrule
+\end{tabular}
+\end{table}