Merge ../AST-Merging-Evaluation-branch-main into exit-status

.github/workflows/check-style.yml

@@ -24,4 +24,4 @@ jobs:
     - name: Install shellcheck and checkbashisms
       run: sudo apt update && sudo apt install shellcheck devscripts
     - name: Check style
-      run: make style
+      run: make check-style

.gitignore

@@ -17,6 +17,10 @@ cache.tar
 output.txt
 replay_logs/
 my.secrets
+.nix/
+.direnv/
+.idea/
+.envrc
 
 repos-small-test
 output/

Makefile

@@ -1,6 +1,8 @@
 all: style gradle-assemble
 
-style: shell-script-style python-style java-style
+fix-style: fix-python-style fix-java-style
+
+check-style: check-shell-script-style check-python-style check-java-style
 
 SH_SCRIPTS   = $(shell grep --exclude-dir=build --exclude-dir=repos --exclude-dir=cache -r -l '^\#! \?\(/bin/\|/usr/bin/env \)sh'   * | grep -v 'git-hires-merge' | grep -v /.git/ | grep -v '~$$' | grep -v '\.tar$$' | grep -v gradlew)
 BASH_SCRIPTS = $(shell grep --exclude-dir=build --exclude-dir=repos --exclude-dir=cache -r -l '^\#! \?\(/bin/\|/usr/bin/env \)bash' * | grep -v /.git/ | grep -v '~$$' | grep -v '\.tar$$' | grep -v gradlew)
@@ -13,23 +15,30 @@ CSV_RESULTS = $(CSV_RESULTS_COMBINED)
 
 NUM_PROCESSES = 0
 
-shell-script-style:
-	shellcheck -e SC2153 -x -P SCRIPTDIR --format=gcc ${SH_SCRIPTS} ${BASH_SCRIPTS}
-	checkbashisms ${SH_SCRIPTS}
-
 showvars:
 	@echo "SH_SCRIPTS=${SH_SCRIPTS}"
 	@echo "BASH_SCRIPTS=${BASH_SCRIPTS}"
 	@echo "PYTHON_FILES=${PYTHON_FILES}"
 
-python-style:
+check-shell-script-style:
+	shellcheck -e SC2153 -x -P SCRIPTDIR --format=gcc ${SH_SCRIPTS} ${BASH_SCRIPTS}
+	checkbashisms ${SH_SCRIPTS}
+
+fix-python-style:
 	ruff format ${PYTHON_FILES}
 	ruff check ${PYTHON_FILES} --fix
 
 check-python-style:
 	ruff format ${PYTHON_FILES} --check
 	ruff check ${PYTHON_FILES}
 
+fix-java-style:
+	./gradlew -q spotlessApply -g ../.gradle/
+
+check-java-style:
+	./gradlew -q spotlessCheck javadoc requireJavadoc -g ../.gradle/
+
+
 # This target deletes files that are not committed to version control.
 clean:
 	${MAKE} clean-workdir
@@ -117,6 +126,9 @@ update-figures:
 	./run_greatest_hits.sh -op --no_timing
 	./run_reaper.sh -op --no_timing
 
+update-figures-small:
+	AST_REPOS_PATH=repos-small-test ./run_small.sh -op --no_timing
+
 run-all-without-timing:
 	${MAKE} clean-workdir
 	${MAKE} small-test-without-cleaning
@@ -160,9 +172,6 @@ check-merges-reproducibility:
 protect-repos:
 	find repos -mindepth 1 -type d -exec chmod a-w {} +
 
-java-style:
-	./gradlew -q spotlessCheck javadoc requireJavadoc -g ../.gradle/
-
 download-merge-tools: jars/IntelliMerge-1.0.9-all.jar jars/spork.jar
 
 jars/IntelliMerge-1.0.9-all.jar:
@@ -178,4 +187,4 @@ tags:
 	etags ${SH_SCRIPTS} ${BASH_SCRIPTS} ${PYTHON_FILES}
 
 run:
-	nice -n 5 sh run_full.sh | tee output.txt
+	nice -n 5 sh run_combined.sh | tee output.txt

README.md

@@ -6,6 +6,10 @@
 
 ## Requirements
 
+### Download the cached data
+
+Download the compressed cached data [here](https://zenodo.org/records/13366866) and put it in the root directory of the project. Be aware the the uncompressed cache size is 84GB as of 09-23-2024.
+
 ### Python
 
 To install all the Python requirements, create a conda or mamba environment:
@@ -82,13 +86,13 @@ The output data appears in `results/small/`.
 To run the stack on all repos:
 
 ```bash
-./run_full.sh
+./run_combined.sh
 ```
 
 To run the stack on all repos and also diff the merges' outputs:
 
 ```bash
-./run_full.sh -d
+./run_combined.sh -d
 ```
 
 This will run the entire code on all the repos and automatically decompress the cache if `cache/` does not exist.
@@ -97,7 +101,7 @@ The final result is found in `results/result.csv`.
 Directory `results/merges` contains all the merges for each repo.
 Directory `results/merges_tested` contains all the merges that have been tested.
 
-To execute `run_full.sh` on multiple machines in parallel create a machine address list in `machines.txt` and run:
+To execute `run_combined.sh` on multiple machines in parallel create a machine address list in `machines.txt` and run:
 
 ```bash
 ./src/scripts/utils/run_multiple_machines.sh main machines.txt <project_path_on_machine>
@@ -109,10 +113,9 @@ If `make small-test` fails in a branch that you wish to merge into the main
 branch, run `make small-test` in the main branch (which should succeed) and also
 in your branch, and investigate the differences.
 
-
 ### Load the stored cache
 
-To decompress the cache run `make decompress-cache`. This is done automatically in `run_full.sh` if `cache/` does not exist.
+To decompress the cache run `make decompress-cache`. This is done automatically in `run_combined.sh` if `cache/` does not exist.
 
 ### Store the cache
 
@@ -144,7 +147,13 @@ To run style checking run `make style`.
 
 * run_small.sh -> This file executes the stack on two repositories.
 
-* run_full.sh -> This file executes the stack on all the repositories.
+* run_combined.sh -> This file executes the stack on all the repositories.
+
+* run_greatest_hits.sh -> This file executes the stack on the greatest hits repositories.
+
+* run_reaper.sh -> This file executes the stack on the reaper repositories.
+
+* run_1000.sh -> This file executes the stack on the 1000 repositories.
 
 * src/ -> contains the following scripts:
 
@@ -188,6 +197,8 @@ To run style checking run `make style`.
 
       * run_multiple_machine.sh -> Runs the full stack on multiple remote machines.
 
+      * diff_statistics.py -> Various diff commands to compare the results of two merges.
+
   * src/main/java/astmergeevaluation/FindMergeCommits.java -> Finds all merge commits in a repo.
 
 * input_data/ -> Input data, which is a list of repositories; see its README.md.
@@ -226,3 +237,7 @@ To investigate differences between two mergers:
 * run `src/python/utils/select_from_results.py` to create a .csv database containing only the differences.
 * Set `DELETE_WORKDIRS` to `false` in `src/python/variables.py`.
 * run `src/python/replay_merge.py --idx INDEX` (maybe add `-test`) for the index of the merge you are interested in.
+
+## Overwriting results manually
+
+In some cases it might be worth to overwrite the computed results. To do that you should modify the `results/manual_override.csv` file. In that file for the merge you want to overwrite a result of you should include at least the information `repository,merge,left,right` and a new column for the result you want to overwrite. You can overwrite anything you want but if there is a column you don't want to overwrite either do not include that column or leave the entry blanck i.e. `,,`. See the file for an example.

results/combined/defs.tex

@@ -15,6 +15,12 @@
 \def\combinedMergesTotal{5999\xspace}
 
 % Results
+\def\combinedAverageTriesUntilPass{1.0149383127709237\xspace}
+\def\combinedNumberofMergesWith1TriesUntilPass{44569\xspace}
+\def\combinedNumberofMergesWith3TriesUntilPass{99\xspace}
+\def\combinedNumberofMergesWith5TriesUntilPass{35\xspace}
+\def\combinedNumberofMergesWith2TriesUntilPass{256\xspace}
+\def\combinedNumberofMergesWith4TriesUntilPass{26\xspace}
 \def\combinedSporkOverOrtCorrect{515\xspace}
 \def\combinedSporkOverOrtIncorrect{487\xspace}
 \def\combinedMainBranchMerges{3532\xspace}

results/combined/merges_analyzed/01org/graphbuilder.csv

@@ -1,3 +1,3 @@
-idx,branch_name,merge,left,right,notes,test merge,diff contains java file,sampled for testing
-1,refs/remotes/origin/pull/2,34ad099f9deda4bca61c92b4bfa29e8179b77ba9,4aa0b8450b9beb637b550633b0e56352276f1836,ec9cdb514c3bc9a454bd183b3ff0588dfc2c5434,,False,False,False
-2,refs/remotes/origin/pull/2,4aa0b8450b9beb637b550633b0e56352276f1836,fb01619e4e99f20ad5f6dd7eed7d3dfe44e4375b,d633ba404308022cca0cebe7b905c2dae39494ee,,False,False,False
+idx,branch_name,merge,left,right,notes,diff contains java file,imports_involved,non_java_involved,num_diff_files,num_diff_hunks,num_diff_lines,num_intersecting_files,test merge,union_diff_files,sampled for testing
+1,refs/remotes/origin/pull/2,34ad099f9deda4bca61c92b4bfa29e8179b77ba9,4aa0b8450b9beb637b550633b0e56352276f1836,ec9cdb514c3bc9a454bd183b3ff0588dfc2c5434,,False,True,True,24,200,4209,2,False,24,False
+2,refs/remotes/origin/pull/2,4aa0b8450b9beb637b550633b0e56352276f1836,fb01619e4e99f20ad5f6dd7eed7d3dfe44e4375b,d633ba404308022cca0cebe7b905c2dae39494ee,,False,True,True,1434,1328,196808,3,False,1434,False

results/combined/merges_analyzed/3breadt/dd-plist.csv

@@ -1,12 +1,12 @@
-idx,branch_name,merge,left,right,notes,test merge,diff contains java file,sampled for testing
-1,refs/heads/master,fc51e4778e650af221994f5f8c6db4b68fda0e0b,36f98160266bff7b262fe435ecd6747e36490c02,91f8ba7fd92a9a503484a540e5fd3e03f411d98a,,False,False,False
-2,refs/heads/master,743ffecd4c389f5476c8d1a2285d515a384d3b86,c3117754c4fe4cfd380c352e0ac1ae9cf56425d9,537a2177825470d7d6a0c962739545c900c37582,,False,False,False
-3,refs/heads/master,2c8c164d66272b00ac0684577b095b1aa20a0f81,2120e77a988d28fba214714710a43521b4ac62b8,101e5a636d7b59c2dade073645a075b6eeee9614,,False,False,False
-5,refs/heads/master,b9be0f46797aab66cf5302584d69bd62a00c8e64,69189ffa141974628e9c0b547e8d3c9ebd322e53,12602e55c0997aa55903f1f86fd6f3f6331549a8,,False,False,False
-7,refs/heads/master,a78d2509870567ef42bc2edea887a9e10ccaeff4,0f6cbd92ebdf646db29ec938192319147026b8c3,9d891d285bb3fab4e893c2b2c9145e7c7ba8615a,,False,False,False
-12,refs/heads/master,a1aa6645688f8a150ba3f78caa7e4feeb2293b3d,138da8ea7813113f0b22175afd18bca8ef2d7984,36d3419679bacfd3121d541f0732098de30c7357,,False,False,False
-18,refs/heads/master,9e1a7d997a274ded5456c85d0c91e4ce1dc7af79,15482f0821c80781817975b266cfecc74ede2cd2,a599fad535fa6d8ef91e8418eea9810301de3290,,False,False,False
-20,refs/remotes/origin/pull/27,31e4d0e7165c62c8e776f4a14845def3e53f876f,65e704efb8a695efd08ca2a996ddd6c03c469e63,2c8c164d66272b00ac0684577b095b1aa20a0f81,,False,False,False
-21,refs/remotes/origin/pull/30,012791e428d9a5cad23dc001a9c42aee4121d764,31e4d0e7165c62c8e776f4a14845def3e53f876f,297614189d591b2fa82205b299a02558b124a839,,False,False,False
-22,refs/remotes/origin/pull/56,da735ab2a75ca50a9ea77441997ceb8cac2a53a3,5df44adc532b16b80998efb5c1082ed699801bc5,1c06f944907022a9e6d9af0ad40a85dc2f2f44bb,,False,False,False
-23,refs/remotes/origin/pull/56,5df44adc532b16b80998efb5c1082ed699801bc5,f6911b193a27eb940dfd40034453dec2dd361369,f96e82d6420867d010687a220a90107778c74da9,,False,False,False
+idx,branch_name,merge,left,right,notes,diff contains java file,imports_involved,non_java_involved,num_diff_files,num_diff_hunks,num_diff_lines,num_intersecting_files,test merge,union_diff_files,sampled for testing
+1,refs/heads/master,fc51e4778e650af221994f5f8c6db4b68fda0e0b,36f98160266bff7b262fe435ecd6747e36490c02,91f8ba7fd92a9a503484a540e5fd3e03f411d98a,,False,True,True,9,25,97,1,False,9,False
+2,refs/heads/master,743ffecd4c389f5476c8d1a2285d515a384d3b86,c3117754c4fe4cfd380c352e0ac1ae9cf56425d9,537a2177825470d7d6a0c962739545c900c37582,,False,True,True,6,98,389,1,False,6,False
+3,refs/heads/master,2c8c164d66272b00ac0684577b095b1aa20a0f81,2120e77a988d28fba214714710a43521b4ac62b8,101e5a636d7b59c2dade073645a075b6eeee9614,,False,True,True,6,31,229,1,False,6,False
+5,refs/heads/master,b9be0f46797aab66cf5302584d69bd62a00c8e64,69189ffa141974628e9c0b547e8d3c9ebd322e53,12602e55c0997aa55903f1f86fd6f3f6331549a8,,False,True,True,9,12,44,1,False,9,False
+7,refs/heads/master,a78d2509870567ef42bc2edea887a9e10ccaeff4,0f6cbd92ebdf646db29ec938192319147026b8c3,9d891d285bb3fab4e893c2b2c9145e7c7ba8615a,,False,True,True,16,25,1797,1,False,16,False
+12,refs/heads/master,a1aa6645688f8a150ba3f78caa7e4feeb2293b3d,138da8ea7813113f0b22175afd18bca8ef2d7984,36d3419679bacfd3121d541f0732098de30c7357,,False,False,True,3,9,40,1,False,3,False
+18,refs/heads/master,9e1a7d997a274ded5456c85d0c91e4ce1dc7af79,15482f0821c80781817975b266cfecc74ede2cd2,a599fad535fa6d8ef91e8418eea9810301de3290,,False,True,True,19,44,540,1,False,19,False
+20,refs/remotes/origin/pull/27,31e4d0e7165c62c8e776f4a14845def3e53f876f,65e704efb8a695efd08ca2a996ddd6c03c469e63,2c8c164d66272b00ac0684577b095b1aa20a0f81,,False,True,True,5,21,127,1,False,5,False
+21,refs/remotes/origin/pull/30,012791e428d9a5cad23dc001a9c42aee4121d764,31e4d0e7165c62c8e776f4a14845def3e53f876f,297614189d591b2fa82205b299a02558b124a839,,False,False,False,0,Error,0,2,False,2,False
+22,refs/remotes/origin/pull/56,da735ab2a75ca50a9ea77441997ceb8cac2a53a3,5df44adc532b16b80998efb5c1082ed699801bc5,1c06f944907022a9e6d9af0ad40a85dc2f2f44bb,,False,True,True,92,850,22211,0,False,92,False
+23,refs/remotes/origin/pull/56,5df44adc532b16b80998efb5c1082ed699801bc5,f6911b193a27eb940dfd40034453dec2dd361369,f96e82d6420867d010687a220a90107778c74da9,,False,False,False,0,Error,0,2,False,2,False

results/combined/merges_analyzed/3scale/3scale_ws_api_for_java.csv

@@ -1,9 +1,9 @@
-idx,branch_name,merge,left,right,notes,test merge,diff contains java file,left_tree_fingerprint,left parent test result,right_tree_fingerprint,right parent test result,parents pass,sampled for testing
-12,refs/remotes/origin/pull/10,c77b1297186f967622adaeaaa8996a27f77cf1fc,3c9c2af6b97384850a9acd7245d7c62ec7afd593,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,False,True,29f6d014333b808c5b2979ba16de7e6bac963e80ad0cba35ae14faa710e7c431,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,Tests_failed,False,False
-13,refs/remotes/origin/pull/10,d5be99e116542e2f465caba331c0ec589bf726d9,c68f61f0fa127bfeaea5e4e255e7b3730df4a0f6,f85b6777113505109cc1f1d4a9edea615fd403f5,,False,False,,,,,,False
-14,refs/remotes/origin/pull/10,ea282678896734ff75f85799cfd7ae3f3515790c,ea143d850887e020a5f505f0917da33cf186af75,94607fd9ce59988a15dca58958c7ae30bf51155f,,False,False,,,,,,False
-15,refs/remotes/origin/pull/10,c3b5f5d43f5699e2e4d698bd32a74dad9ac1d477,78eceb43ad7a9dd89e7c2f36f82b0e36061e40ff,85d75a23e94054bbb8c219f21036872149150a90,,False,True,257e4c08564208c7ebf0f6e8185853cf539495c4508abc372e8e39cbae743174,Tests_failed,9d947990e32bc40c172cff6bdce6a3e34ff00f241abe661da55bae8a1e399000,Tests_failed,False,False
-16,refs/remotes/origin/pull/10,d0c36bece9c7b6409a6d57008f327287bfdffa06,d2ddc1e3a56d5b420971b107bdcd3b99bd48d0ef,799779c8c760115a7e8db66d8c24bfccd5a55416,,False,False,,,,,,False
-17,refs/remotes/origin/pull/10,332b6ed4e324199565e235cd915368f9bc1b43d2,4650578dee712b2b08f2ead2bf6a531f82b1e0e9,d623f491daa2f14f06f53338265d28a489138a6b,,False,True,78bd4536994ebea95ea8ebd2ea2e7c2b38f2a1fec25ecf322a0e8ccd1406b50b,Tests_failed,aa6ad3f4018260b629417aa44faa0ed893f6092c9f5dd08efdae1fd8223de2ba,Tests_failed,False,False
-18,refs/remotes/origin/pull/11,9616295df7f50bfc3ec267047e569e83b6cdb0fa,5ac6e3f5b0c069465004538a3b95a37fcaf7b57e,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,False,True,d91c6fc51df617b83f7695a8eee5308421e57fe98f4a0c678e994c79904f69f0,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,Tests_failed,False,False
-20,refs/remotes/origin/pull/13,3054b1c14ba79e67394915d27d5e5d766f3b3db0,f39a51f3fa3259bd4cabd16fe9c025fe4dbbea1f,3c11b36e5054ec09cdc29ec3d0e9273406163498,,False,False,,,,,,False
+idx,branch_name,merge,left,right,notes,diff contains java file,imports_involved,non_java_involved,num_diff_files,num_diff_hunks,num_diff_lines,num_intersecting_files,test merge,union_diff_files,left parent test result,left_tree_fingerprint,parents pass,right parent test result,right_tree_fingerprint,sampled for testing
+12,refs/remotes/origin/pull/10,c77b1297186f967622adaeaaa8996a27f77cf1fc,3c9c2af6b97384850a9acd7245d7c62ec7afd593,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,True,True,True,10,29,266,4,False,10,Tests_failed,29f6d014333b808c5b2979ba16de7e6bac963e80ad0cba35ae14faa710e7c431,False,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,False
+13,refs/remotes/origin/pull/10,d5be99e116542e2f465caba331c0ec589bf726d9,c68f61f0fa127bfeaea5e4e255e7b3730df4a0f6,f85b6777113505109cc1f1d4a9edea615fd403f5,,False,True,True,6,11,193,1,False,6,,,,,,False
+14,refs/remotes/origin/pull/10,ea282678896734ff75f85799cfd7ae3f3515790c,ea143d850887e020a5f505f0917da33cf186af75,94607fd9ce59988a15dca58958c7ae30bf51155f,,False,True,True,6,8,47,1,False,6,,,,,,False
+15,refs/remotes/origin/pull/10,c3b5f5d43f5699e2e4d698bd32a74dad9ac1d477,78eceb43ad7a9dd89e7c2f36f82b0e36061e40ff,85d75a23e94054bbb8c219f21036872149150a90,,True,True,True,275,1285,43918,4,False,275,Tests_failed,257e4c08564208c7ebf0f6e8185853cf539495c4508abc372e8e39cbae743174,False,Tests_failed,9d947990e32bc40c172cff6bdce6a3e34ff00f241abe661da55bae8a1e399000,False
+16,refs/remotes/origin/pull/10,d0c36bece9c7b6409a6d57008f327287bfdffa06,d2ddc1e3a56d5b420971b107bdcd3b99bd48d0ef,799779c8c760115a7e8db66d8c24bfccd5a55416,,False,False,False,0,Error,0,210,False,210,,,,,,False
+17,refs/remotes/origin/pull/10,332b6ed4e324199565e235cd915368f9bc1b43d2,4650578dee712b2b08f2ead2bf6a531f82b1e0e9,d623f491daa2f14f06f53338265d28a489138a6b,,True,True,True,112,200,4448,4,False,113,Tests_failed,78bd4536994ebea95ea8ebd2ea2e7c2b38f2a1fec25ecf322a0e8ccd1406b50b,False,Tests_failed,aa6ad3f4018260b629417aa44faa0ed893f6092c9f5dd08efdae1fd8223de2ba,False
+18,refs/remotes/origin/pull/11,9616295df7f50bfc3ec267047e569e83b6cdb0fa,5ac6e3f5b0c069465004538a3b95a37fcaf7b57e,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,True,True,True,9,31,119,3,False,9,Tests_failed,d91c6fc51df617b83f7695a8eee5308421e57fe98f4a0c678e994c79904f69f0,False,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,False
+20,refs/remotes/origin/pull/13,3054b1c14ba79e67394915d27d5e5d766f3b3db0,f39a51f3fa3259bd4cabd16fe9c025fe4dbbea1f,3c11b36e5054ec09cdc29ec3d0e9273406163498,,False,True,True,10,108,1126,2,False,10,,,,,,False

results/combined/merges_analyzed/42lines/wicket-bean-validation.csv

@@ -1,2 +1,2 @@
-idx,branch_name,merge,left,right,notes,test merge,diff contains java file,sampled for testing
-1,refs/heads/master,ab027c7dd3c2d43b9e8482c9f7985fed29a566a8,e294b31dc730f1165ee3036c2014c069eb92c5db,691b58657341c29c8ba7744aa1e731da712cbe7a,,False,False,False
+idx,branch_name,merge,left,right,notes,diff contains java file,imports_involved,non_java_involved,num_diff_files,num_diff_hunks,num_diff_lines,num_intersecting_files,test merge,union_diff_files,sampled for testing
+1,refs/heads/master,ab027c7dd3c2d43b9e8482c9f7985fed29a566a8,e294b31dc730f1165ee3036c2014c069eb92c5db,691b58657341c29c8ba7744aa1e731da712cbe7a,,False,False,True,3,4,14,1,False,3,False