Check merges reproducibility (#283)

.github/workflows/small-test.yml

@@ -53,6 +53,7 @@ jobs:
         maven-version: ${{ matrix.maven }}
     - name: Clean caches & workspace
       run: make clean
+    - run: echo "LOGURU_COLORIZE=NO" >> $GITHUB_ENV
     - name: Run small test
       run: |
         git config --global user.email "[email protected]"

Makefile

@@ -6,6 +6,11 @@ SH_SCRIPTS   = $(shell grep --exclude-dir=build --exclude-dir=repos --exclude-di
 BASH_SCRIPTS = $(shell grep --exclude-dir=build --exclude-dir=repos --exclude-dir=cache -r -l '^\#! \?\(/bin/\|/usr/bin/env \)bash' * | grep -v /.git/ | grep -v '~$$' | grep -v '\.tar$$' | grep -v gradlew)
 PYTHON_FILES = $(shell find .  -name '*.py' ! -path './repos/*' -not -path "./.workdir/*" -not -path "./cache*/*" | grep -v '/__pycache__/' | grep -v '/.git/' | grep -v gradlew)
 
+CSV_RESULTS_COMBINED = results/combined/result.csv
+CSV_RESULTS_GREATEST_HITS = results/greatest_hits/result.csv
+CSV_RESULTS_REAPER = results/reaper/result.csv
+CSV_RESULTS = $(CSV_RESULTS_COMBINED)
+
 shell-script-style:
 	shellcheck -e SC2153 -x -P SCRIPTDIR --format=gcc ${SH_SCRIPTS} ${BASH_SCRIPTS}
 	checkbashisms ${SH_SCRIPTS}
@@ -108,6 +113,7 @@ run-all:
 	./run_greatest_hits.sh
 	./run_reaper.sh
 
+
 small-test-diff:
 	python3 test/check_equal_csv.py --actual_folder results/small/ --goal_folder test/small-goal-files/
 	@echo
@@ -123,6 +129,10 @@ clean-local:
 	${MAKE} clean-workdir
 	rm -rf repos
 
+check-merges-reproducibility:
+	@echo "Running replay_merge for each idx in parallel..."
+	@tail -n +2 $(CSV_RESULTS) | awk -F, '{print $$1}' | parallel -u --halt now,fail=1 -j 0 'python3 src/python/replay_merge.py -delete_workdir --idx {}'
+
 protect-repos:
 	find repos -mindepth 1 -type d -exec chmod a-w {} +
 

results/combined/defs.tex

@@ -4,22 +4,22 @@
 \def\combinedMergesPer{100\xspace}
 \def\combinedMergesNonTrivial{85979\xspace}
 \def\combinedReposNonTrivial{2712\xspace}
-\def\combinedMergesJavaDiff{21971\xspace}
+\def\combinedMergesJavaDiff{21861\xspace}
 \def\combinedReposJavaDiff{1653\xspace}
-\def\combinedMergesJavaDiffAndParentsPass{6106\xspace}
-\def\combinedReposJavaDiffAndParentsPass{1121\xspace}
-\def\combinedReposSampled{1121\xspace}
-\def\combinedMergesSampled{6096\xspace}
-\def\combinedReposYieldedFull{2\xspace}
+\def\combinedMergesJavaDiffAndParentsPass{6050\xspace}
+\def\combinedReposJavaDiffAndParentsPass{1120\xspace}
+\def\combinedReposSampled{1120\xspace}
+\def\combinedMergesSampled{6040\xspace}
+\def\combinedReposYieldedFull{1\xspace}
 \def\combinedReposTotal{1120\xspace}
-\def\combinedMergesTotal{6076\xspace}
+\def\combinedMergesTotal{6040\xspace}
 
 % Results
-\def\combinedSporkOverOrtCorrect{525\xspace}
-\def\combinedSporkOverOrtIncorrect{488\xspace}
-\def\combinedMainBranchMerges{3572\xspace}
+\def\combinedSporkOverOrtCorrect{518\xspace}
+\def\combinedSporkOverOrtIncorrect{487\xspace}
+\def\combinedMainBranchMerges{3557\xspace}
 \def\combinedMainBranchMergesPercent{59\xspace}
-\def\combinedOtherBranceshMerges{2504\xspace}
+\def\combinedOtherBranceshMerges{2483\xspace}
 \def\combinedOtherBranchesMergesPercent{41\xspace}
 \def\combinedReposJava{42092\xspace}
 

results/combined/merges_analyzed/3scale/3scale_ws_api_for_java.csv

@@ -1,9 +1,9 @@
-idx,branch_name,merge,left,right,notes,left_tree_fingerprint,left parent test result,right_tree_fingerprint,right parent test result,parents pass,diff contains java file,test merge,sampled for testing
-12,refs/remotes/origin/pull/10,c77b1297186f967622adaeaaa8996a27f77cf1fc,3c9c2af6b97384850a9acd7245d7c62ec7afd593,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,29f6d014333b808c5b2979ba16de7e6bac963e80ad0cba35ae14faa710e7c431,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,Tests_failed,False,True,False,False
-13,refs/remotes/origin/pull/10,d5be99e116542e2f465caba331c0ec589bf726d9,c68f61f0fa127bfeaea5e4e255e7b3730df4a0f6,f85b6777113505109cc1f1d4a9edea615fd403f5,,,,,,,False,False,False
-14,refs/remotes/origin/pull/10,ea282678896734ff75f85799cfd7ae3f3515790c,ea143d850887e020a5f505f0917da33cf186af75,94607fd9ce59988a15dca58958c7ae30bf51155f,,,,,,,False,False,False
-15,refs/remotes/origin/pull/10,c3b5f5d43f5699e2e4d698bd32a74dad9ac1d477,78eceb43ad7a9dd89e7c2f36f82b0e36061e40ff,85d75a23e94054bbb8c219f21036872149150a90,,257e4c08564208c7ebf0f6e8185853cf539495c4508abc372e8e39cbae743174,Tests_failed,9d947990e32bc40c172cff6bdce6a3e34ff00f241abe661da55bae8a1e399000,Tests_failed,False,True,False,False
-16,refs/remotes/origin/pull/10,d0c36bece9c7b6409a6d57008f327287bfdffa06,d2ddc1e3a56d5b420971b107bdcd3b99bd48d0ef,799779c8c760115a7e8db66d8c24bfccd5a55416,,,,,,,False,False,False
-17,refs/remotes/origin/pull/10,332b6ed4e324199565e235cd915368f9bc1b43d2,4650578dee712b2b08f2ead2bf6a531f82b1e0e9,d623f491daa2f14f06f53338265d28a489138a6b,,78bd4536994ebea95ea8ebd2ea2e7c2b38f2a1fec25ecf322a0e8ccd1406b50b,Tests_failed,aa6ad3f4018260b629417aa44faa0ed893f6092c9f5dd08efdae1fd8223de2ba,Tests_failed,False,True,False,False
-18,refs/remotes/origin/pull/11,9616295df7f50bfc3ec267047e569e83b6cdb0fa,5ac6e3f5b0c069465004538a3b95a37fcaf7b57e,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,d91c6fc51df617b83f7695a8eee5308421e57fe98f4a0c678e994c79904f69f0,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,Tests_failed,False,True,False,False
-20,refs/remotes/origin/pull/13,3054b1c14ba79e67394915d27d5e5d766f3b3db0,f39a51f3fa3259bd4cabd16fe9c025fe4dbbea1f,3c11b36e5054ec09cdc29ec3d0e9273406163498,,,,,,,False,False,False
+idx,branch_name,merge,left,right,notes,test merge,diff contains java file,left_tree_fingerprint,left parent test result,right_tree_fingerprint,right parent test result,parents pass,sampled for testing
+12,refs/remotes/origin/pull/10,c77b1297186f967622adaeaaa8996a27f77cf1fc,3c9c2af6b97384850a9acd7245d7c62ec7afd593,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,False,True,29f6d014333b808c5b2979ba16de7e6bac963e80ad0cba35ae14faa710e7c431,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,Tests_failed,False,False
+13,refs/remotes/origin/pull/10,d5be99e116542e2f465caba331c0ec589bf726d9,c68f61f0fa127bfeaea5e4e255e7b3730df4a0f6,f85b6777113505109cc1f1d4a9edea615fd403f5,,False,False,,,,,,False
+14,refs/remotes/origin/pull/10,ea282678896734ff75f85799cfd7ae3f3515790c,ea143d850887e020a5f505f0917da33cf186af75,94607fd9ce59988a15dca58958c7ae30bf51155f,,False,False,,,,,,False
+15,refs/remotes/origin/pull/10,c3b5f5d43f5699e2e4d698bd32a74dad9ac1d477,78eceb43ad7a9dd89e7c2f36f82b0e36061e40ff,85d75a23e94054bbb8c219f21036872149150a90,,False,True,257e4c08564208c7ebf0f6e8185853cf539495c4508abc372e8e39cbae743174,Tests_failed,9d947990e32bc40c172cff6bdce6a3e34ff00f241abe661da55bae8a1e399000,Tests_failed,False,False
+16,refs/remotes/origin/pull/10,d0c36bece9c7b6409a6d57008f327287bfdffa06,d2ddc1e3a56d5b420971b107bdcd3b99bd48d0ef,799779c8c760115a7e8db66d8c24bfccd5a55416,,False,False,,,,,,False
+17,refs/remotes/origin/pull/10,332b6ed4e324199565e235cd915368f9bc1b43d2,4650578dee712b2b08f2ead2bf6a531f82b1e0e9,d623f491daa2f14f06f53338265d28a489138a6b,,False,True,78bd4536994ebea95ea8ebd2ea2e7c2b38f2a1fec25ecf322a0e8ccd1406b50b,Tests_failed,aa6ad3f4018260b629417aa44faa0ed893f6092c9f5dd08efdae1fd8223de2ba,Tests_failed,False,False
+18,refs/remotes/origin/pull/11,9616295df7f50bfc3ec267047e569e83b6cdb0fa,5ac6e3f5b0c069465004538a3b95a37fcaf7b57e,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,False,True,d91c6fc51df617b83f7695a8eee5308421e57fe98f4a0c678e994c79904f69f0,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,Tests_failed,False,False
+20,refs/remotes/origin/pull/13,3054b1c14ba79e67394915d27d5e5d766f3b3db0,f39a51f3fa3259bd4cabd16fe9c025fe4dbbea1f,3c11b36e5054ec09cdc29ec3d0e9273406163498,,False,False,,,,,,False

results/combined/merges_analyzed/abel533/echarts.csv

@@ -1,8 +1,8 @@
-idx,branch_name,merge,left,right,notes,left_tree_fingerprint,left parent test result,right_tree_fingerprint,right parent test result,parents pass,diff contains java file,test merge,sampled for testing
-2,refs/heads/master,dfb2ce7f6c904b05104ec2d71368cc5d5e57021b,fdcf985b0eb8415325c8d190e6496d17d02897f5,d1e9574a29ffe7dfd87b14fcdac07c963f859aae,,,,,,,False,False,False
-3,refs/heads/master,fdcf985b0eb8415325c8d190e6496d17d02897f5,065b89d61758b64740de96e802c4ab16adfcab87,58e500f1e09ed1331548d559979b5061d8140084,,,,,,,False,False,False
-9,refs/heads/master,c6ad45b63a9ee7e20b45c4176695a72efa31606a,69757fc487bf247b74339de637cb48ed296e5a67,721cb6fa847595de1b4b58fb80ec6079d15d9a95,,05376667afdba29e22c97c992edaa40bf1866e03660bb315bdf828004927fae8,Tests_passed,0dd38c0c7dc253f9adc1ed53fbf0e6b43aca5ad15576b740bffdc47c97a43a47,Tests_passed,True,True,True,True
-10,refs/heads/master,fc65c67a4855f65e1b778b4962e5b734d82ab1d7,ae9fbef36b16b3ddc58d39e14269403cee92e636,668ac0edc5624f380c088b47433fe783ba11f5e8,,,,,,,False,False,False
-11,refs/heads/master,668ac0edc5624f380c088b47433fe783ba11f5e8,d5e6271d5363a335bd9ea5a2c902f163b2bdb514,5f0a4b3bc2ba2c4f0ab80556267bb2df31788204,,,,,,,False,False,False
-12,refs/heads/master,d5e6271d5363a335bd9ea5a2c902f163b2bdb514,1d50fd3a2c9a61fec56842fdac50c9cceeb0ba00,54bfdd7a70edfd19c9da38a29f2225fece2fe422,,,,,,,False,False,False
-13,refs/heads/master,b61c1c173475f94ea92b7629d0f9edf0fb105632,2842cb7a669c6bad015edc8b85abb655294e310e,ad79acbd73b33c6c35dee8be40fccb0ba4c651c1,,,,,,,False,False,False
+idx,branch_name,merge,left,right,notes,test merge,diff contains java file,left_tree_fingerprint,left parent test result,right_tree_fingerprint,right parent test result,parents pass,sampled for testing
+2,refs/heads/master,dfb2ce7f6c904b05104ec2d71368cc5d5e57021b,fdcf985b0eb8415325c8d190e6496d17d02897f5,d1e9574a29ffe7dfd87b14fcdac07c963f859aae,,False,False,,,,,,False
+3,refs/heads/master,fdcf985b0eb8415325c8d190e6496d17d02897f5,065b89d61758b64740de96e802c4ab16adfcab87,58e500f1e09ed1331548d559979b5061d8140084,,False,False,,,,,,False
+9,refs/heads/master,c6ad45b63a9ee7e20b45c4176695a72efa31606a,69757fc487bf247b74339de637cb48ed296e5a67,721cb6fa847595de1b4b58fb80ec6079d15d9a95,,True,True,05376667afdba29e22c97c992edaa40bf1866e03660bb315bdf828004927fae8,Tests_passed,0dd38c0c7dc253f9adc1ed53fbf0e6b43aca5ad15576b740bffdc47c97a43a47,Tests_passed,True,True
+10,refs/heads/master,fc65c67a4855f65e1b778b4962e5b734d82ab1d7,ae9fbef36b16b3ddc58d39e14269403cee92e636,668ac0edc5624f380c088b47433fe783ba11f5e8,,False,False,,,,,,False
+11,refs/heads/master,668ac0edc5624f380c088b47433fe783ba11f5e8,d5e6271d5363a335bd9ea5a2c902f163b2bdb514,5f0a4b3bc2ba2c4f0ab80556267bb2df31788204,,False,False,,,,,,False
+12,refs/heads/master,d5e6271d5363a335bd9ea5a2c902f163b2bdb514,1d50fd3a2c9a61fec56842fdac50c9cceeb0ba00,54bfdd7a70edfd19c9da38a29f2225fece2fe422,,False,False,,,,,,False
+13,refs/heads/master,b61c1c173475f94ea92b7629d0f9edf0fb105632,2842cb7a669c6bad015edc8b85abb655294e310e,ad79acbd73b33c6c35dee8be40fccb0ba4c651c1,,False,False,,,,,,False