Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Check merges reproducibility #283

Merged
merged 3 commits into from
May 5, 2024
Merged

Check merges reproducibility #283

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
eb34783
Added checks for merge replays
benedikt-schesch May 4, 2024
3c429fe
Added corrected data
benedikt-schesch May 5, 2024
1d77022
Merge replay checker
benedikt-schesch May 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
The diff you're trying to view is too large. We only load the first 3000 changed files.
3 changes: 2 additions & 1 deletion .github/workflows/small-test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ jobs:
test:
strategy:
matrix:
maven: [ '3.9.2']
maven: [ '3.9.2' ]
runs-on: ubuntu-latest
defaults:
run:
Expand Down Expand Up @@ -52,6 +52,7 @@ jobs:
maven-version: ${{ matrix.maven }}
- name: Clean caches & workspace
run: make clean
- run: echo "LOGURU_COLORIZE=NO" >> $GITHUB_ENV
- name: Run small test
run: |
git config --global user.email "[email protected]"
Expand Down
10 changes: 10 additions & 0 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,11 @@ SH_SCRIPTS = $(shell grep --exclude-dir=build --exclude-dir=repos --exclude-di
BASH_SCRIPTS = $(shell grep --exclude-dir=build --exclude-dir=repos --exclude-dir=cache -r -l '^\#! \?\(/bin/\|/usr/bin/env \)bash' * | grep -v /.git/ | grep -v '~$$' | grep -v '\.tar$$' | grep -v gradlew)
PYTHON_FILES = $(shell find . -name '*.py' ! -path './repos/*' -not -path "./.workdir/*" -not -path "./cache*/*" | grep -v '/__pycache__/' | grep -v '/.git/' | grep -v gradlew)

CSV_RESULTS_COMBINED = results/combined/result.csv
CSV_RESULTS_GREATEST_HITS = results/greatest_hits/result.csv
CSV_RESULTS_REAPER = results/reaper/result.csv
CSV_RESULTS = $(CSV_RESULTS_COMBINED)

shell-script-style:
shellcheck -e SC2153 -x -P SCRIPTDIR --format=gcc ${SH_SCRIPTS} ${BASH_SCRIPTS}
checkbashisms ${SH_SCRIPTS}
Expand Down Expand Up @@ -108,6 +113,7 @@ run-all:
./run_greatest_hits.sh
./run_reaper.sh


small-test-diff:
python3 test/check_equal_csv.py --actual_folder results/small/ --goal_folder test/small-goal-files/
@echo
Expand All @@ -123,6 +129,10 @@ clean-local:
${MAKE} clean-workdir
rm -rf repos

check-merges-reproducibility:
@echo "Running replay_merge for each idx in parallel..."
@tail -n +2 $(CSV_RESULTS) | awk -F, '{print $$1}' | parallel -u --halt now,fail=1 -j 0 'python3 src/python/replay_merge.py -delete_workdir --idx {}'

protect-repos:
find repos -mindepth 1 -type d -exec chmod a-w {} +

Expand Down
22 changes: 11 additions & 11 deletions results/combined/defs.tex
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,22 @@
\def\combinedMergesPer{100\xspace}
\def\combinedMergesNonTrivial{85979\xspace}
\def\combinedReposNonTrivial{2712\xspace}
\def\combinedMergesJavaDiff{21971\xspace}
\def\combinedMergesJavaDiff{21861\xspace}
\def\combinedReposJavaDiff{1653\xspace}
\def\combinedMergesJavaDiffAndParentsPass{6106\xspace}
\def\combinedReposJavaDiffAndParentsPass{1121\xspace}
\def\combinedReposSampled{1121\xspace}
\def\combinedMergesSampled{6096\xspace}
\def\combinedReposYieldedFull{2\xspace}
\def\combinedMergesJavaDiffAndParentsPass{6050\xspace}
\def\combinedReposJavaDiffAndParentsPass{1120\xspace}
\def\combinedReposSampled{1120\xspace}
\def\combinedMergesSampled{6040\xspace}
\def\combinedReposYieldedFull{1\xspace}
\def\combinedReposTotal{1120\xspace}
\def\combinedMergesTotal{6076\xspace}
\def\combinedMergesTotal{6040\xspace}

% Results
\def\combinedSporkOverOrtCorrect{525\xspace}
\def\combinedSporkOverOrtIncorrect{488\xspace}
\def\combinedMainBranchMerges{3572\xspace}
\def\combinedSporkOverOrtCorrect{518\xspace}
\def\combinedSporkOverOrtIncorrect{487\xspace}
\def\combinedMainBranchMerges{3557\xspace}
\def\combinedMainBranchMergesPercent{59\xspace}
\def\combinedOtherBranceshMerges{2504\xspace}
\def\combinedOtherBranceshMerges{2483\xspace}
\def\combinedOtherBranchesMergesPercent{41\xspace}
\def\combinedReposJava{42092\xspace}

Expand Down
18 changes: 9 additions & 9 deletions results/combined/merges_analyzed/3scale/3scale_ws_api_for_java.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
idx,branch_name,merge,left,right,notes,left_tree_fingerprint,left parent test result,right_tree_fingerprint,right parent test result,parents pass,diff contains java file,test merge,sampled for testing
12,refs/remotes/origin/pull/10,c77b1297186f967622adaeaaa8996a27f77cf1fc,3c9c2af6b97384850a9acd7245d7c62ec7afd593,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,29f6d014333b808c5b2979ba16de7e6bac963e80ad0cba35ae14faa710e7c431,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,Tests_failed,False,True,False,False
13,refs/remotes/origin/pull/10,d5be99e116542e2f465caba331c0ec589bf726d9,c68f61f0fa127bfeaea5e4e255e7b3730df4a0f6,f85b6777113505109cc1f1d4a9edea615fd403f5,,,,,,,False,False,False
14,refs/remotes/origin/pull/10,ea282678896734ff75f85799cfd7ae3f3515790c,ea143d850887e020a5f505f0917da33cf186af75,94607fd9ce59988a15dca58958c7ae30bf51155f,,,,,,,False,False,False
15,refs/remotes/origin/pull/10,c3b5f5d43f5699e2e4d698bd32a74dad9ac1d477,78eceb43ad7a9dd89e7c2f36f82b0e36061e40ff,85d75a23e94054bbb8c219f21036872149150a90,,257e4c08564208c7ebf0f6e8185853cf539495c4508abc372e8e39cbae743174,Tests_failed,9d947990e32bc40c172cff6bdce6a3e34ff00f241abe661da55bae8a1e399000,Tests_failed,False,True,False,False
16,refs/remotes/origin/pull/10,d0c36bece9c7b6409a6d57008f327287bfdffa06,d2ddc1e3a56d5b420971b107bdcd3b99bd48d0ef,799779c8c760115a7e8db66d8c24bfccd5a55416,,,,,,,False,False,False
17,refs/remotes/origin/pull/10,332b6ed4e324199565e235cd915368f9bc1b43d2,4650578dee712b2b08f2ead2bf6a531f82b1e0e9,d623f491daa2f14f06f53338265d28a489138a6b,,78bd4536994ebea95ea8ebd2ea2e7c2b38f2a1fec25ecf322a0e8ccd1406b50b,Tests_failed,aa6ad3f4018260b629417aa44faa0ed893f6092c9f5dd08efdae1fd8223de2ba,Tests_failed,False,True,False,False
18,refs/remotes/origin/pull/11,9616295df7f50bfc3ec267047e569e83b6cdb0fa,5ac6e3f5b0c069465004538a3b95a37fcaf7b57e,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,d91c6fc51df617b83f7695a8eee5308421e57fe98f4a0c678e994c79904f69f0,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,Tests_failed,False,True,False,False
20,refs/remotes/origin/pull/13,3054b1c14ba79e67394915d27d5e5d766f3b3db0,f39a51f3fa3259bd4cabd16fe9c025fe4dbbea1f,3c11b36e5054ec09cdc29ec3d0e9273406163498,,,,,,,False,False,False
idx,branch_name,merge,left,right,notes,test merge,diff contains java file,left_tree_fingerprint,left parent test result,right_tree_fingerprint,right parent test result,parents pass,sampled for testing
12,refs/remotes/origin/pull/10,c77b1297186f967622adaeaaa8996a27f77cf1fc,3c9c2af6b97384850a9acd7245d7c62ec7afd593,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,False,True,29f6d014333b808c5b2979ba16de7e6bac963e80ad0cba35ae14faa710e7c431,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,Tests_failed,False,False
13,refs/remotes/origin/pull/10,d5be99e116542e2f465caba331c0ec589bf726d9,c68f61f0fa127bfeaea5e4e255e7b3730df4a0f6,f85b6777113505109cc1f1d4a9edea615fd403f5,,False,False,,,,,,False
14,refs/remotes/origin/pull/10,ea282678896734ff75f85799cfd7ae3f3515790c,ea143d850887e020a5f505f0917da33cf186af75,94607fd9ce59988a15dca58958c7ae30bf51155f,,False,False,,,,,,False
15,refs/remotes/origin/pull/10,c3b5f5d43f5699e2e4d698bd32a74dad9ac1d477,78eceb43ad7a9dd89e7c2f36f82b0e36061e40ff,85d75a23e94054bbb8c219f21036872149150a90,,False,True,257e4c08564208c7ebf0f6e8185853cf539495c4508abc372e8e39cbae743174,Tests_failed,9d947990e32bc40c172cff6bdce6a3e34ff00f241abe661da55bae8a1e399000,Tests_failed,False,False
16,refs/remotes/origin/pull/10,d0c36bece9c7b6409a6d57008f327287bfdffa06,d2ddc1e3a56d5b420971b107bdcd3b99bd48d0ef,799779c8c760115a7e8db66d8c24bfccd5a55416,,False,False,,,,,,False
17,refs/remotes/origin/pull/10,332b6ed4e324199565e235cd915368f9bc1b43d2,4650578dee712b2b08f2ead2bf6a531f82b1e0e9,d623f491daa2f14f06f53338265d28a489138a6b,,False,True,78bd4536994ebea95ea8ebd2ea2e7c2b38f2a1fec25ecf322a0e8ccd1406b50b,Tests_failed,aa6ad3f4018260b629417aa44faa0ed893f6092c9f5dd08efdae1fd8223de2ba,Tests_failed,False,False
18,refs/remotes/origin/pull/11,9616295df7f50bfc3ec267047e569e83b6cdb0fa,5ac6e3f5b0c069465004538a3b95a37fcaf7b57e,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,False,True,d91c6fc51df617b83f7695a8eee5308421e57fe98f4a0c678e994c79904f69f0,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,Tests_failed,False,False
20,refs/remotes/origin/pull/13,3054b1c14ba79e67394915d27d5e5d766f3b3db0,f39a51f3fa3259bd4cabd16fe9c025fe4dbbea1f,3c11b36e5054ec09cdc29ec3d0e9273406163498,,False,False,,,,,,False
16 changes: 8 additions & 8 deletions results/combined/merges_analyzed/abel533/echarts.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
idx,branch_name,merge,left,right,notes,left_tree_fingerprint,left parent test result,right_tree_fingerprint,right parent test result,parents pass,diff contains java file,test merge,sampled for testing
2,refs/heads/master,dfb2ce7f6c904b05104ec2d71368cc5d5e57021b,fdcf985b0eb8415325c8d190e6496d17d02897f5,d1e9574a29ffe7dfd87b14fcdac07c963f859aae,,,,,,,False,False,False
3,refs/heads/master,fdcf985b0eb8415325c8d190e6496d17d02897f5,065b89d61758b64740de96e802c4ab16adfcab87,58e500f1e09ed1331548d559979b5061d8140084,,,,,,,False,False,False
9,refs/heads/master,c6ad45b63a9ee7e20b45c4176695a72efa31606a,69757fc487bf247b74339de637cb48ed296e5a67,721cb6fa847595de1b4b58fb80ec6079d15d9a95,,05376667afdba29e22c97c992edaa40bf1866e03660bb315bdf828004927fae8,Tests_passed,0dd38c0c7dc253f9adc1ed53fbf0e6b43aca5ad15576b740bffdc47c97a43a47,Tests_passed,True,True,True,True
10,refs/heads/master,fc65c67a4855f65e1b778b4962e5b734d82ab1d7,ae9fbef36b16b3ddc58d39e14269403cee92e636,668ac0edc5624f380c088b47433fe783ba11f5e8,,,,,,,False,False,False
11,refs/heads/master,668ac0edc5624f380c088b47433fe783ba11f5e8,d5e6271d5363a335bd9ea5a2c902f163b2bdb514,5f0a4b3bc2ba2c4f0ab80556267bb2df31788204,,,,,,,False,False,False
12,refs/heads/master,d5e6271d5363a335bd9ea5a2c902f163b2bdb514,1d50fd3a2c9a61fec56842fdac50c9cceeb0ba00,54bfdd7a70edfd19c9da38a29f2225fece2fe422,,,,,,,False,False,False
13,refs/heads/master,b61c1c173475f94ea92b7629d0f9edf0fb105632,2842cb7a669c6bad015edc8b85abb655294e310e,ad79acbd73b33c6c35dee8be40fccb0ba4c651c1,,,,,,,False,False,False
idx,branch_name,merge,left,right,notes,test merge,diff contains java file,left_tree_fingerprint,left parent test result,right_tree_fingerprint,right parent test result,parents pass,sampled for testing
2,refs/heads/master,dfb2ce7f6c904b05104ec2d71368cc5d5e57021b,fdcf985b0eb8415325c8d190e6496d17d02897f5,d1e9574a29ffe7dfd87b14fcdac07c963f859aae,,False,False,,,,,,False
3,refs/heads/master,fdcf985b0eb8415325c8d190e6496d17d02897f5,065b89d61758b64740de96e802c4ab16adfcab87,58e500f1e09ed1331548d559979b5061d8140084,,False,False,,,,,,False
9,refs/heads/master,c6ad45b63a9ee7e20b45c4176695a72efa31606a,69757fc487bf247b74339de637cb48ed296e5a67,721cb6fa847595de1b4b58fb80ec6079d15d9a95,,True,True,05376667afdba29e22c97c992edaa40bf1866e03660bb315bdf828004927fae8,Tests_passed,0dd38c0c7dc253f9adc1ed53fbf0e6b43aca5ad15576b740bffdc47c97a43a47,Tests_passed,True,True
10,refs/heads/master,fc65c67a4855f65e1b778b4962e5b734d82ab1d7,ae9fbef36b16b3ddc58d39e14269403cee92e636,668ac0edc5624f380c088b47433fe783ba11f5e8,,False,False,,,,,,False
11,refs/heads/master,668ac0edc5624f380c088b47433fe783ba11f5e8,d5e6271d5363a335bd9ea5a2c902f163b2bdb514,5f0a4b3bc2ba2c4f0ab80556267bb2df31788204,,False,False,,,,,,False
12,refs/heads/master,d5e6271d5363a335bd9ea5a2c902f163b2bdb514,1d50fd3a2c9a61fec56842fdac50c9cceeb0ba00,54bfdd7a70edfd19c9da38a29f2225fece2fe422,,False,False,,,,,,False
13,refs/heads/master,b61c1c173475f94ea92b7629d0f9edf0fb105632,2842cb7a669c6bad015edc8b85abb655294e310e,ad79acbd73b33c6c35dee8be40fccb0ba4c651c1,,False,False,,,,,,False
Loading
Loading