Consolidated results (#282)

.github/workflows/small-test.yml

@@ -27,9 +27,10 @@ jobs:
         distribution: 'zulu'
         java-version: 17
     - run: echo "JAVA17_HOME=$JAVA_HOME" >> $GITHUB_ENV
-    - run: echo "${GITHUB_WORKSPACE}/src/scripts/merge_tools" >> $GITHUB_PATH
     - run: java -version
     - uses: actions/checkout@v4
+      with:
+        submodules: 'true'
     - name: Set up Python 3.8
       uses: actions/setup-python@v5
       with:
@@ -51,8 +52,6 @@ jobs:
         maven-version: ${{ matrix.maven }}
     - name: Clean caches & workspace
       run: make clean
-    - name: Build Java programs
-      run: ./gradlew build
     - name: Run small test
       run: |
         git config --global user.email "[email protected]"

Makefile

@@ -25,7 +25,7 @@ check-python-style:
 
 # This target deletes files that are not committed to version control.
 clean:
-	rm -rf .workdir
+	${MAKE} clean-workdir
 	rm -rf repos
 	rm -rf scratch
 	rm -rf results/small
@@ -91,12 +91,19 @@ small-test:
 	./run_small.sh --include_trivial_merges --no_timing
 	${MAKE} small-test-diff
 
+small-test-without-cleaning:
+	${MAKE} clean-test-cache
+	./run_small.sh --include_trivial_merges --no_timing
+	${MAKE} small-test-diff
+
 update-figures:
 	./run_combined.sh -op
 	./run_greatest_hits.sh -op
 	./run_reaper.sh -op
 
 run-all:
+	${MAKE} clean-workdir
+	${MAKE} small-test-without-cleaning
 	./run_combined.sh
 	./run_greatest_hits.sh
 	./run_reaper.sh
@@ -108,8 +115,13 @@ small-test-diff:
 gradle-assemble:
 	./gradlew -q assemble -g ../.gradle/
 
+clean-workdir:
+	if [ -d .workdir ]; then chmod -R u+w .workdir; fi
+	rm -rf .workdir
+
 clean-local:
-	rm -rf repos .workdir
+	${MAKE} clean-workdir
+	rm -rf repos
 
 protect-repos:
 	find repos -mindepth 1 -type d -exec chmod a-w {} +

results/combined/all_repos_head_test_results.csv

@@ -12263,7 +12263,7 @@ idx,repository,head hash,head tree fingerprint,head test result
 13017,guuuo/dubai,209abb098265c4ef19711d68617e16b7350f6a18,3092a210e730e4aa3236a71eb018ab3d5841d906851b4c0ad1e90dcb98339ff9,Tests_passed
 13018,soarcn/coco-accessory,4abba9f09c4055ead412c86bd25e4404565dd5d4,d97031b0d82b873ff02eea9e8194b5244d61f89514624705cd931ee25d049f86,Tests_failed
 13019,hoverruan/sae-spring-template,8e81017cc6b6e32ff8dbe15a34bcb8f9bf5aaa84,1cfd1d9c42e5fb43d658194421a3154882d1ef8b1f11ea1d02d49fb63f60b978,Tests_passed
-13020,sonarcommunity/sonar-web,aee1b2c066bad507918b662973c602a728ef9491,ebf71ae6f83b183f55f268ddc48c16ecd0904dca02873b3543cb6348d1dde5fc,Tests_passed
+13020,sonarcommunity/sonar-web,aee1b2c066bad507918b662973c602a728ef9491,f7934ed60c94334bfca1398c09f6580d902a1732ee362810a37410b1196579c7,Tests_passed
 13021,slimroms/apktool,c1ab20ee4b03364d5120d7e71cf02dba346bfcd5,34128662c0608d553f6829488e38d8a822b2eee9df6da4770b507cb6bbc6aff1,Tests_failed
 13022,ahiraz/pushnotificationdemo,2714e0c135608914dca0a3b07f2364eb580ee4ba,d55b62dc5e771438d67fbab71b624de7c573cf7a5918470ec3e40474a969cb0b,Tests_failed
 13023,melin/dubbo-rpc-swift,32673ba8cf1c733a6b6672f345382e95809e73fb,44c8dbed2417e90c89907fe654cdc7d060e331fdf0426c491e1b70ae772e4b91,Tests_passed
@@ -15965,7 +15965,7 @@ idx,repository,head hash,head tree fingerprint,head test result
 16925,chrislongo/download-hls,243722ecf156881c8cf42d6eb9fe8459bb6f5c55,89a624c18459772025110f492c3d5f631877d17ca27122c431d518d8c671b24c,Tests_passed
 16926,carbonfive/hellod,01423b6118177dcc7a44388543775369780da6e7,4deb0d911c36a7b75bf44cfc893b96734450bb0fc29a3cb5972a08b7f93d41cc,Tests_failed
 16927,shipilev/java-object-layout,d4b41c8a8bcb897aee3ae3ca91410738c4a2ef3d,a6e0064545d231464c0a688cf1a06a4987460ad5e6a211d65c445565723a75d7,Tests_failed
-16928,deegree/deegree3,3088ebfab9e27898aa80577d27a9ca8b51ce5fbc,8e5119315831dda2ce955a960abb450132de37d9df0a7ed3305eb5b03063e952,Tests_passed
+16928,deegree/deegree3,3088ebfab9e27898aa80577d27a9ca8b51ce5fbc,f28128951414277a486e5a3fc556b6d9677e76e6515bc5479fd060c6d6e0a4d6,Tests_failed
 16929,sethm/symon,ecd4bbbd9a448f4f546eef9648149700b93c91ec,a1d3eeda91c9902d4a63a9d16ae5ad2317dc3e3dd418f8e764345aa1e9e38601,Tests_passed
 16930,yannis666/coherence-jsr-107,b6d95fe625ad6369f7733be82092ecb4affe0102,b51a16742cb3ea0ce04992157aa2e6f6deeedfb1ccd84bbe84c367526cd20a82,Tests_failed
 16931,geonode/geoserver-geonode-ext,c553cbf6d702e82f1febb8911b280fa7e1307c5a,d56d71dd8f0d4007db31c8d5960a14f68137809146773ebc49c16b0f9731ddb1,Tests_failed

results/combined/defs.tex

@@ -1,26 +1,26 @@
 \def\combinedReposInitial{42092\xspace}
-\def\combinedReposValid{4073\xspace}
-\def\combinedMergesInitial{295387\xspace}
+\def\combinedReposValid{4072\xspace}
+\def\combinedMergesInitial{294715\xspace}
 \def\combinedMergesPer{100\xspace}
-\def\combinedMergesNonTrivial{86064\xspace}
-\def\combinedReposNonTrivial{2713\xspace}
-\def\combinedMergesJavaDiff{22070\xspace}
-\def\combinedReposJavaDiff{1654\xspace}
-\def\combinedMergesJavaDiffAndParentsPass{6387\xspace}
-\def\combinedReposJavaDiffAndParentsPass{1126\xspace}
-\def\combinedReposSampled{1126\xspace}
-\def\combinedMergesSampled{6365\xspace}
+\def\combinedMergesNonTrivial{85979\xspace}
+\def\combinedReposNonTrivial{2712\xspace}
+\def\combinedMergesJavaDiff{21971\xspace}
+\def\combinedReposJavaDiff{1653\xspace}
+\def\combinedMergesJavaDiffAndParentsPass{6106\xspace}
+\def\combinedReposJavaDiffAndParentsPass{1121\xspace}
+\def\combinedReposSampled{1121\xspace}
+\def\combinedMergesSampled{6096\xspace}
 \def\combinedReposYieldedFull{2\xspace}
-\def\combinedReposTotal{1126\xspace}
-\def\combinedMergesTotal{6365\xspace}
+\def\combinedReposTotal{1120\xspace}
+\def\combinedMergesTotal{6076\xspace}
 
 % Results
-\def\combinedSporkOverOrtCorrect{553\xspace}
-\def\combinedSporkOverOrtIncorrect{514\xspace}
-\def\combinedMainBranchMerges{3662\xspace}
-\def\combinedMainBranchMergesPercent{58\xspace}
-\def\combinedOtherBranceshMerges{2703\xspace}
-\def\combinedOtherBranchesMergesPercent{42\xspace}
+\def\combinedSporkOverOrtCorrect{525\xspace}
+\def\combinedSporkOverOrtIncorrect{488\xspace}
+\def\combinedMainBranchMerges{3572\xspace}
+\def\combinedMainBranchMergesPercent{59\xspace}
+\def\combinedOtherBranceshMerges{2504\xspace}
+\def\combinedOtherBranchesMergesPercent{41\xspace}
 \def\combinedReposJava{42092\xspace}
 
 % Timeout

results/combined/local_repos.csv

@@ -1820,7 +1820,7 @@ idx,repository,head hash,head tree fingerprint,head test result
 13009,pmcfadin/cql3-bulk-loader-example,7b3eaaaaff1173970beebcf24e612bc088387527,a00ee351ead615c353a5b52f926e9d39360da016e981e3754396cb20ebcdffd7,Tests_passed
 13017,guuuo/dubai,209abb098265c4ef19711d68617e16b7350f6a18,3092a210e730e4aa3236a71eb018ab3d5841d906851b4c0ad1e90dcb98339ff9,Tests_passed
 13019,hoverruan/sae-spring-template,8e81017cc6b6e32ff8dbe15a34bcb8f9bf5aaa84,1cfd1d9c42e5fb43d658194421a3154882d1ef8b1f11ea1d02d49fb63f60b978,Tests_passed
-13020,sonarcommunity/sonar-web,aee1b2c066bad507918b662973c602a728ef9491,ebf71ae6f83b183f55f268ddc48c16ecd0904dca02873b3543cb6348d1dde5fc,Tests_passed
+13020,sonarcommunity/sonar-web,aee1b2c066bad507918b662973c602a728ef9491,f7934ed60c94334bfca1398c09f6580d902a1732ee362810a37410b1196579c7,Tests_passed
 13023,melin/dubbo-rpc-swift,32673ba8cf1c733a6b6672f345382e95809e73fb,44c8dbed2417e90c89907fe654cdc7d060e331fdf0426c491e1b70ae772e4b91,Tests_passed
 13024,rcarz/jira-client,868a5ca8977e0c13b792aeed7a793d55cea743ec,d16373a76725cfa20771c9139d9eb231986b7eff0b800392d7765e3013c379ad,Tests_passed
 13027,bujiio/buji-pac4j,1a206c74705d26c402cb6dfe05180a6a0b2c52ba,162312102ab254408789a72ab977e7ab332fdae98273f4989fb418303fae033a,Tests_passed
@@ -2374,7 +2374,6 @@ idx,repository,head hash,head tree fingerprint,head test result
 16898,bryncooke/cdi-unit,b5f5e1c3a9e80b7302369ebc4eb00d2512c679cf,4c364900e88708a8b45bbfc6f2d8a2ab9b12efa5d08149dc6b074cb9b46c1301,Tests_passed
 16907,mercadopago/sdk-java,e9b49e80d0c8d0951d531ddf67ceb0afd35a9810,ebff11d3f5db162bd30cda108ae33abcc752202e7d1b003009b158a51a23f47e,Tests_passed
 16925,chrislongo/download-hls,243722ecf156881c8cf42d6eb9fe8459bb6f5c55,89a624c18459772025110f492c3d5f631877d17ca27122c431d518d8c671b24c,Tests_passed
-16928,deegree/deegree3,3088ebfab9e27898aa80577d27a9ca8b51ce5fbc,8e5119315831dda2ce955a960abb450132de37d9df0a7ed3305eb5b03063e952,Tests_passed
 16929,sethm/symon,ecd4bbbd9a448f4f546eef9648149700b93c91ec,a1d3eeda91c9902d4a63a9d16ae5ad2317dc3e3dd418f8e764345aa1e9e38601,Tests_passed
 16937,apache/commons-validator,c55e303cd0a9ead18271664e705d684ea8cfd471,7b212b774e7db369ecd527a48a7ae91f19464525c74f28d2a6b5fa2441055cc4,Tests_passed
 16938,spinscale/elasticsearch-facet-georegion,1fcf2128aa7e1bda53464698495b8c6327642091,578944ad4e429e3294ba6086bc181667ecf25cc929a79f6d9119950089f198c3,Tests_passed

results/combined/merges_analyzed/3scale/3scale_ws_api_for_java.csv

@@ -1,9 +1,9 @@
-idx,branch_name,merge,left,right,notes,test merge,diff contains java file,left_tree_fingerprint,left parent test result,right_tree_fingerprint,right parent test result,parents pass,sampled for testing
-12,refs/remotes/origin/pull/10,c77b1297186f967622adaeaaa8996a27f77cf1fc,3c9c2af6b97384850a9acd7245d7c62ec7afd593,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,False,True,29f6d014333b808c5b2979ba16de7e6bac963e80ad0cba35ae14faa710e7c431,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,Tests_failed,False,False
-13,refs/remotes/origin/pull/10,d5be99e116542e2f465caba331c0ec589bf726d9,c68f61f0fa127bfeaea5e4e255e7b3730df4a0f6,f85b6777113505109cc1f1d4a9edea615fd403f5,,False,False,,,,,,False
-14,refs/remotes/origin/pull/10,ea282678896734ff75f85799cfd7ae3f3515790c,ea143d850887e020a5f505f0917da33cf186af75,94607fd9ce59988a15dca58958c7ae30bf51155f,,False,False,,,,,,False
-15,refs/remotes/origin/pull/10,c3b5f5d43f5699e2e4d698bd32a74dad9ac1d477,78eceb43ad7a9dd89e7c2f36f82b0e36061e40ff,85d75a23e94054bbb8c219f21036872149150a90,,False,True,257e4c08564208c7ebf0f6e8185853cf539495c4508abc372e8e39cbae743174,Tests_failed,9d947990e32bc40c172cff6bdce6a3e34ff00f241abe661da55bae8a1e399000,Tests_failed,False,False
-16,refs/remotes/origin/pull/10,d0c36bece9c7b6409a6d57008f327287bfdffa06,d2ddc1e3a56d5b420971b107bdcd3b99bd48d0ef,799779c8c760115a7e8db66d8c24bfccd5a55416,,False,False,,,,,,False
-17,refs/remotes/origin/pull/10,332b6ed4e324199565e235cd915368f9bc1b43d2,4650578dee712b2b08f2ead2bf6a531f82b1e0e9,d623f491daa2f14f06f53338265d28a489138a6b,,False,True,78bd4536994ebea95ea8ebd2ea2e7c2b38f2a1fec25ecf322a0e8ccd1406b50b,Tests_failed,aa6ad3f4018260b629417aa44faa0ed893f6092c9f5dd08efdae1fd8223de2ba,Tests_failed,False,False
-18,refs/remotes/origin/pull/11,9616295df7f50bfc3ec267047e569e83b6cdb0fa,5ac6e3f5b0c069465004538a3b95a37fcaf7b57e,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,False,True,d91c6fc51df617b83f7695a8eee5308421e57fe98f4a0c678e994c79904f69f0,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,Tests_failed,False,False
-20,refs/remotes/origin/pull/13,3054b1c14ba79e67394915d27d5e5d766f3b3db0,f39a51f3fa3259bd4cabd16fe9c025fe4dbbea1f,3c11b36e5054ec09cdc29ec3d0e9273406163498,,False,False,,,,,,False
+idx,branch_name,merge,left,right,notes,left_tree_fingerprint,left parent test result,right_tree_fingerprint,right parent test result,parents pass,diff contains java file,test merge,sampled for testing
+12,refs/remotes/origin/pull/10,c77b1297186f967622adaeaaa8996a27f77cf1fc,3c9c2af6b97384850a9acd7245d7c62ec7afd593,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,29f6d014333b808c5b2979ba16de7e6bac963e80ad0cba35ae14faa710e7c431,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,Tests_failed,False,True,False,False
+13,refs/remotes/origin/pull/10,d5be99e116542e2f465caba331c0ec589bf726d9,c68f61f0fa127bfeaea5e4e255e7b3730df4a0f6,f85b6777113505109cc1f1d4a9edea615fd403f5,,,,,,,False,False,False
+14,refs/remotes/origin/pull/10,ea282678896734ff75f85799cfd7ae3f3515790c,ea143d850887e020a5f505f0917da33cf186af75,94607fd9ce59988a15dca58958c7ae30bf51155f,,,,,,,False,False,False
+15,refs/remotes/origin/pull/10,c3b5f5d43f5699e2e4d698bd32a74dad9ac1d477,78eceb43ad7a9dd89e7c2f36f82b0e36061e40ff,85d75a23e94054bbb8c219f21036872149150a90,,257e4c08564208c7ebf0f6e8185853cf539495c4508abc372e8e39cbae743174,Tests_failed,9d947990e32bc40c172cff6bdce6a3e34ff00f241abe661da55bae8a1e399000,Tests_failed,False,True,False,False
+16,refs/remotes/origin/pull/10,d0c36bece9c7b6409a6d57008f327287bfdffa06,d2ddc1e3a56d5b420971b107bdcd3b99bd48d0ef,799779c8c760115a7e8db66d8c24bfccd5a55416,,,,,,,False,False,False
+17,refs/remotes/origin/pull/10,332b6ed4e324199565e235cd915368f9bc1b43d2,4650578dee712b2b08f2ead2bf6a531f82b1e0e9,d623f491daa2f14f06f53338265d28a489138a6b,,78bd4536994ebea95ea8ebd2ea2e7c2b38f2a1fec25ecf322a0e8ccd1406b50b,Tests_failed,aa6ad3f4018260b629417aa44faa0ed893f6092c9f5dd08efdae1fd8223de2ba,Tests_failed,False,True,False,False
+18,refs/remotes/origin/pull/11,9616295df7f50bfc3ec267047e569e83b6cdb0fa,5ac6e3f5b0c069465004538a3b95a37fcaf7b57e,88d410aa7befa2c5499395a1fec6da34c9cb0b5f,,d91c6fc51df617b83f7695a8eee5308421e57fe98f4a0c678e994c79904f69f0,Tests_failed,5cdfe7b9b14c055deb49591da977b31082e45249cd70e299661f2de943586fc8,Tests_failed,False,True,False,False
+20,refs/remotes/origin/pull/13,3054b1c14ba79e67394915d27d5e5d766f3b3db0,f39a51f3fa3259bd4cabd16fe9c025fe4dbbea1f,3c11b36e5054ec09cdc29ec3d0e9273406163498,,,,,,,False,False,False