Skip to content

Commit

Permalink
Merge branch 'adjacent-plus-plumelib' of github.com:mernst/AST-Mergin…
Browse files Browse the repository at this point in the history 
…g-Evaluation into adjacent-plus-plumelib
  • Loading branch information
@mernst
mernst committed Sep 23, 2024
2 parents d01a9e7 + e770590 commit 170cca7
Show file tree
Hide file tree
Showing 18 changed files with 381 additions and 2 deletions.
3 changes: 3 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ fix-java-style:
check-java-style:
./gradlew -q spotlessCheck javadoc requireJavadoc -g ../.gradle/

update-small-results:
rm -rf test/small-goal-files/
rsync -av --exclude='*.pdf' --exclude='*.png' --exclude='*unhandled_and_failed_merges_without_intellimerge*' --exclude='*.pgf' results/small/ test/small-goal-files/

# This target deletes files that are not committed to version control.
clean:
Expand Down
5 changes: 4 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,10 @@ If you make a change to the mergers that changes merge results, you need to
update the goal files or else reproducibility checks will fail.
Copy certain files from `results/small/` to `test/small-goal-files/`.

To update the reproducibility tests, run `make run-all`.
To update the reproducibility tests, run `make run-all`
(this takes a long time!) and commit the results.
This will run merges in parallel. If the load on your machine becomes very low
(like no parallelism is happening), then terminate the process and restart it.

### Load the stored cache

Expand Down
16 changes: 15 additions & 1 deletion test/check_equal_csv.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,21 @@ def remove_run_time(df):
print(f"Checking {goal_file}")
actual_file = actual_folder / goal_file
assert actual_file.exists(), f"{actual_file} does not exist"
goal_df = pd.read_csv(goal_folder / goal_file, header=0, index_col="idx")
try:
goal_df = pd.read_csv(goal_folder / goal_file, header=0, index_col="idx")
except Exception:
goal_df = pd.read_csv(goal_folder / goal_file)
actual_df = pd.read_csv(actual_file)
if not goal_df.equals(actual_df):
print(f"{goal_folder/goal_file} and {actual_file} are not equal")
print("Goal file:")
print(goal_df)
print("Actual file:")
print(actual_df)
raise ValueError(
f"{goal_folder/goal_file} and {actual_file} are not equal"
)
continue
actual_df = pd.read_csv(actual_file, header=0, index_col="idx")
goal_df = remove_run_time(goal_df)
actual_df = remove_run_time(actual_df)
Expand Down
5 changes: 5 additions & 0 deletions test/small-goal-files/all_repos_head_test_results.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
idx,repository,language,architecture,community,continuous_integration,documentation,history,issues,license,size,unit_test,stars,scorebased_org,randomforest_org,scorebased_utl,randomforest_utl,head hash,head tree fingerprint,head test result
0,benedikt-schesch/git-hires-merge-example,Java,1.0,1,0,0.433617,10.5,0.0,1,34146,0.448443,54,1,0,1,1,4ae40d33c3b959e3a15e458eb9b0676251e36a41,1cebc99cd5f8a1b210b66b6985f8aa5c38c4af194a32ad23dc3c6261f43d2968,Tests_passed
1,mangstadt/ez-vcard,Java,1.0,1,0,0.433617,10.5,0.0,1,34146,0.448443,54,1,0,1,1,95298de036f1df38fd7ca7e3225c4fd5ad5c44e6,134a29559bbdf9bcea93d2153b59e65ef205410c0c295f63b6d82a5e9dc828f9,Tests_passed
2,pedrovgs/algorithms,Java,0.932927,1,1,0.390942,89.0,0.75,1,6616,0.556371,1377,1,1,1,1,c108e3a4b4fa2556b8c88d7ab6803084b01a4364,3c7ee13d6f66c42510e5edb8cff2102d8c61ed625f7cc871397eaa2b1e529895,Tests_passed
4,tntim96/jscover,Java,0.985816,1,0,0.490264,25.0,6.448276,1,54612,0.627527,298,1,1,1,1,fff7a84ce12a037518e3c58e1d941a4d3dabb7e2,db8ac3637a22e488a08c15d4ddae3d115f42359e9fd0f42e2b2246aa8767d43e,Tests_passed
30 changes: 30 additions & 0 deletions test/small-goal-files/defs.tex
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
\def\smallReposInitial{4\xspace}
\def\smallReposValid{4\xspace}
\def\smallMergesInitial{63\xspace}
\def\smallMergesPer{50\xspace}
\def\smallMergesNonTrivial{33\xspace}
\def\smallReposNonTrivial{3\xspace}
\def\smallMergesJavaDiff{10\xspace}
\def\smallReposJavaDiff{4\xspace}
\def\smallMergesJavaDiffAndParentsPass{6\xspace}
\def\smallReposJavaDiffAndParentsPass{4\xspace}
\def\smallReposSampled{4\xspace}
\def\smallMergesSampled{6\xspace}
\def\smallReposYieldedFull{0\xspace}
\def\smallReposTotal{4\xspace}
\def\smallMergesTotal{6\xspace}

% Results
\def\smallAverageTriesUntilPass{1.0\xspace}
\def\smallNumberofMergesWith1TriesUntilPass{68\xspace}
\def\smallSporkOverOrtCorrect{3\xspace}
\def\smallSporkOverOrtIncorrect{0\xspace}
\def\smallMainBranchMerges{4\xspace}
\def\smallMainBranchMergesPercent{67\xspace}
\def\smallOtherBranchMerges{2\xspace}
\def\smallOtherBranchMergesPercent{33\xspace}
\def\smallReposJava{4\xspace}

% Timeout
\def\smallParentTestTimeout{30\xspace}
\def\smallMergeTestTimeout{45\xspace}
5 changes: 5 additions & 0 deletions test/small-goal-files/repos_head_passes.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
idx,repository,language,architecture,community,continuous_integration,documentation,history,issues,license,size,unit_test,stars,scorebased_org,randomforest_org,scorebased_utl,randomforest_utl,head hash,head tree fingerprint,head test result
0,benedikt-schesch/git-hires-merge-example,Java,1.0,1,0,0.433617,10.5,0.0,1,34146,0.448443,54,1,0,1,1,4ae40d33c3b959e3a15e458eb9b0676251e36a41,1cebc99cd5f8a1b210b66b6985f8aa5c38c4af194a32ad23dc3c6261f43d2968,Tests_passed
1,mangstadt/ez-vcard,Java,1.0,1,0,0.433617,10.5,0.0,1,34146,0.448443,54,1,0,1,1,95298de036f1df38fd7ca7e3225c4fd5ad5c44e6,134a29559bbdf9bcea93d2153b59e65ef205410c0c295f63b6d82a5e9dc828f9,Tests_passed
2,pedrovgs/algorithms,Java,0.932927,1,1,0.390942,89.0,0.75,1,6616,0.556371,1377,1,1,1,1,c108e3a4b4fa2556b8c88d7ab6803084b01a4364,3c7ee13d6f66c42510e5edb8cff2102d8c61ed625f7cc871397eaa2b1e529895,Tests_passed
4,tntim96/jscover,Java,0.985816,1,0,0.490264,25.0,6.448276,1,54612,0.627527,298,1,1,1,1,fff7a84ce12a037518e3c58e1d941a4d3dabb7e2,db8ac3637a22e488a08c15d4ddae3d115f42359e9fd0f42e2b2246aa8767d43e,Tests_passed
50 changes: 50 additions & 0 deletions test/small-goal-files/tables/all/table_feature_main_summary.tex
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
% Do not edit. This file is automatically generated.
\setlength{\tabcolsep}{.285\tabcolsep}
\begin{tabular}{c|cc|cc|cc}
Tool &
\multicolumn{6}{c}{Merges} \\ \cline{2-7}
&
\multicolumn{2}{c|}{Correct} &
\multicolumn{2}{c|}{Unhandled} &
\multicolumn{2}{c}{Incorrect} \\
&
\multicolumn{1}{c}{Main} &
\multicolumn{1}{c|}{Other} &
\multicolumn{1}{c}{Main} &
\multicolumn{1}{c|}{Other} &
\multicolumn{1}{c}{Main} &
\multicolumn{1}{c}{Other} \\
\hline
Gitmerge-ort & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Gitmerge-ort-ignorespace & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Gitmerge-recursive-histogram & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Gitmerge-recursive-myers-ignorespace & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Gitmerge-recursive-minimal & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Gitmerge-recursive-myers & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Gitmerge-recursive-patience & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Gitmerge-resolve & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Hires-Merge & 75\% & 0\% & 25\% & 100\% & 0\% & 0\% \\
Spork & 100\% & 50\% & 0\% & 50\% & 0\% & 0\% \\
IntelliMerge & 50\% & 50\% & 0\% & 50\% & 50\% & 0\% \\
Adjacent & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Imports & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Version Numbers & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
IVn & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
IVn-ignorespace & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Gitmerge-ort-plus & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Gitmerge-ort-ignorespace-plus & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Gitmerge-recursive-histogram-plus & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Gitmerge-recursive-myers-ignorespace-plus & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Gitmerge-recursive-minimal-plus & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Gitmerge-recursive-myers-plus & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Gitmerge-recursive-patience-plus & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Gitmerge-resolve-plus & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Git-hires-merge-plus & 75\% & 0\% & 25\% & 100\% & 0\% & 0\% \\
Spork-plus & 100\% & 50\% & 0\% & 50\% & 0\% & 0\% \\
Intellimerge-plus & 50\% & 50\% & 0\% & 50\% & 50\% & 0\% \\
Adjacent-plus & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Imports-plus & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Version-numbers-plus & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Ivn-plus & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
Ivn-ignorespace-plus & 50\% & 0\% & 50\% & 100\% & 0\% & 0\% \\
\end{tabular}
41 changes: 41 additions & 0 deletions test/small-goal-files/tables/all/table_summary.tex
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
% Do not edit. This file is automatically generated.
\begin{tabular}{l|c c|c c|c c|}
Tool & \multicolumn{6}{c|}{Merges} \\ \cline{2-7}
& \multicolumn{2}{c|}{Correct} &
\multicolumn{2}{c|}{Unhandled} &
\multicolumn{2}{c|}{Incorrect} \\
& \# & \% & \# & \% & \# & \% \\
\hline
Gitmerge-ort & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Gitmerge-ort-ignorespace & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Gitmerge-recursive-histogram & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Gitmerge-recursive-myers-ignorespace & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Gitmerge-recursive-minimal & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Gitmerge-recursive-myers & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Gitmerge-recursive-patience & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Gitmerge-resolve & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Hires-Merge & 3 & 50\% & 3 & 50\% & 0 & 0\% \\
Spork & 5 & 83\% & 1 & 17\% & 0 & 0\% \\
IntelliMerge & 3 & 50\% & 1 & 17\% & 2 & 33\% \\
Adjacent & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Imports & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Version Numbers & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
IVn & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
IVn-ignorespace & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Gitmerge-ort-plus & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Gitmerge-ort-ignorespace-plus & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Gitmerge-recursive-histogram-plus & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Gitmerge-recursive-myers-ignorespace-plus & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Gitmerge-recursive-minimal-plus & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Gitmerge-recursive-myers-plus & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Gitmerge-recursive-patience-plus & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Gitmerge-resolve-plus & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Git-hires-merge-plus & 3 & 50\% & 3 & 50\% & 0 & 0\% \\
Spork-plus & 5 & 83\% & 1 & 17\% & 0 & 0\% \\
Intellimerge-plus & 3 & 50\% & 1 & 17\% & 2 & 33\% \\
Adjacent-plus & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Imports-plus & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Version-numbers-plus & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Ivn-plus & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
Ivn-ignorespace-plus & 2 & 33\% & 4 & 67\% & 0 & 0\% \\
\end{tabular}
33 changes: 33 additions & 0 deletions test/small-goal-files/tables/all/tool_comparison_table.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
,gitmerge_ort,gitmerge_ort_ignorespace,gitmerge_recursive_histogram,gitmerge_recursive_myers_ignorespace,gitmerge_recursive_minimal,gitmerge_recursive_myers,gitmerge_recursive_patience,gitmerge_resolve,git_hires_merge,spork,intellimerge,adjacent,imports,version_numbers,ivn,ivn_ignorespace,gitmerge_ort_plus,gitmerge_ort_ignorespace_plus,gitmerge_recursive_histogram_plus,gitmerge_recursive_myers_ignorespace_plus,gitmerge_recursive_minimal_plus,gitmerge_recursive_myers_plus,gitmerge_recursive_patience_plus,gitmerge_resolve_plus,git_hires_merge_plus,spork_plus,intellimerge_plus,adjacent_plus,imports_plus,version_numbers_plus,ivn_plus,ivn_ignorespace_plus
gitmerge_ort,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
gitmerge_ort_ignorespace,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
gitmerge_recursive_histogram,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
gitmerge_recursive_myers_ignorespace,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
gitmerge_recursive_minimal,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
gitmerge_recursive_myers,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
gitmerge_recursive_patience,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
gitmerge_resolve,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
git_hires_merge,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
spork,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
intellimerge,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
adjacent,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
imports,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
version_numbers,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
ivn,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
ivn_ignorespace,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
gitmerge_ort_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
gitmerge_ort_ignorespace_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0,0
gitmerge_recursive_histogram_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0,0
gitmerge_recursive_myers_ignorespace_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0,0
gitmerge_recursive_minimal_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0,0
gitmerge_recursive_myers_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0,0
gitmerge_recursive_patience_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0,0
gitmerge_resolve_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0,0
git_hires_merge_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0,0
spork_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0,0
intellimerge_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0,0
adjacent_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0,0
imports_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0,0
version_numbers_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0,0
ivn_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,0
ivn_ignorespace_plus,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-
Loading

0 comments on commit 170cca7

Please sign in to comment.