Partial changes from update-imports branch

.github/workflows/check-reproducibility.yml

@@ -32,18 +32,13 @@ jobs:
         channels: conda-forge,defaults
         activate-environment: AST
         environment-file: environment.yml
-    - name: Install maven
-      uses: s4u/[email protected]
-      with:
-        java-version: 17
-        maven-version: ${{ matrix.maven }}
     - name: Clean caches & workspace
       run: make clean
     - run: echo "LOGURU_COLORIZE=NO" >> $GITHUB_ENV
+    - name: Build
+      run: cd src/scripts/merge_tools/merging && ./gradlew -q shadowJar
     - name: make check-merges-reproducibility
       run: |
-        git config --global user.email "[email protected]"
-        git config --global user.name "Example Example"
         head -n 151 results/combined/result.csv > temp.csv && mv temp.csv results/combined/result.csv
         make check-merges-reproducibility
       env:

.github/workflows/small-test.yml

@@ -31,7 +31,7 @@ jobs:
     - uses: actions/checkout@v4
       with:
         submodules: 'true'
-    - name: Set up Python 3.12
+    - name: Set up Python 3.8
       uses: actions/setup-python@v5
       with:
         python-version: '3.12'
@@ -53,8 +53,6 @@ jobs:
         maven-version: ${{ matrix.maven }}
     - name: Clean caches & workspace
       run: make clean
-    - name: Install killall
-      run: sudo apt update && sudo apt install psmisc -y
     - run: echo "LOGURU_COLORIZE=NO" >> $GITHUB_ENV
     - name: Run small test
       run: |

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "src/scripts/merge_tools/merging"]
+	path = src/scripts/merge_tools/merging
+	url = https://github.com/plume-lib/merging.git
+	ignore = dirty

Makefile

@@ -11,8 +11,6 @@ CSV_RESULTS_GREATEST_HITS = results/greatest_hits/result.csv
 CSV_RESULTS_REAPER = results/reaper/result.csv
 CSV_RESULTS = $(CSV_RESULTS_COMBINED)
 
-NUM_PROCESSES = 0
-
 shell-script-style:
 	shellcheck -e SC2153 -x -P SCRIPTDIR --format=gcc ${SH_SCRIPTS} ${BASH_SCRIPTS}
 	checkbashisms ${SH_SCRIPTS}
@@ -134,7 +132,7 @@ clean-local:
 check-merges-reproducibility:
 	@echo "Running replay_merge for each idx in parallel using GNU Parallel..."
 	@set -e; \
-	tail -n +2 $(CSV_RESULTS) | awk -F, '{print $$1}' | parallel -j 50% python3 src/python/replay_merge.py --merges_csv $(CSV_RESULTS) -delete_workdir --idx {}
+	tail -n +2 $(CSV_RESULTS) | awk -F, '{print $$1}' | parallel --halt now,fail=1 -j 50% python3 src/python/replay_merge.py --merges_csv $(CSV_RESULTS) -skip_build -delete_workdir --idx {}
 
 protect-repos:
 	find repos -mindepth 1 -type d -exec chmod a-w {} +

README.md

@@ -215,6 +215,5 @@ To run style checking run `make style`.
 ## Comparing merge algorithms
 
 To investigate differences between two mergers:
- * edit file `src/python/select_from_results.py` to reflect the differences you are interested in.
- * run `src/python/select_from_results.py` to create a .csv database containing only the differences.
- * run `src/python/replay_merge.py --merges_csv CSV_FILE --idx INDEX` (maybe add `-test`) for the index of the merge you are interested in.
+ * run `src/python/select_from_results.py` to produce a CSV file containing only the differences you are interested in.  See its [documentation (at top of file)](src/python/select_from_results.py) for how to run it.
+ * run `src/python/replay_merge.py --idx INDEX` (maybe add `-test`) for the index of the merge you are interested in.

gitmerge_ort_imports.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env sh
+
+# usage: ./gitmerge_ort_imports.sh <clone_dir> <branch-1> <branch-2>
+
+clone_dir=$1
+branch1=$2
+branch2=$3
+
+export JAVA_HOME="$JAVA17_HOME"
+
+cd "$clone_dir" || exit 1
+
+git checkout "$branch1" --force
+
+attributes_file=".git/info/attributes"
+echo "*.java merge=merge-java" >> "$attributes_file"
+git config --local merge.merge-java.name "Merge Java files"
+git config --local merge.merge-java.driver 'java-merge-driver.sh "%A" "%O" "%B"'
+
+git merge --no-edit "$branch2"
+retVal=$?
+
+# report conflicts
+if [ "$retVal" -ne 0 ]; then
+    echo "Conflict"
+fi
+
+exit "$retVal"

run.sh

@@ -23,6 +23,13 @@ OUT_DIR="results/$RUN_NAME"
 N_MERGES=$3
 CACHE_DIR="${4}"
 
+# Check if it's run on MacOS, if yes raise an error
+backend=$(uname -s)
+if [ "$backend" = "Darwin" ]; then
+    echo "Error: MacOS is not supported. Please run the script on a Linux machine. This is due to the use of readarray in certain merge tools."
+    exit 1
+fi
+
 comparator_flags=""
 no_timing=false
 only_plotting=false
@@ -53,8 +60,13 @@ while [ $# -gt 0 ]; do
 done
 
 PATH=$(pwd)/src/scripts/merge_tools/:$PATH
+PATH=$(pwd)/src/scripts/merge_tools/merging/src/main/sh/:$PATH
 export PATH
 
+# Clone all submodules
+git submodule update --init --recursive
+
+# Empty config file
 GIT_CONFIG_GLOBAL=$(pwd)/.gitconfig
 export GIT_CONFIG_GLOBAL
 
@@ -72,6 +84,9 @@ if [ -z "${JAVA17_HOME:+isset}" ] ; then echo "JAVA17_HOME is not set"; exit 1;
 if [ -z "${machine_id:+isset}" ] ; then machine_id=0; fi
 if [ -z "${num_machines:+isset}" ] ; then num_machines=1; fi
 
+export JAVA_HOME=$JAVA17_HOME
+./src/scripts/merge_tools/merging/gradlew -p src/scripts/merge_tools/merging shadowJar
+
 echo "Machine ID: $machine_id"
 echo "Number of machines: $num_machines"
 echo "Output directory: $OUT_DIR"
@@ -93,6 +108,7 @@ if [ "$only_plotting" = true ]; then
     exit 0
 fi
 
+export JAVA_HOME=$JAVA11_HOME
 ./gradlew -q assemble -g ../.gradle/
 
 mkdir -p "$OUT_DIR"
@@ -125,14 +141,26 @@ java -cp build/libs/astmergeevaluation-all.jar \
     "$OUT_DIR/repos_head_passes.csv" \
     "$OUT_DIR/merges"
 
-# Sample 5*<n_merges> merges
-read -ra merge_comparator_flags <<<"${comparator_flags}"
-python3 src/python/merges_sampler.py \
-    --repos_head_passes_csv "$OUT_DIR/repos_head_passes.csv" \
-    --merges_path "$OUT_DIR/merges/" \
-    --output_dir "$OUT_DIR/merges_sampled/" \
-    --n_merges "$((5 * "$N_MERGES"))" \
-    "${merge_comparator_flags[@]}"
+# Calculate the number of merges
+total_merges=$((5 * N_MERGES))
+
+# Ensure comparator_flags is set, but default to an empty array if not
+if [[ -n "${comparator_flags}" ]]; then
+    read -ra merge_comparator_flags <<< "${comparator_flags}"
+    python3 src/python/merges_sampler.py \
+        --repos_head_passes_csv "$OUT_DIR/repos_head_passes.csv" \
+        --merges_path "$OUT_DIR/merges/" \
+        --output_dir "$OUT_DIR/merges_sampled/" \
+        --n_merges "$total_merges" \
+        "${merge_comparator_flags[@]}"
+else
+    echo "Warning: 'comparator_flags' is empty, continuing without additional flags."
+    python3 src/python/merges_sampler.py \
+        --repos_head_passes_csv "$OUT_DIR/repos_head_passes.csv" \
+        --merges_path "$OUT_DIR/merges/" \
+        --output_dir "$OUT_DIR/merges_sampled/" \
+        --n_merges "$total_merges"
+fi
 
 python3 src/python/split_repos.py \
     --repos_csv "$OUT_DIR/repos_head_passes.csv" \
@@ -153,9 +181,6 @@ python3 src/python/merge_tester.py \
     --output_dir "$OUT_DIR/merges_tested/" \
     --cache_dir "$CACHE_DIR"
 
-# Define an array for additional arguments
-extra_args=()
-
 if [ "$no_timing" = false ]; then
     python3 src/python/merge_runtime_measure.py \
         --repos_head_passes_csv "$OUT_DIR/local_repos.csv" \
@@ -164,15 +189,24 @@ if [ "$no_timing" = false ]; then
         --n_sampled_timing 1 \
         --n_timings 3 \
         --cache_dir "$CACHE_DIR"
-    extra_args+=(--timed_merges_path "$OUT_DIR/merges_timed/")
+
+    python3 src/python/latex_output.py \
+        --run_name "$RUN_NAME" \
+        --merges_path "$OUT_DIR/merges/" \
+        --tested_merges_path "$OUT_DIR/merges_tested/" \
+        --analyzed_merges_path "$OUT_DIR/merges_analyzed/" \
+        --timed_merges_path "$OUT_DIR/merges_timed/" \
+        --full_repos_csv "$REPOS_CSV_WITH_HASHES" \
+        --repos_head_passes_csv "$OUT_DIR/repos_head_passes.csv" \
+        --n_merges "$N_MERGES" \
+        --output_dir "$OUT_DIR"
 fi
 
 python3 src/python/latex_output.py \
     --run_name "$RUN_NAME" \
     --merges_path "$OUT_DIR/merges/" \
     --tested_merges_path "$OUT_DIR/merges_tested/" \
     --analyzed_merges_path "$OUT_DIR/merges_analyzed/" \
-    "${extra_args[@]}" \
     --full_repos_csv "$REPOS_CSV_WITH_HASHES" \
     --repos_head_passes_csv "$OUT_DIR/repos_head_passes.csv" \
     --n_merges "$N_MERGES" \

src/python/delete_adjacent_keys_from_cache.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+""" Delete the keys containing 'imports' in the JSON files in the given directory. """
+import os
+import json
+
+
+def count_import_keys(directory):
+    """Count the number of keys containing 'imports' in the JSON files in the given directory."""
+    count = 0
+    for root, _, files in os.walk(directory):
+        json_files = [f for f in files if f.endswith(".json")]
+        for json_file in json_files:
+            file_path = os.path.join(root, json_file)
+            with open(file_path, "r", encoding="utf-8") as file:
+                data = json.load(file)
+
+            # Count keys containing 'adjacent'
+            keys_to_delete = [key for key in data if "adjacent" in key]
+            count += len(keys_to_delete)
+    return count
+
+
+def delete_import_keys(directory):
+    """Delete the keys containing 'imports' in the JSON files in the given directory."""
+    total_deleted = 0
+    for root, _, files in os.walk(directory):
+        json_files = [f for f in files if f.endswith(".json")]
+        for json_file in json_files:
+            file_path = os.path.join(root, json_file)
+            with open(file_path, "r", encoding="utf-8") as file:
+                data = json.load(file)
+
+            # Record keys to delete
+            keys_to_delete = [key for key in data.keys() if "adjacent" in key]
+            if keys_to_delete:
+                for key in keys_to_delete:
+                    del data[key]
+                    total_deleted += 1
+
+            # Save the modified data back to file
+            with open(file_path, "w", encoding="utf-8") as file:
+                json.dump(data, file, indent=4)
+
+    return total_deleted
+
+
+def main():
+    """Main function."""
+    directory = "cache"
+    potential_deletions = count_import_keys(directory)
+    print(f"Potential deletions: {potential_deletions}")
+    confirm = input("Do you want to proceed with deleting these keys? (yes/no): ")
+    if confirm.lower() == "yes":
+        total_deleted = delete_import_keys(directory)
+        print(f"Total keys deleted: {total_deleted}")
+    else:
+        print("Operation cancelled.")
+
+
+if __name__ == "__main__":
+    main()