From b1f1ce39f87eb7010aea8fddd6770188a2271069 Mon Sep 17 00:00:00 2001 From: dzhovi Date: Sat, 4 Jan 2025 16:31:21 +0300 Subject: [PATCH 01/15] added repo-reasoner functionality --- requirements.txt | 1 + steps/discover.sh | 13 +++++++++++++ steps/env.sh | 6 ++++++ 3 files changed, 20 insertions(+) diff --git a/requirements.txt b/requirements.txt index 32387a44..42cb3da2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ chardet==5.2.0 mypy==1.12.1 cffconvert==2.0.0 samples-filter==0.5.1 +repo-reasoner==0.0.25 \ No newline at end of file diff --git a/steps/discover.sh b/steps/discover.sh index cb98bf6f..c7f8f578 100755 --- a/steps/discover.sh +++ b/steps/discover.sh @@ -69,6 +69,19 @@ elif [ -z "${REPOS}" ] || [ ! -e "${REPOS}" ]; then samples-filter filter "${fargs[@]}" rm "${csv}" mv "${nosamples}" "${csv}" + + maintained=${TARGET}/maintained.csv + declare -a margs=( \ + "--repositories=${csv}" \ + "--output=${maintained}" \ + "--key=${GIGACHAT_KEY}" \ + "--model=${GIGACHAT_MODEL}" + ) + repo-reasoner filter-unmaintained "${margs[@]}" + rm "${csv}" + mv "${maintained}" "${csv}" + + else echo "Using the list of repositories from the '${REPOS}' file (defined by the REPOS environment variable)..." cat "${REPOS}" > "${csv}" diff --git a/steps/env.sh b/steps/env.sh index 993672fd..04c578d9 100755 --- a/steps/env.sh +++ b/steps/env.sh @@ -68,6 +68,12 @@ jq --version multimetric --help > /dev/null +repo-reasoner --help > /dev/null + +rubocop -v + +inkscape --version + awk --version parallel --version From 62fcf9a0ada7193de5c9f3813186e278d19a222a Mon Sep 17 00:00:00 2001 From: dzhovi Date: Sat, 4 Jan 2025 16:46:58 +0300 Subject: [PATCH 02/15] added repo-reasoner functionality --- steps/env.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/steps/env.sh b/steps/env.sh index 04c578d9..bfc5df7a 100755 --- a/steps/env.sh +++ b/steps/env.sh @@ -70,9 +70,6 @@ multimetric --help > /dev/null repo-reasoner --help > /dev/null -rubocop -v - -inkscape --version awk --version From 6f9b6951e6f9d97f05e6c32bd842cf1ddc7dd0ae Mon Sep 17 00:00:00 2001 From: dzhovi Date: Sat, 4 Jan 2025 16:49:41 +0300 Subject: [PATCH 03/15] added repo-reasoner functionality --- steps/discover.sh | 1 - steps/env.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/steps/discover.sh b/steps/discover.sh index c7f8f578..16a97ae3 100755 --- a/steps/discover.sh +++ b/steps/discover.sh @@ -81,7 +81,6 @@ elif [ -z "${REPOS}" ] || [ ! -e "${REPOS}" ]; then rm "${csv}" mv "${maintained}" "${csv}" - else echo "Using the list of repositories from the '${REPOS}' file (defined by the REPOS environment variable)..." cat "${REPOS}" > "${csv}" diff --git a/steps/env.sh b/steps/env.sh index bfc5df7a..919d1b9e 100755 --- a/steps/env.sh +++ b/steps/env.sh @@ -70,7 +70,6 @@ multimetric --help > /dev/null repo-reasoner --help > /dev/null - awk --version parallel --version From b15e3cfa17b005c5d3a574fb445d7b0f7ccdc5ad Mon Sep 17 00:00:00 2001 From: dzhovi Date: Sat, 4 Jan 2025 17:45:30 +0300 Subject: [PATCH 04/15] test-discover-repos.sh fixed. --- tests/steps/test-discover-repos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/steps/test-discover-repos.sh b/tests/steps/test-discover-repos.sh index cff48ff6..3cb31196 100755 --- a/tests/steps/test-discover-repos.sh +++ b/tests/steps/test-discover-repos.sh @@ -32,7 +32,7 @@ tex=${TARGET}/foo.tex rm -f "${csv}" msg=$("${LOCAL}/steps/discover-repos.rb" --dry --pause=0 --total=3 --page-size=1 --min-stars=100 --max-stars=1000 "--csv=${csv}" "--tex=${tex}") echo "${msg}" - echo "${msg}" | grep "Completed querying for year 2024. Found 3 repositories so far." + echo "${msg}" | grep "Completed querying for year 2025. Found 3 repositories so far." echo "${msg}" | grep "Found 3 total repositories in GitHub" test -e "${csv}" test -s "${tex}" From 017eaf84b99026e06d692bf2adb418693d612a7b Mon Sep 17 00:00:00 2001 From: dzhovi Date: Sat, 11 Jan 2025 16:46:25 +0300 Subject: [PATCH 05/15] fix --- Makefile | 2 +- steps/discover.sh | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 0408a582..abd5b927 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,7 @@ define step endef # The main goal -all: env $(TARGET)/start.txt $(TARGET)/repositories.csv polish clone unregister jpeek filter measure aggregate summarize zip +all: env $(TARGET)/start.txt $(TARGET)/repositories.csv clone unregister jpeek filter measure aggregate summarize zip echo -e "\n\nSUCCESS (made by yegor256/cam $(VERSION)$$("$${LOCAL}/help/tdiff.sh" "$$(cat "$(TARGET)/start.txt")"))!" install: diff --git a/steps/discover.sh b/steps/discover.sh index 16a97ae3..a88acc5e 100755 --- a/steps/discover.sh +++ b/steps/discover.sh @@ -71,14 +71,15 @@ elif [ -z "${REPOS}" ] || [ ! -e "${REPOS}" ]; then mv "${nosamples}" "${csv}" maintained=${TARGET}/maintained.csv - declare -a margs=( \ - "--repositories=${csv}" \ - "--output=${maintained}" \ - "--key=${GIGACHAT_KEY}" \ - "--model=${GIGACHAT_MODEL}" - ) + declare -a margs=( \ + "--repositories=${csv}" \ + "--output=${maintained}" \ + "--key=${GIGACHAT_KEY}" \ + "--model=${GIGACHAT_MODEL}" + ) repo-reasoner filter-unmaintained "${margs[@]}" - rm "${csv}" + original_csv="${csv}.original" + mv "${csv}" "${original_csv}" mv "${maintained}" "${csv}" else From b32b8e4ba7aec10ed4f6fe88773aa173fead21cb Mon Sep 17 00:00:00 2001 From: dzhovi Date: Thu, 16 Jan 2025 15:06:40 +0300 Subject: [PATCH 06/15] Revert "fix" This reverts commit 017eaf84 --- steps/aggregation-functions/90-percentile.sh | 2 ++ steps/aggregation-functions/mean.sh | 3 +++ steps/aggregation-functions/median.sh | 3 +++ 3 files changed, 8 insertions(+) diff --git a/steps/aggregation-functions/90-percentile.sh b/steps/aggregation-functions/90-percentile.sh index 2d583b40..07326d26 100755 --- a/steps/aggregation-functions/90-percentile.sh +++ b/steps/aggregation-functions/90-percentile.sh @@ -24,6 +24,8 @@ set -e set -o pipefail +export LC_NUMERIC=C + metric_file=$1 output_folder=$2 metric_name=$3 diff --git a/steps/aggregation-functions/mean.sh b/steps/aggregation-functions/mean.sh index 96c1aed9..aad834a4 100755 --- a/steps/aggregation-functions/mean.sh +++ b/steps/aggregation-functions/mean.sh @@ -24,6 +24,9 @@ set -e set -o pipefail +export LC_NUMERIC=C + + metric_file=$1 output_folder=$2 metric_name=$3 diff --git a/steps/aggregation-functions/median.sh b/steps/aggregation-functions/median.sh index a63ab03f..d1f10521 100755 --- a/steps/aggregation-functions/median.sh +++ b/steps/aggregation-functions/median.sh @@ -24,6 +24,9 @@ set -e set -o pipefail +export LC_NUMERIC=C + + metric_file=$1 output_folder=$2 metric_name=$3 From 94a42eb6de3464fe1f45711c9b7d61f742748531 Mon Sep 17 00:00:00 2001 From: dzhovi Date: Sat, 18 Jan 2025 15:26:53 +0300 Subject: [PATCH 07/15] Revert "added repo-reasoner functionality" This reverts commit 6f9b6951e6f9d97f05e6c32bd842cf1ddc7dd0ae. --- steps/discover.sh | 1 + steps/env.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/steps/discover.sh b/steps/discover.sh index a88acc5e..fb6747b6 100755 --- a/steps/discover.sh +++ b/steps/discover.sh @@ -82,6 +82,7 @@ elif [ -z "${REPOS}" ] || [ ! -e "${REPOS}" ]; then mv "${csv}" "${original_csv}" mv "${maintained}" "${csv}" + else echo "Using the list of repositories from the '${REPOS}' file (defined by the REPOS environment variable)..." cat "${REPOS}" > "${csv}" diff --git a/steps/env.sh b/steps/env.sh index 919d1b9e..bfc5df7a 100755 --- a/steps/env.sh +++ b/steps/env.sh @@ -70,6 +70,7 @@ multimetric --help > /dev/null repo-reasoner --help > /dev/null + awk --version parallel --version From e4507457582f9b80e93a315a65a6c1375a1159e6 Mon Sep 17 00:00:00 2001 From: dzhovi Date: Sat, 18 Jan 2025 15:26:53 +0300 Subject: [PATCH 08/15] Revert "added repo-reasoner functionality" This reverts commit 62fcf9a0ada7193de5c9f3813186e278d19a222a. --- steps/env.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/steps/env.sh b/steps/env.sh index bfc5df7a..04c578d9 100755 --- a/steps/env.sh +++ b/steps/env.sh @@ -70,6 +70,9 @@ multimetric --help > /dev/null repo-reasoner --help > /dev/null +rubocop -v + +inkscape --version awk --version From e6fab0203ec83efe6eb24e988b92e2f2dc9c2f07 Mon Sep 17 00:00:00 2001 From: dzhovi Date: Sat, 18 Jan 2025 15:33:25 +0300 Subject: [PATCH 09/15] Revert "added repo-reasoner functionality" This reverts commit b1f1ce39 --- help/check-make.sh | 2 +- metrics/aocih.sh | 12 +++++++++--- requirements.txt | 1 - steps/discover.sh | 14 -------------- steps/env.sh | 6 ------ 5 files changed, 10 insertions(+), 25 deletions(-) diff --git a/help/check-make.sh b/help/check-make.sh index 9f28cd50..5ec4bb96 100755 --- a/help/check-make.sh +++ b/help/check-make.sh @@ -24,7 +24,7 @@ set -e set -o pipefail -make_version=$(make --version | head -n1 | cut -d' ' -f3) +make_version=$(make --version | awk 'NR==1 {print $3}') make_version_int=$(echo "${make_version}" | cut -d'.' -f1) if [ "${make_version_int}" -lt 4 ]; then diff --git a/metrics/aocih.sh b/metrics/aocih.sh index 6a4d1a5a..5da8f6e3 100755 --- a/metrics/aocih.sh +++ b/metrics/aocih.sh @@ -31,12 +31,18 @@ age_in_hours=0 cd "$(dirname "${java_file}")" if git status > /dev/null 2>&1; then - repo_first_commit=$(git log --reverse --format=%at | head -1) - file_first_commit=$(git log --diff-filter=A --format=%at -- "$java_file" | tail -1) - if [[ -n "$file_first_commit" ]]; then + repo_first_commit=$(git log --reverse --format=%at | head -n 1 || true) + file_first_commit=$(git log --diff-filter=A --format=%at -- "$java_file" | tail -n 1 || true) + + if [[ -n "$repo_first_commit" && -n "$file_first_commit" ]]; then age_in_seconds=$((file_first_commit - repo_first_commit)) age_in_hours=$((age_in_seconds / 3600)) + else + echo "Warning: Unable to calculate commit age for $java_file" >&2 fi +else + echo "Error: Not a Git repository or unable to access it." >&2 fi + echo "AoCiH $age_in_hours Age of Class in Hours" > "$output" diff --git a/requirements.txt b/requirements.txt index 42cb3da2..32387a44 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,3 @@ chardet==5.2.0 mypy==1.12.1 cffconvert==2.0.0 samples-filter==0.5.1 -repo-reasoner==0.0.25 \ No newline at end of file diff --git a/steps/discover.sh b/steps/discover.sh index fb6747b6..cb98bf6f 100755 --- a/steps/discover.sh +++ b/steps/discover.sh @@ -69,20 +69,6 @@ elif [ -z "${REPOS}" ] || [ ! -e "${REPOS}" ]; then samples-filter filter "${fargs[@]}" rm "${csv}" mv "${nosamples}" "${csv}" - - maintained=${TARGET}/maintained.csv - declare -a margs=( \ - "--repositories=${csv}" \ - "--output=${maintained}" \ - "--key=${GIGACHAT_KEY}" \ - "--model=${GIGACHAT_MODEL}" - ) - repo-reasoner filter-unmaintained "${margs[@]}" - original_csv="${csv}.original" - mv "${csv}" "${original_csv}" - mv "${maintained}" "${csv}" - - else echo "Using the list of repositories from the '${REPOS}' file (defined by the REPOS environment variable)..." cat "${REPOS}" > "${csv}" diff --git a/steps/env.sh b/steps/env.sh index 04c578d9..993672fd 100755 --- a/steps/env.sh +++ b/steps/env.sh @@ -68,12 +68,6 @@ jq --version multimetric --help > /dev/null -repo-reasoner --help > /dev/null - -rubocop -v - -inkscape --version - awk --version parallel --version From f10ec83aa41cd915ef73554330cea569ec16d6a3 Mon Sep 17 00:00:00 2001 From: dzhovi Date: Sat, 18 Jan 2025 16:45:59 +0300 Subject: [PATCH 10/15] fix --- Makefile | 2 +- steps/aggregation-functions/90-percentile.sh | 2 -- steps/aggregation-functions/mean.sh | 3 --- steps/aggregation-functions/median.sh | 3 --- 4 files changed, 1 insertion(+), 9 deletions(-) diff --git a/Makefile b/Makefile index abd5b927..0408a582 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,7 @@ define step endef # The main goal -all: env $(TARGET)/start.txt $(TARGET)/repositories.csv clone unregister jpeek filter measure aggregate summarize zip +all: env $(TARGET)/start.txt $(TARGET)/repositories.csv polish clone unregister jpeek filter measure aggregate summarize zip echo -e "\n\nSUCCESS (made by yegor256/cam $(VERSION)$$("$${LOCAL}/help/tdiff.sh" "$$(cat "$(TARGET)/start.txt")"))!" install: diff --git a/steps/aggregation-functions/90-percentile.sh b/steps/aggregation-functions/90-percentile.sh index 07326d26..2d583b40 100755 --- a/steps/aggregation-functions/90-percentile.sh +++ b/steps/aggregation-functions/90-percentile.sh @@ -24,8 +24,6 @@ set -e set -o pipefail -export LC_NUMERIC=C - metric_file=$1 output_folder=$2 metric_name=$3 diff --git a/steps/aggregation-functions/mean.sh b/steps/aggregation-functions/mean.sh index aad834a4..96c1aed9 100755 --- a/steps/aggregation-functions/mean.sh +++ b/steps/aggregation-functions/mean.sh @@ -24,9 +24,6 @@ set -e set -o pipefail -export LC_NUMERIC=C - - metric_file=$1 output_folder=$2 metric_name=$3 diff --git a/steps/aggregation-functions/median.sh b/steps/aggregation-functions/median.sh index d1f10521..a63ab03f 100755 --- a/steps/aggregation-functions/median.sh +++ b/steps/aggregation-functions/median.sh @@ -24,9 +24,6 @@ set -e set -o pipefail -export LC_NUMERIC=C - - metric_file=$1 output_folder=$2 metric_name=$3 From 6746fed439a077d0294cbe3f45c5d089439f1357 Mon Sep 17 00:00:00 2001 From: dzhovi Date: Sun, 19 Jan 2025 13:16:49 +0300 Subject: [PATCH 11/15] automatic year determination in test-discover-repos.sh --- tests/steps/test-discover-repos.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/steps/test-discover-repos.sh b/tests/steps/test-discover-repos.sh index 3cb31196..b4935119 100755 --- a/tests/steps/test-discover-repos.sh +++ b/tests/steps/test-discover-repos.sh @@ -32,7 +32,7 @@ tex=${TARGET}/foo.tex rm -f "${csv}" msg=$("${LOCAL}/steps/discover-repos.rb" --dry --pause=0 --total=3 --page-size=1 --min-stars=100 --max-stars=1000 "--csv=${csv}" "--tex=${tex}") echo "${msg}" - echo "${msg}" | grep "Completed querying for year 2025. Found 3 repositories so far." + echo "${msg}" | grep "Completed querying for year $(date +%Y). Found 3 repositories so far." echo "${msg}" | grep "Found 3 total repositories in GitHub" test -e "${csv}" test -s "${tex}" From 2a4d295935e463cd987880d8c82e475553dc8338 Mon Sep 17 00:00:00 2001 From: dzhovi Date: Tue, 21 Jan 2025 18:26:55 +0300 Subject: [PATCH 12/15] temp --- steps/env.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/steps/env.sh b/steps/env.sh index 993672fd..30bc4bff 100755 --- a/steps/env.sh +++ b/steps/env.sh @@ -42,6 +42,7 @@ fi ruby -v rubocop -v + if [[ "$(python3 --version 2>&1 | cut -f2 -d' ')" =~ ^[1-2] ]]; then python3 --version echo "Python must be 3+" From 9b99f3b37520a8d153aa1c2422c8bf80df0b4a40 Mon Sep 17 00:00:00 2001 From: dzhovi Date: Wed, 22 Jan 2025 17:37:31 +0300 Subject: [PATCH 13/15] removal of emply lines --- metrics/aocih.sh | 2 -- steps/env.sh | 1 - 2 files changed, 3 deletions(-) diff --git a/metrics/aocih.sh b/metrics/aocih.sh index 5da8f6e3..8816fba4 100755 --- a/metrics/aocih.sh +++ b/metrics/aocih.sh @@ -33,7 +33,6 @@ cd "$(dirname "${java_file}")" if git status > /dev/null 2>&1; then repo_first_commit=$(git log --reverse --format=%at | head -n 1 || true) file_first_commit=$(git log --diff-filter=A --format=%at -- "$java_file" | tail -n 1 || true) - if [[ -n "$repo_first_commit" && -n "$file_first_commit" ]]; then age_in_seconds=$((file_first_commit - repo_first_commit)) age_in_hours=$((age_in_seconds / 3600)) @@ -44,5 +43,4 @@ else echo "Error: Not a Git repository or unable to access it." >&2 fi - echo "AoCiH $age_in_hours Age of Class in Hours" > "$output" diff --git a/steps/env.sh b/steps/env.sh index 30bc4bff..993672fd 100755 --- a/steps/env.sh +++ b/steps/env.sh @@ -42,7 +42,6 @@ fi ruby -v rubocop -v - if [[ "$(python3 --version 2>&1 | cut -f2 -d' ')" =~ ^[1-2] ]]; then python3 --version echo "Python must be 3+" From 985db8c90833cad455ac0dbadab70cfadb3643f7 Mon Sep 17 00:00:00 2001 From: dzhovi Date: Wed, 22 Jan 2025 18:17:47 +0300 Subject: [PATCH 14/15] added lc_numeric --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 0408a582..df981f5d 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,9 @@ JPEEK = /opt/app/jpeek.jar # Options for all Java processes JAVA_OPTS=-Xmx128m +# defines rules and symbols for formatting numeric information +LC_NUMERIC=C + # Make all variables from this Makefile visible in all steps/*.sh export From 3b99874ef77017e22124039954974edd2c9f17fa Mon Sep 17 00:00:00 2001 From: dzhovi Date: Wed, 22 Jan 2025 18:36:54 +0300 Subject: [PATCH 15/15] shortened url, the error encountered was: The length of the URL in 'howpublished' is too big (67 > 64), use URL shoftener: 'https://github.blog/2023-01-25-100-million-developers-and-counting/', in the 'dohmke2023' entry --- tex/report.bib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tex/report.bib b/tex/report.bib index 20664eff..eaa93db8 100644 --- a/tex/report.bib +++ b/tex/report.bib @@ -161,7 +161,7 @@ @article{bieman1995cohesion @misc{dohmke2023, author = {Dohmke, Thomas}, - howpublished = {\url{https://github.blog/2023-01-25-100-million-developers-and-counting/}}, + howpublished = {\url{https://shorturl.at/oWGK4}}, note = {[Online; accessed 13-03-2024]}, title = {{100 Million Developers and Counting}}, year = {2023},