chore(script/update-packages): Fetch github releases/tags via batched…

… GraphQL queries
termux · Feb 1, 2025 · a77d809 · a77d809
1 parent 00f03e9
commit a77d809
Showing 1 changed file with 112 additions and 4 deletions.
diff --git a/scripts/bin/update-packages b/scripts/bin/update-packages
@@ -31,6 +31,9 @@ TERMUX_PACKAGES_DIRECTORIES=$(jq --raw-output 'del(.pkg_format) | keys | .[]' "$
 # shellcheck source=scripts/properties.sh
 . "${TERMUX_SCRIPTDIR}/scripts/properties.sh"
 
+# my GH_TOKEN for debugging, this is already passed in for the CI.
+. "${TERMUX_SCRIPTDIR}/gh.env" || :
+
 # Utility function to write error message to stderr.
 # shellcheck source=scripts/updates/utils/termux_error_exit.sh
 . "${TERMUX_SCRIPTDIR}"/scripts/updates/utils/termux_error_exit.sh
@@ -77,7 +80,7 @@ TERMUX_PACKAGES_DIRECTORIES=$(jq --raw-output 'del(.pkg_format) | keys | .[]' "$
 # shellcheck source=scripts/updates/termux_pkg_auto_update.sh
 . "${TERMUX_SCRIPTDIR}"/scripts/updates/termux_pkg_auto_update.sh
 
-# Converts milliseconds to human-readable format. 
+# Converts milliseconds to human-readable format.
 # Example: `ms_to_human_readable 123456789` => 34h 17m 36s 789ms
 ms_to_human_readable() {
 	echo "$(($1/3600000))h $(($1%3600000/60000))m $(($1%60000/1000))s $(($1%1000))ms" | sed 's/0h //;s/0m //;s/0s //'
@@ -94,6 +97,15 @@ quiet() {
 	return $?
 }
 
+# Check GitHub rate limits
+_gh_check_rate_limit() {
+	curl -L \
+		-H "Accept: application/vnd.github+json" \
+		-H "Authorization: Bearer $GITHUB_TOKEN" \
+		-H "X-GitHub-Api-Version: 2022-11-28" \
+		https://api.github.com/rate_limit | jq '.resources.core'
+}
+
 _update() {
 	export TERMUX_PKG_NAME
 	TERMUX_PKG_NAME="$(basename "$1")"
@@ -117,6 +129,10 @@ _update() {
 	. "${TERMUX_PKG_BUILDER_DIR}"/build.sh 2>/dev/null
 	set -e -u
 
+	[[ "$TERMUX_PKG_SRCURL" == *"github.com"* ]] && {
+		_gh_check_rate_limit
+	}
+
 	echo # Newline.
 	echo "INFO: Updating ${TERMUX_PKG_NAME} [Current version: ${TERMUX_PKG_VERSION}]"
 	termux_pkg_auto_update
@@ -148,17 +164,109 @@ _fetch_and_cache_tags() {
 	echo "INFO: Fetching and caching tags"
 
 	# First invocation of termux_repology_api_get_latest_version fetches and caches repology metadata.
-	quiet termux_repology_api_get_latest_version ' '
+	#quiet termux_repology_api_get_latest_version ' '
 
 	local __PACKAGES=()
+	local __GITHUB_PACKAGES=()
 	for repo_dir in $(jq --raw-output 'del(.pkg_format) | keys | .[]' "${TERMUX_SCRIPTDIR}/repo.json"); do
 		for pkg_dir in "${repo_dir}"/*; do
 			quiet _should_update "${pkg_dir}" || continue # Skip if not needed.
 			grep -q '^termux_pkg_auto_update' "${pkg_dir}/build.sh" && continue # Skip if package has custom auto-update
-			__PACKAGES+=("${pkg_dir}")
+			if grep '^TERMUX_PKG_SRCURL=' "${pkg_dir}/build.sh" | grep -q 'github.com'; then
+				__GITHUB_PACKAGES+=("${pkg_dir}")
+			else
+				__PACKAGES+=("${pkg_dir}")
+			fi
 		done
 	done
 
+
+	local GITHUB_GRAPHQL_QUERIES=() COUNTER=0 FILTER_REGEX=""
+	declare -A __GITHUB_GRAPHQL_PACKAGES=()
+	# Ignore non-constant sources
+	# shellcheck disable=SC1091
+	for pkg_dir in "${__GITHUB_PACKAGES[@]}"; do
+		local PKG_SRCURL TAG_TYPE OWNER REPO
+		read -r PKG_SRCURL TAG_TYPE < <(
+			set +u
+			source "${pkg_dir}/build.sh"
+			echo "${TERMUX_PKG_SRCURL} ${TERMUX_PKG_UPDATE_TAG_TYPE}"
+		)
+
+		read -r OWNER REPO < <(sed -E 's|^(git\+)?https?://github.com/([^/]+)/([^/]+).*|\2 \3|' <<< "$PKG_SRCURL")
+
+		if [[ -z "${TAG_TYPE}" ]]; then # If not set, then decide on the basis of url.
+			if [[ "${PKG_SRCURL:0:4}" == "git+" ]]; then
+				# Get newest tag.
+				TAG_TYPE="newest-tag"
+			else
+				# Get the latest release tag.
+				TAG_TYPE="latest-release-tag"
+			fi
+		fi
+
+		# This holds the source repositories, and query type for each repo we need to query.
+		# Newlines need to be escaped for the query parser
+		GITHUB_GRAPHQL_QUERIES+=( "_$((COUNTER++)): repository(owner: \\\"${OWNER}\\\", name: \\\"${REPO}\\\") { ..._${TAG_TYPE//-/_} }" )
+
+		unset PKG_SRCURL TAG_TYPE OWNER REPO
+	done
+
+	# Batch size for fetching tags, 250 seems to work consistently.
+	local BATCH BATCH_SIZE=100
+	echo "// Begin test: $(date)" > /tmp/query-12345
+	for (( BATCH = 0; ${#GITHUB_GRAPHQL_QUERIES[@]} >= BATCH_SIZE * BATCH ; BATCH++ )); do
+		# JSON strings cannot contain tabs or newlines
+		# so shutup shellcheck complaining about escapes in single quotes
+		local QUERY
+
+		# Start the GraphQL query with our two fragments for getting the latest tag from a release, and from refs/tags
+		# These are defined only if needed, so this one is for '_latest_release_tag'
+		grep -q '_latest_release_tag' <<< "${GITHUB_GRAPHQL_QUERIES[@]:$BATCH * $BATCH_SIZE:$BATCH_SIZE}" && {
+			QUERY+="$(printf '%s\n' \
+			'fragment _latest_release_tag on Repository {' \
+			'  releases(first: 1, orderBy: {field: CREATED_AT, direction: DESC}) {' \
+			'    nodes { tagName }' \
+			'  }' \
+			'}')"
+		}
+
+		grep -q '_newest_tag' <<< "${GITHUB_GRAPHQL_QUERIES[@]:$BATCH * $BATCH_SIZE:$BATCH_SIZE}" && {
+			QUERY+="$(printf '%s\n' \
+			'fragment _newest_tag on Repository {' \
+			'  refs( refPrefix: \"refs/tags/\" first: 1 orderBy: {field: TAG_COMMIT_DATE, direction: DESC}) {' \
+			'    nodes { name }' \
+			'  }' \
+			'}')"
+		}
+
+		QUERY+='query {'
+
+		# Fill out the query body with the package repos we need to query for updates
+		# Lastly fetch the rate limit utilization
+		echo "// Batch: $BATCH" >> /tmp/query-12345
+		echo "Starting Batch $BATCH at: ${GITHUB_GRAPHQL_QUERIES[$BATCH * $BATCH_SIZE]}"
+		printf -v QUERY '%s\n' \
+				"${QUERY}" \
+				"${GITHUB_GRAPHQL_QUERIES[@]:$BATCH * $BATCH_SIZE:$BATCH_SIZE}" \
+				'ratelimit: rateLimit { cost limit remaining used resetAt }' \
+				'}' \
+
+		printf '%s' "${QUERY}" >> /tmp/query-12345
+		response="$(printf '{ "query": "%s" }' "${QUERY//$'\n'/ }" | curl -fsL \
+			-H "Authorization: token ${GITHUB_TOKEN}" \
+			-H "Accept: application/vnd.github.v3+json" \
+			-H 'Content-Type: application/json' \
+			-X POST \
+			--data @- \
+			https://api.github.com/graphql
+		)"
+			# --write-out '%{json}%{header_json}' \
+		jq . <<< "$response"
+		unset QUERY
+	done
+	exit 0 # Just bail for now while we're still testing this.
+
 	# This is called indirectly in the ninja file
 	# So silence shellcheck's unreachable code warning
 	# shellcheck disable=SC2317
@@ -194,7 +302,7 @@ _fetch_and_cache_tags() {
 	local LATEST_TAGS
 	LATEST_TAGS="$(
 		F="$(declare -p TERMUX_SCRIPTDIR GITHUB_TOKEN TERMUX_REPOLOGY_DATA_FILE); $(declare -f __main__ | sed 's/__main__ ()//')" \
-		env --chdir=/tmp ninja -f /dev/stdin < <(__generate__ |& grep "^PKG|")
+		env --chdir=/tmp ninja -f /dev/stdin < <(__generate__) |& grep "^PKG|"
 	)"
 
 	unset -f __main__ __generate__