fix: some failing pre-commits and add conventional commit message for…

…mat check

ISSUE: #1935

CHANGELOG:
- [x] Addresses error handling and parallelized formatting for toml files.
- [x] Configures the ruff formatter to use requested format.
- [x] Clean up some lint.
- [x] Update copyright headers in files missing them.
- [x] Add scripts to add and check license headers.
- [x] Add checks to ensure we have a conventional commit message format.
- [x] Add a commit message template that uses the conventional commit message format.
- [x] Add biomejs configuration for later use.

.github/labeler.yml

@@ -1,3 +1,6 @@
+# Copyright 2025 Google LLC
+# SPDX-License-Identifier: Apache-2.0
+
 # Add 'root' label to any root file changes
 # Quotation marks are required for the leading asterisk
 root:

.github/workflows/labeler.yml

@@ -1,3 +1,6 @@
+# Copyright 2025 Google LLC
+# SPDX-License-Identifier: Apache-2.0
+
 name: "Pull Request Labeler"
 
 on:

.github/workflows/python.yml

@@ -1,3 +1,6 @@
+# Copyright 2025 Google LLC
+# SPDX-License-Identifier: Apache-2.0
+
 name: Python Checks
 
 on: pull_request
@@ -11,9 +14,6 @@ jobs:
       matrix:
         python-version:
           - "3.12"
-    defaults:
-      run:
-        working-directory: py
 
     steps:
       - uses: actions/checkout@v4
@@ -28,16 +28,19 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Format check
-        run: uv run ruff format --check .
+        run: uv run --directory py ruff format --check .
 
       - name: Lint with ruff
-        run: uv run ruff check .
+        run: uv run --directory py ruff check .
+
+      - name: Check licenses
+        run: ./bin/check_license
 
       - name: Run tests
-        run: ./bin/run_tests
+        run: ./py/bin/run_python_tests
 
       - name: Build documentation
-        run: uv run mkdocs build --strict
+        run: uv run --directory py mkdocs build --strict
 
       - name: Build distributions
-        run: ./bin/build_dists
+        run: ./py/bin/build_dists

.hooks/conventional-commit-msg

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+#
+# Copyright 2025 Google LLC
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+cat "$1" | convco check --from-stdin

COMMIT_MESSAGE_TEMPLATE

@@ -0,0 +1,45 @@
+feat:
+
+ISSUE:
+
+CHANGELOG:
+- [ ]
+
+## COMMIT MESSAGE FULL EXAMPLE
+#
+# feat(user-authentication): Implement two-factor authentication
+#
+# This commit introduces two-factor authentication for enhanced security.
+# It uses TOTP and requires users to configure an authenticator app.
+#
+# ISSUE: #123
+#
+# CHANGELOG:
+# - [ ] Add support for two-factor authentication
+# - [ ] Update user login endpoint to require two-factor authentication
+#
+# BREAKING CHANGE: The API endpoint for user login has been modified.
+
+## CONVENTIONAL COMMIT TEMPLATE
+#
+# Subject line (required, max 50 characters, use imperative mood):
+# <type>(<scope>): <short description>
+# Example: feat(user-authentication): Implement two-factor authentication
+#
+# Body (optional, wrap at 72 characters, explain the change in more detail, mention why and what):
+# <detailed description>
+
+## TYPES OF CHANGE (choose one):
+# - feat:       A new feature
+# - fix:        A bug fix
+# - docs:       Documentation changes
+# - style:      Code style changes (formatting, etc.)
+# - refactor:   Code refactoring (no new features or bug fixes)
+# - perf:       Performance improvements
+# - test:       Adding or modifying tests
+# - build:      Changes that affect the build system or external dependencies
+# - ci:         Changes to CI configuration files and scripts
+# - chore:      Routine tasks, build process changes, etc.
+# - revert:     Revert a previous commit
+#
+## SCOPE (optional, specify the affected area, e.g., component, module):

bin/add_license

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+#
+# Copyright 2025 Google LLC
+# SPDX-License-Identifier: Apache-2.0
+#
+# Adds a license header to all files that don't already have it.
+
+# set -x # Uncomment to enable tracing.
+set -euo pipefail
+
+TOP_DIR=$(git rev-parse --show-toplevel)
+
+if ! command -v addlicense &>/dev/null; then
+  if ! command -v go &>/dev/null; then
+    echo "Please install go"
+    exit 1
+  fi
+  echo "Installing addlicense..."
+  go install github.com/google/addlicense@latest
+fi
+
+# NOTE: If you edit the ignore patterns, make sure to update the ignore patterns
+# in the corresponding check_license script.
+$HOME/go/bin/addlicense \
+  -c "Google LLC" \
+  -s=only \
+  -l apache \
+  -ignore '**/.dist/**/*' \
+  -ignore '**/.eggs/**/*' \
+  -ignore '**/.idea/**/*' \
+  -ignore '**/.mypy_cache/**/*' \
+  -ignore '**/.next/**/*' \
+  -ignore '**/.output/**/*' \
+  -ignore '**/.pytest_cache/**/*' \
+  -ignore '**/.ruff_cache/**/*' \
+  -ignore '**/.venv/**/*' \
+  -ignore '**/.wxt/**/*' \
+  -ignore '**/__pycache__/**/*' \
+  -ignore '**/bazel-*/**/*' \
+  -ignore '**/coverage/**/*' \
+  -ignore '**/develop-eggs/**/*' \
+  -ignore '**/dist/**/*' \
+  -ignore '**/node_modules/**/*' \
+  -ignore '**/pnpm-lock.yaml' \
+  -ignore '.nx/**/*' \
+  -ignore '.trunk/**/*' \
+  "$TOP_DIR"

bin/check_license

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+#
+# Copyright 2025 Google LLC
+# SPDX-License-Identifier: Apache-2.0
+#
+# Checks that all files have a license header.
+
+# set -x # Uncomment to enable tracing.
+set -euo pipefail
+
+TOP_DIR=$(git rev-parse --show-toplevel)
+
+if ! command -v addlicense &>/dev/null; then
+  if ! command -v go &>/dev/null; then
+    echo "Please install go"
+    exit 1
+  fi
+  echo "Installing addlicense..."
+  go install github.com/google/addlicense@latest
+fi
+
+export PATH=$(go env GOPATH):$PATH
+
+# NOTE: If you edit the ignore patterns, make sure to update the ignore patterns
+# in the corresponding add_license script.
+$HOME/go/bin/addlicense \
+  -check \
+  -c "Google LLC" \
+  -s=only \
+  -l apache \
+  -ignore '**/.dist/**/*' \
+  -ignore '**/.eggs/**/*' \
+  -ignore '**/.idea/**/*' \
+  -ignore '**/.mypy_cache/**/*' \
+  -ignore '**/.next/**/*' \
+  -ignore '**/.output/**/*' \
+  -ignore '**/.pytest_cache/**/*' \
+  -ignore '**/.ruff_cache/**/*' \
+  -ignore '**/.venv/**/*' \
+  -ignore '**/.wxt/**/*' \
+  -ignore '**/__pycache__/**/*' \
+  -ignore '**/bazel-*/**/*' \
+  -ignore '**/coverage/**/*' \
+  -ignore '**/develop-eggs/**/*' \
+  -ignore '**/dist/**/*' \
+  -ignore '**/node_modules/**/*' \
+  -ignore '**/pnpm-lock.yaml' \
+  -ignore '.nx/**/*' \
+  -ignore '.trunk/**/*' \
+  "$TOP_DIR"

py/bin/fmt → bin/fmt

@@ -15,25 +15,17 @@ fi
 
 TOP_DIR=$(git rev-parse --show-toplevel)
 
-addlicense \
-  -c "Google LLC" \
-  -s=only \
-  -ignore '**/.github/**/*' \
-  -ignore '**/.mypy_cache/**/*' \
-  -ignore '**/bazel-*/**/*' \
-  -ignore '**/docs/**/*' \
-  -ignore '**/node_modules/**/*' \
-  -ignore '**/pnpm-lock.yaml' \
-  "$TOP_DIR"
+# Add license header to all files that don't already have it.
+"${TOP_DIR}/bin/add_license"
 
 # Format all TOML files.
-"${TOP_DIR}/py/bin/format_toml_files"
+"${TOP_DIR}/bin/format_toml_files"
 if [[ $? -ne 0 ]]; then
   exit 1
 fi
 
 # Format all Python code.
-uvx ruff format "${TOP_DIR}/py"
+uv run --directory "${TOP_DIR}/py" ruff format .
 if [[ $? -ne 0 ]]; then
   exit 1
 fi
@@ -47,9 +39,13 @@ fi
 popd
 
 # Format all TypeScript code.
-pushd ${TOP_DIR}
-pnpm run format
-if [[ $? -ne 0 ]]; then
-  exit 1
-fi
-popd
+#
+# TODO: Re-enable once we have biome configured and enabled because that is
+# several times faster and compatible.
+#
+#pushd ${TOP_DIR}
+#pnpm run format
+#if [[ $? -ne 0 ]]; then
+#  exit 1
+#fi
+#popd

bin/format_toml_files

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+#
+# Format all TOML files in the project.
+#
+# Copyright 2025 Google LLC
+# SPDX-License-Identifier: Apache-2.0
+
+set -euo pipefail
+
+TOP_DIR=$(git rev-parse --show-toplevel)
+
+if command -v taplo >/dev/null 2>&1; then
+  if [ ! -f "${TOP_DIR}/taplo.toml" ]; then
+    echo "error: config file not found at ${TOP_DIR}/taplo.toml"
+    exit 1
+  fi
+
+  FORMATTER_COMMAND="taplo format --config ${TOP_DIR}/taplo.toml"
+  if command -v rust-parallel >/dev/null 2>&1; then
+    FORMATTER_COMMAND="rust-parallel -j4 ${FORMATTER_COMMAND}"
+  else
+    echo "warning: it is recommended to install https://crates.io/crates/rust-parallel for faster formatting"
+  fi
+
+  pushd "${TOP_DIR}"
+  if command -v fd >/dev/null 2>&1; then
+    echo "Using fd"
+    fd -e toml \
+      --exclude '**/*.egg-info/**' \
+      --exclude '**/.dist/**' \
+      --exclude '**/.next/**' \
+      --exclude '**/.output/**' \
+      --exclude '**/.pytest_cache/**' \
+      --exclude '**/.venv/**' \
+      --exclude '**/__pycache__/**' \
+      --exclude '**/bazel-*/**' \
+      --exclude '**/build/**' \
+      --exclude '**/develop-eggs/**' \
+      --exclude '**/dist/**' \
+      --exclude '**/eggs/**' \
+      --exclude '**/node_modules/**' \
+      --exclude '**/sdist/**' \
+      --exclude '**/site/**' \
+      --exclude '**/target/**' \
+      --exclude '**/venv/**' \
+      --exclude '**/wheels/**' |
+      ${FORMATTER_COMMAND}
+  else
+    echo "Please install https://github.com/sharkdp/fd to find files to format."
+  fi
+  popd
+else
+  echo "Please install https://github.com/tamasfe/taplo to format TOML files."
+fi

py/bin/setup → bin/setup

@@ -5,6 +5,15 @@
 # Copyright 2025 Google LLC
 # SPDX-License-Identifier: Apache-2.0
 
+# NOTE: This script is not specific to any particular runtime. It is intended to
+# be used as a convenience script for eng so that all the runtimes are set up in
+# a consistent manner so that pre-commit hooks run properly and the environment
+# is consistent.
+
+# TODO: This script is nowhere close to perfect. At a later date, we can replace
+# this with something like nix to have a reproducible environment. For now this
+# is a convenience script just to get eng started as quickly as possible.
+
 if ((EUID == 0)) && [[ -z ${DANGEROUSLY_RUN_AS_ROOT+x} ]]; then
   echo "Please do not run as root unless DANGEROUSLY_RUN_AS_ROOT is set."
   exit 1
@@ -124,7 +133,7 @@ function genkit::install_prerequisites() {
       python3 \
       ripgrep
   else
-    echo "Unsupported OS. Please install protoc manually."
+    echo "Unsupported OS. Please install tools manually."
   fi
 
   genkit::install_rust
@@ -172,11 +181,10 @@ function genkit::install_google_cloud_sdk() {
   if command -v gcloud &>/dev/null; then
     gcloud config set disable_usage_reporting true
     gcloud components update
-    return 0
+  else
+    curl https://sdk.cloud.google.com | bash -s -- --disable-prompts
+    gcloud config set disable_usage_reporting true
   fi
-
-  curl https://sdk.cloud.google.com | bash -s -- --disable-prompts
-  gcloud config set disable_usage_reporting true
 }
 
 # Install all the required tools that have been written in Go.
@@ -202,6 +210,7 @@ function genkit::install_go_cli_tools_eng() {
 function genkit::install_cargo_cli_tools_eng() {
   cargo install --locked \
     convco \
+    pylyzer \
     rust-parallel \
     taplo-cli
 }
@@ -240,9 +249,17 @@ function genkit::install_docs_cli_tools() {
     --with mkdocstrings[python]
 }
 
+# Configure the commit message template.
+function genkit::configure_commit_template() {
+  echo "Setting up commit message template..."
+  ln -sf "${TOP_DIR}/COMMIT_MESSAGE_TEMPLATE" "${TOP_DIR}/.git/COMMIT_MESSAGE_TEMPLATE"
+  git config commit.template "${TOP_DIR}/.git/COMMIT_MESSAGE_TEMPLATE"
+}
+
 # Install pre-commit hooks.
 function genkit::install_pre_commit_hooks() {
-  captainhook install -f -c "${TOP_DIR}/py/captainhook.json"
+  genkit::configure_commit_template
+  captainhook install -f -c "${TOP_DIR}/captainhook.json"
 }
 
 # Setup genkit.
@@ -273,8 +290,8 @@ function genkit::install_eng_packages() {
   genkit::install_common_packages
   genkit::install_go_cli_tools_eng
   genkit::install_cargo_cli_tools_eng
-  genkit::install_google_cloud_sdk
   genkit::install_pre_commit_hooks
+  genkit::install_google_cloud_sdk
   genkit::setup_genkit
 }