From 8a495a4b50b8156419ae6f5b3a650ff728847e02 Mon Sep 17 00:00:00 2001 From: Yesudeep Mangalapilly Date: Fri, 14 Feb 2025 00:15:11 -0800 Subject: [PATCH] fix: formatting conventional commit messages and fix some pre-commit breakage #1935 ISSUE: https://github.com/firebase/genkit/issues/1935 CHANGELOG: - [ ] Addresses error handling and parallelized formatting for toml files. - [ ] Configures the ruff formatter to use requested format. - [ ] Clean up some lint. - [ ] Update copyright headers in files missing them. - [ ] Add scripts to add and check license headers. - [ ] Add checks to ensure we have a conventional commit message format. - [ ] Add a commit message template that uses the conventional commit messafe format. - [ ] Add biomejs configuration for later use. --- .github/labeler.yml | 3 + .github/workflows/labeler.yml | 3 + .github/workflows/python.yml | 19 ++--- .../commit-message-format-pre-push | 0 .hooks/conventional-commit-msg | 8 +++ {py/.hooks => .hooks}/no-commits-on-branches | 0 COMMIT_MESSAGE_TEMPLATE | 45 ++++++++++++ bin/add_license | 47 +++++++++++++ {py/bin => bin}/check-licenses | 0 bin/check_license | 50 ++++++++++++++ {py/bin => bin}/fmt | 32 ++++----- bin/format_toml_files | 54 +++++++++++++++ {py/bin => bin}/setup | 31 +++++++-- biome.json | 69 +++++++++++++++++++ py/captainhook.json => captainhook.json | 18 ++--- package.json | 1 + py/bin/format_toml_files | 60 ---------------- py/bin/generate_schema_types | 20 +++--- py/bin/{run_tests => run_python_tests} | 0 py/bin/sanitize_schemas.py | 5 +- py/packages/genkit/src/genkit/core/action.py | 5 +- .../genkit/src/genkit/core/reflection.py | 12 ++-- py/packages/genkit/src/genkit/core/tracing.py | 10 ++- py/pyproject.toml | 30 ++++---- py/uv.lock | 57 ++++----------- py/taplo.toml => taplo.toml | 0 26 files changed, 396 insertions(+), 183 deletions(-) rename {py/.hooks => .hooks}/commit-message-format-pre-push (100%) create mode 100755 .hooks/conventional-commit-msg rename {py/.hooks => .hooks}/no-commits-on-branches (100%) create mode 100644 COMMIT_MESSAGE_TEMPLATE create mode 100755 bin/add_license rename {py/bin => bin}/check-licenses (100%) create mode 100755 bin/check_license rename {py/bin => bin}/fmt (58%) create mode 100755 bin/format_toml_files rename {py/bin => bin}/setup (87%) create mode 100644 biome.json rename py/captainhook.json => captainhook.json (81%) delete mode 100755 py/bin/format_toml_files rename py/bin/{run_tests => run_python_tests} (100%) rename py/taplo.toml => taplo.toml (100%) diff --git a/.github/labeler.yml b/.github/labeler.yml index e71752dc3..77856994e 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,3 +1,6 @@ +# Copyright 2025 Google LLC +# SPDX-License-Identifier: Apache-2.0 + # Add 'root' label to any root file changes # Quotation marks are required for the leading asterisk root: diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 09cf4ca17..a960b4a70 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,3 +1,6 @@ +# Copyright 2025 Google LLC +# SPDX-License-Identifier: Apache-2.0 + name: "Pull Request Labeler" on: diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 2c8b96fcb..4cb0e98cb 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -1,3 +1,6 @@ +# Copyright 2025 Google LLC +# SPDX-License-Identifier: Apache-2.0 + name: Python Checks on: pull_request @@ -11,9 +14,6 @@ jobs: matrix: python-version: - "3.12" - defaults: - run: - working-directory: py steps: - uses: actions/checkout@v4 @@ -28,16 +28,19 @@ jobs: python-version: ${{ matrix.python-version }} - name: Format check - run: uv run ruff format --check . + run: uv run --directory py ruff format --check . - name: Lint with ruff - run: uv run ruff check . + run: uv run --directory py ruff check . + + - name: Check licenses + run: ./bin/check_license - name: Run tests - run: ./bin/run_tests + run: ./py/bin/run_python_tests - name: Build documentation - run: uv run mkdocs build --strict + run: uv run --directory py mkdocs build --strict - name: Build distributions - run: ./bin/build_dists + run: ./py/bin/build_dists diff --git a/py/.hooks/commit-message-format-pre-push b/.hooks/commit-message-format-pre-push similarity index 100% rename from py/.hooks/commit-message-format-pre-push rename to .hooks/commit-message-format-pre-push diff --git a/.hooks/conventional-commit-msg b/.hooks/conventional-commit-msg new file mode 100755 index 000000000..f04673cfe --- /dev/null +++ b/.hooks/conventional-commit-msg @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# +# Copyright 2025 Google LLC +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +cat "$1" | convco check --from-stdin diff --git a/py/.hooks/no-commits-on-branches b/.hooks/no-commits-on-branches similarity index 100% rename from py/.hooks/no-commits-on-branches rename to .hooks/no-commits-on-branches diff --git a/COMMIT_MESSAGE_TEMPLATE b/COMMIT_MESSAGE_TEMPLATE new file mode 100644 index 000000000..45f669ea7 --- /dev/null +++ b/COMMIT_MESSAGE_TEMPLATE @@ -0,0 +1,45 @@ +feat: + +ISSUE: + +CHANGELOG: +- [ ] + +## COMMIT MESSAGE FULL EXAMPLE +# +# feat(user-authentication): Implement two-factor authentication +# +# This commit introduces two-factor authentication for enhanced security. +# It uses TOTP and requires users to configure an authenticator app. +# +# ISSUE: #123 +# +# CHANGELOG: +# - [ ] Add support for two-factor authentication +# - [ ] Update user login endpoint to require two-factor authentication +# +# BREAKING CHANGE: The API endpoint for user login has been modified. + +## CONVENTIONAL COMMIT TEMPLATE +# +# Subject line (required, max 50 characters, use imperative mood): +# (): +# Example: feat(user-authentication): Implement two-factor authentication +# +# Body (optional, wrap at 72 characters, explain the change in more detail, mention why and what): +# + +## TYPES OF CHANGE (choose one): +# - feat: A new feature +# - fix: A bug fix +# - docs: Documentation changes +# - style: Code style changes (formatting, etc.) +# - refactor: Code refactoring (no new features or bug fixes) +# - perf: Performance improvements +# - test: Adding or modifying tests +# - build: Changes that affect the build system or external dependencies +# - ci: Changes to CI configuration files and scripts +# - chore: Routine tasks, build process changes, etc. +# - revert: Revert a previous commit +# +## SCOPE (optional, specify the affected area, e.g., component, module): diff --git a/bin/add_license b/bin/add_license new file mode 100755 index 000000000..e8fdb9a16 --- /dev/null +++ b/bin/add_license @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# +# Copyright 2025 Google LLC +# SPDX-License-Identifier: Apache-2.0 +# +# Adds a license header to all files that don't already have it. + +# set -x # Uncomment to enable tracing. +set -euo pipefail + +TOP_DIR=$(git rev-parse --show-toplevel) + +if ! command -v addlicense &>/dev/null; then + if ! command -v go &>/dev/null; then + echo "Please install go" + exit 1 + fi + echo "Installing addlicense..." + go install github.com/google/addlicense@latest +fi + +# NOTE: If you edit the ignore patterns, make sure to update the ignore patterns +# in the corresponding check_license script. +$HOME/go/bin/addlicense \ + -c "Google LLC" \ + -s=only \ + -l apache \ + -ignore '**/.dist/**/*' \ + -ignore '**/.eggs/**/*' \ + -ignore '**/.idea/**/*' \ + -ignore '**/.mypy_cache/**/*' \ + -ignore '**/.next/**/*' \ + -ignore '**/.output/**/*' \ + -ignore '**/.pytest_cache/**/*' \ + -ignore '**/.ruff_cache/**/*' \ + -ignore '**/.venv/**/*' \ + -ignore '**/.wxt/**/*' \ + -ignore '**/__pycache__/**/*' \ + -ignore '**/bazel-*/**/*' \ + -ignore '**/coverage/**/*' \ + -ignore '**/develop-eggs/**/*' \ + -ignore '**/dist/**/*' \ + -ignore '**/node_modules/**/*' \ + -ignore '**/pnpm-lock.yaml' \ + -ignore '.nx/**/*' \ + -ignore '.trunk/**/*' \ + "$TOP_DIR" diff --git a/py/bin/check-licenses b/bin/check-licenses similarity index 100% rename from py/bin/check-licenses rename to bin/check-licenses diff --git a/bin/check_license b/bin/check_license new file mode 100755 index 000000000..c48621876 --- /dev/null +++ b/bin/check_license @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# +# Copyright 2025 Google LLC +# SPDX-License-Identifier: Apache-2.0 +# +# Checks that all files have a license header. + +# set -x # Uncomment to enable tracing. +set -euo pipefail + +TOP_DIR=$(git rev-parse --show-toplevel) + +if ! command -v addlicense &>/dev/null; then + if ! command -v go &>/dev/null; then + echo "Please install go" + exit 1 + fi + echo "Installing addlicense..." + go install github.com/google/addlicense@latest +fi + +export PATH=$(go env GOPATH):$PATH + +# NOTE: If you edit the ignore patterns, make sure to update the ignore patterns +# in the corresponding add_license script. +$HOME/go/bin/addlicense \ + -check \ + -c "Google LLC" \ + -s=only \ + -l apache \ + -ignore '**/.dist/**/*' \ + -ignore '**/.eggs/**/*' \ + -ignore '**/.idea/**/*' \ + -ignore '**/.mypy_cache/**/*' \ + -ignore '**/.next/**/*' \ + -ignore '**/.output/**/*' \ + -ignore '**/.pytest_cache/**/*' \ + -ignore '**/.ruff_cache/**/*' \ + -ignore '**/.venv/**/*' \ + -ignore '**/.wxt/**/*' \ + -ignore '**/__pycache__/**/*' \ + -ignore '**/bazel-*/**/*' \ + -ignore '**/coverage/**/*' \ + -ignore '**/develop-eggs/**/*' \ + -ignore '**/dist/**/*' \ + -ignore '**/node_modules/**/*' \ + -ignore '**/pnpm-lock.yaml' \ + -ignore '.nx/**/*' \ + -ignore '.trunk/**/*' \ + "$TOP_DIR" diff --git a/py/bin/fmt b/bin/fmt similarity index 58% rename from py/bin/fmt rename to bin/fmt index fda0432d2..4e92254c7 100755 --- a/py/bin/fmt +++ b/bin/fmt @@ -15,25 +15,17 @@ fi TOP_DIR=$(git rev-parse --show-toplevel) -addlicense \ - -c "Google LLC" \ - -s=only \ - -ignore '**/.github/**/*' \ - -ignore '**/.mypy_cache/**/*' \ - -ignore '**/bazel-*/**/*' \ - -ignore '**/docs/**/*' \ - -ignore '**/node_modules/**/*' \ - -ignore '**/pnpm-lock.yaml' \ - "$TOP_DIR" +# Add license header to all files that don't already have it. +"${TOP_DIR}/bin/add_license" # Format all TOML files. -"${TOP_DIR}/py/bin/format_toml_files" +"${TOP_DIR}/bin/format_toml_files" if [[ $? -ne 0 ]]; then exit 1 fi # Format all Python code. -uvx ruff format "${TOP_DIR}/py" +uv run --directory "${TOP_DIR}/py" ruff format . if [[ $? -ne 0 ]]; then exit 1 fi @@ -47,9 +39,13 @@ fi popd # Format all TypeScript code. -pushd ${TOP_DIR} -pnpm run format -if [[ $? -ne 0 ]]; then - exit 1 -fi -popd +# +# TODO: Re-enable once we have biome configured and enabled because that is +# several times faster and compatible. +# +#pushd ${TOP_DIR} +#pnpm run format +#if [[ $? -ne 0 ]]; then +# exit 1 +#fi +#popd diff --git a/bin/format_toml_files b/bin/format_toml_files new file mode 100755 index 000000000..90e1689cc --- /dev/null +++ b/bin/format_toml_files @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# +# Format all TOML files in the project. +# +# Copyright 2025 Google LLC +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +TOP_DIR=$(git rev-parse --show-toplevel) + +if command -v taplo >/dev/null 2>&1; then + if [ ! -f "${TOP_DIR}/taplo.toml" ]; then + echo "error: config file not found at ${TOP_DIR}/taplo.toml" + exit 1 + fi + + FORMATTER_COMMAND="taplo format --config ${TOP_DIR}/taplo.toml" + if command -v rust-parallel >/dev/null 2>&1; then + FORMATTER_COMMAND="rust-parallel -j4 ${FORMATTER_COMMAND}" + else + echo "warning: it is recommended to install https://crates.io/crates/rust-parallel for faster formatting" + fi + + pushd "${TOP_DIR}" + if command -v fd >/dev/null 2>&1; then + echo "Using fd" + fd -e toml \ + --exclude '**/*.egg-info/**' \ + --exclude '**/.dist/**' \ + --exclude '**/.next/**' \ + --exclude '**/.output/**' \ + --exclude '**/.pytest_cache/**' \ + --exclude '**/.venv/**' \ + --exclude '**/__pycache__/**' \ + --exclude '**/bazel-*/**' \ + --exclude '**/build/**' \ + --exclude '**/develop-eggs/**' \ + --exclude '**/dist/**' \ + --exclude '**/eggs/**' \ + --exclude '**/node_modules/**' \ + --exclude '**/sdist/**' \ + --exclude '**/site/**' \ + --exclude '**/target/**' \ + --exclude '**/venv/**' \ + --exclude '**/wheels/**' | + ${FORMATTER_COMMAND} + else + echo "Please install https://github.com/sharkdp/fd to find files to format." + fi + popd +else + echo "Please install https://github.com/tamasfe/taplo to format TOML files." +fi diff --git a/py/bin/setup b/bin/setup similarity index 87% rename from py/bin/setup rename to bin/setup index 77180dc10..c18d59858 100755 --- a/py/bin/setup +++ b/bin/setup @@ -5,6 +5,15 @@ # Copyright 2025 Google LLC # SPDX-License-Identifier: Apache-2.0 +# NOTE: This script is not specific to any particular runtime. It is intended to +# be used as a convenience script for eng so that all the runtimes are set up in +# a consistent manner so that pre-commit hooks run properly and the environment +# is consistent. + +# TODO: This script is nowhere close to perfect. At a later date, we can replace +# this with something like nix to have a reproducible environment. For now this +# is a convenience script just to get eng started as quickly as possible. + if ((EUID == 0)) && [[ -z ${DANGEROUSLY_RUN_AS_ROOT+x} ]]; then echo "Please do not run as root unless DANGEROUSLY_RUN_AS_ROOT is set." exit 1 @@ -124,7 +133,7 @@ function genkit::install_prerequisites() { python3 \ ripgrep else - echo "Unsupported OS. Please install protoc manually." + echo "Unsupported OS. Please install tools manually." fi genkit::install_rust @@ -172,11 +181,10 @@ function genkit::install_google_cloud_sdk() { if command -v gcloud &>/dev/null; then gcloud config set disable_usage_reporting true gcloud components update - return 0 + else + curl https://sdk.cloud.google.com | bash -s -- --disable-prompts + gcloud config set disable_usage_reporting true fi - - curl https://sdk.cloud.google.com | bash -s -- --disable-prompts - gcloud config set disable_usage_reporting true } # Install all the required tools that have been written in Go. @@ -202,6 +210,7 @@ function genkit::install_go_cli_tools_eng() { function genkit::install_cargo_cli_tools_eng() { cargo install --locked \ convco \ + pylyzer \ rust-parallel \ taplo-cli } @@ -240,9 +249,17 @@ function genkit::install_docs_cli_tools() { --with mkdocstrings[python] } +# Configure the commit message template. +function genkit::configure_commit_template() { + echo "Setting up commit message template..." + ln -sf "${TOP_DIR}/COMMIT_MESSAGE_TEMPLATE" "${TOP_DIR}/.git/COMMIT_MESSAGE_TEMPLATE" + git config commit.template "${TOP_DIR}/.git/COMMIT_MESSAGE_TEMPLATE" +} + # Install pre-commit hooks. function genkit::install_pre_commit_hooks() { - captainhook install -f -c "${TOP_DIR}/py/captainhook.json" + genkit::configure_commit_template + captainhook install -f -c "${TOP_DIR}/captainhook.json" } # Setup genkit. @@ -273,8 +290,8 @@ function genkit::install_eng_packages() { genkit::install_common_packages genkit::install_go_cli_tools_eng genkit::install_cargo_cli_tools_eng - genkit::install_google_cloud_sdk genkit::install_pre_commit_hooks + genkit::install_google_cloud_sdk genkit::setup_genkit } diff --git a/biome.json b/biome.json new file mode 100644 index 000000000..c89309e04 --- /dev/null +++ b/biome.json @@ -0,0 +1,69 @@ +{ + "$schema": "https://biomejs.dev/schemas/1.9.4/schema.json", + "organizeImports": { + "enabled": true + }, + "linter": { + "enabled": true, + "rules": { + "recommended": true + } + }, + "files": { + "ignoreUnknown": true, + "ignore": [ + "**/.dist/**", + "**/.eggs/**", + "**/.idea/**", + "**/.mypy_cache/**", + "**/.next/**", + "**/.output/**", + "**/.pytest_cache/**", + "**/.ruff_cache/**", + "**/.venv/**", + "**/.wxt/**", + "**/__pycache__/**", + "**/coverage/**", + "**/develop-eggs/**", + "**/dist/**", + "**/manifest.*.json", + "**/node_modules/**", + ".nx/**", + ".trunk/**", + "bazel-*/**", + "node_modules/**", + "third_party/**" + ] + }, + "formatter": { + "useEditorconfig": true + }, + "vcs": { + "clientKind": "git", + "defaultBranch": "main", + "enabled": true, + "useIgnoreFile": true + }, + "javascript": { + "formatter": { + "arrowParentheses": "asNeeded", + "attributePosition": "auto", + "bracketSpacing": true, + "jsxQuoteStyle": "double", + "lineEnding": "lf", + "lineWidth": 80, + "quoteStyle": "single", + "semicolons": "always", + "trailingCommas": "es5" + } + }, + "json": { + "formatter": { + "indentStyle": "space", + "indentWidth": 2, + "lineEnding": "lf", + "lineWidth": 80, + "trailingCommas": "none" + } + } +} diff --git a/py/captainhook.json b/captainhook.json similarity index 81% rename from py/captainhook.json rename to captainhook.json index 964b2ae04..8f460291d 100644 --- a/py/captainhook.json +++ b/captainhook.json @@ -6,14 +6,14 @@ "commit-msg": { "actions": [ { - "run": "convco check -n 1" + "run": ".hooks/conventional-commit-msg {$MESSAGE_FILE}" } ] }, "pre-commit": { "actions": [ { - "run": "py/.hooks/no-commits-on-branches main" + "run": ".hooks/no-commits-on-branches main" }, { "run": "CaptainHook::File.MaxSize", @@ -37,13 +37,13 @@ "run": "py/bin/generate_schema_types" }, { - "run": "py/bin/fmt" + "run": "bin/fmt" }, { - "run": "uvx --directory py ruff check --fix ." + "run": "uv run --directory py ruff check --fix ." }, { - "run": "py/bin/run_tests" + "run": "py/bin/run_python_tests" }, { "run": "uv run --directory py mkdocs build" @@ -77,13 +77,13 @@ "run": "py/bin/generate_schema_types" }, { - "run": "py/bin/fmt" + "run": "bin/fmt" }, { - "run": "uvx --directory py ruff check --fix ." + "run": "uv run --directory py ruff check --fix ." }, { - "run": "py/bin/run_tests" + "run": "py/bin/run_python_tests" }, { "run": "uv run --directory py mkdocs build" @@ -95,7 +95,7 @@ "run": "go test go/..." }, { - "run": "py/.hooks/commit-message-format-pre-push" + "run": ".hooks/commit-message-format-pre-push" } ] } diff --git a/package.json b/package.json index 31b5f16af..fb14278ec 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "setup": "npm-run-all pnpm-install-js pnpm-install-genkit-tools build link-genkit-cli", "format": "(prettier . --write) && (tsx scripts/copyright.ts)", "format:check": "(prettier . --check) && (tsx scripts/copyright.ts --check)", + "format:todo": "pnpm dlx @biomejs/biome format --write .", "build": "pnpm build:js && pnpm build:genkit-tools", "build:js": "cd js && pnpm i && pnpm build", "build:genkit-tools": "cd genkit-tools && pnpm i && pnpm build", diff --git a/py/bin/format_toml_files b/py/bin/format_toml_files deleted file mode 100755 index eef13cf5f..000000000 --- a/py/bin/format_toml_files +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash -# -# Format all TOML files in the project. -# -# Copyright 2025 Google LLC -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -GIT_ROOT=$(git rev-parse --show-toplevel) - -if command -v rust-parallel >/dev/null 2>&1; then - if command -v fd >/dev/null 2>&1; then - fd -e toml \ - --exclude 'py/**/*.egg-info/**' \ - --exclude 'py/**/.dist/**' \ - --exclude 'py/**/.next/**' \ - --exclude 'py/**/.output/**' \ - --exclude 'py/**/.pytest_cache/**' \ - --exclude 'py/**/.venv/**' \ - --exclude 'py/**/__pycache__/**' \ - --exclude 'py/**/build/**' \ - --exclude 'py/**/develop-eggs/**' \ - --exclude 'py/**/dist/**' \ - --exclude 'py/**/eggs/**' \ - --exclude 'py/**/node_modules/**' \ - --exclude 'py/**/sdist/**' \ - --exclude 'py/**/site/**' \ - --exclude 'py/**/target/**' \ - --exclude 'py/**/venv/**' \ - --exclude 'py/**/wheels/**' | - rust-parallel -j4 \ - taplo format --config "${GIT_ROOT}/py/taplo.toml" - else - echo "Using find" - find "${GIT_ROOT}" -name "*.toml" \ - ! -path 'py/**/*.egg-info/**' \ - ! -path 'py/**/.dist/**' \ - ! -path 'py/**/.next/**' \ - ! -path 'py/**/.output/**' \ - ! -path 'py/**/.pytest_cache/**' \ - ! -path 'py/**/.venv/**' \ - ! -path 'py/**/__pycache__/**' \ - ! -path 'py/**/build/**' \ - ! -path 'py/**/develop-eggs/**' \ - ! -path 'py/**/dist/**' \ - ! -path 'py/**/eggs/**' \ - ! -path 'py/**/node_modules/**' \ - ! -path 'py/**/sdist/**' \ - ! -path 'py/**/site/**' \ - ! -path 'py/**/target/**' \ - ! -path 'py/**/venv/**' \ - ! -path 'py/**/wheels/**' \ - -print0 | - rust-parallel -j4 \ - taplo format --config "${GIT_ROOT}/py/taplo.toml" - fi -else - echo "Please install GNU parallel to use this script" -fi diff --git a/py/bin/generate_schema_types b/py/bin/generate_schema_types index 03ff2be78..1e770e8d3 100755 --- a/py/bin/generate_schema_types +++ b/py/bin/generate_schema_types @@ -6,30 +6,30 @@ set -euo pipefail TOP_DIR=$(git rev-parse --show-toplevel) -SCHEMA_FILE="$TOP_DIR/py/packages/genkit/src/genkit/core/schemas.py" +SCHEMA_FILE="${TOP_DIR}/py/packages/genkit/src/genkit/core/schemas.py" # Generate types using configuration from pyproject.toml -uv run --directory "$TOP_DIR/py" datamodel-codegen +uv run --directory "${TOP_DIR}/py" datamodel-codegen # This isn't causing runtime errors at the moment so letting it be. -#sed -i '' '/^class Model(RootModel\[Any\]):$/,/^ root: Any$/d' "$SCHEMA_FILE" +#sed -i '' '/^class Model(RootModel\[Any\]):$/,/^ root: Any$/d' "${SCHEMA_FILE}" # Sanitize the generated schema. -python3 "${TOP_DIR}/py/bin/sanitize_schemas.py" "$SCHEMA_FILE" +python3 "${TOP_DIR}/py/bin/sanitize_schemas.py" "${SCHEMA_FILE}" # Add a generated by `generate_schema_types` comment. sed -i '' '1i\ # DO NOT EDIT: Generated by `generate_schema_types` from `genkit-schemas.json`. -' "$SCHEMA_FILE" +' "${SCHEMA_FILE}" # Add license header. addlicense \ -c "Google LLC" \ -s=only \ - "$SCHEMA_FILE" + "${SCHEMA_FILE}" # Checks and formatting. -uv run --directory "$TOP_DIR/py" \ - ruff check --fix "$SCHEMA_FILE" -uv run --directory "$TOP_DIR/py" \ - ruff format "$SCHEMA_FILE" +uv run --directory "${TOP_DIR}/py" \ + ruff format "${TOP_DIR}" +uv run --directory "${TOP_DIR}/py" \ + ruff check --fix "${SCHEMA_FILE}" diff --git a/py/bin/run_tests b/py/bin/run_python_tests similarity index 100% rename from py/bin/run_tests rename to py/bin/run_python_tests diff --git a/py/bin/sanitize_schemas.py b/py/bin/sanitize_schemas.py index 25e7d39b9..94c7aaa00 100644 --- a/py/bin/sanitize_schemas.py +++ b/py/bin/sanitize_schemas.py @@ -24,8 +24,9 @@ def is_rootmodel_class(self, node: ast.ClassDef) -> bool: return True return False - def visit_ClassDef(self, node: ast.ClassDef) -> ast.ClassDef: - """Visit class definitions and remove model_config if class inherits from RootModel.""" + def visit_ClassDef(self, node: ast.ClassDef) -> ast.ClassDef: # noqa: N802 + """Visit class definitions and remove model_config if class + inherits from RootModel.""" if self.is_rootmodel_class(node): # Filter out model_config assignments new_body = [] diff --git a/py/packages/genkit/src/genkit/core/action.py b/py/packages/genkit/src/genkit/core/action.py index 11ce5c68f..1995452e0 100644 --- a/py/packages/genkit/src/genkit/core/action.py +++ b/py/packages/genkit/src/genkit/core/action.py @@ -1,5 +1,6 @@ # Copyright 2025 Google LLC # SPDX-License-Identifier: Apache-2. + import inspect import json from collections.abc import Callable @@ -13,7 +14,7 @@ class ActionResponse(BaseModel): model_config = ConfigDict(extra='forbid') response: Any - trace_id: str + traceId: str # noqa: N815 class Action: @@ -63,7 +64,7 @@ def fn_to_call(*args, **kwargs): else: span.set_attribute('genkit:output', json.dumps(output)) - return ActionResponse(response=output, trace_id=trace_id) + return ActionResponse(response=output, traceId=trace_id) self.fn = fn_to_call self.description = description diff --git a/py/packages/genkit/src/genkit/core/reflection.py b/py/packages/genkit/src/genkit/core/reflection.py index 68167deaa..b413aefa7 100644 --- a/py/packages/genkit/src/genkit/core/reflection.py +++ b/py/packages/genkit/src/genkit/core/reflection.py @@ -37,8 +37,8 @@ def do_GET(self) -> None: # noqa: N802 actions[key] = { 'key': key, 'name': action.name, - 'inputSchema': action.input_schema, - 'outputSchema': action.output_schema, + 'inputSchema': action.inputSchema, + 'outputSchema': action.outputSchema, 'metadata': action.metadata, } @@ -61,11 +61,11 @@ def do_POST(self) -> None: # noqa: N802 print(payload) action = registry.lookup_by_absolute_name(payload['key']) if '/flow/' in payload['key']: - input_action = action.input_type.validate_python( + input_action = action.inputType.validate_python( payload['input']['start']['input'] ) else: - input_action = action.input_type.validate_python( + input_action = action.inputType.validate_python( payload['input'] ) @@ -82,7 +82,7 @@ def do_POST(self) -> None: # noqa: N802 '{"result": ' + output.response.model_dump_json() + ', "traceId": "' - + output.trace_id + + output.traceId + '"}', self.ENCODING, ) @@ -93,7 +93,7 @@ def do_POST(self) -> None: # noqa: N802 json.dumps( { 'result': output.response, - 'telemetry': {'traceId': output.trace_id}, + 'telemetry': {'traceId': output.traceId}, } ), self.ENCODING, diff --git a/py/packages/genkit/src/genkit/core/tracing.py b/py/packages/genkit/src/genkit/core/tracing.py index 08ef95ad5..7269a1451 100644 --- a/py/packages/genkit/src/genkit/core/tracing.py +++ b/py/packages/genkit/src/genkit/core/tracing.py @@ -8,7 +8,7 @@ import os import sys from collections.abc import Sequence -from typing import Any, cast +from typing import Any import requests # type: ignore[import-untyped] from opentelemetry import trace as trace_api @@ -36,9 +36,7 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: 'traceId': f'{span.context.trace_id}', 'startTime': span.start_time / 1000000, 'endTime': span.end_time / 1000000, - 'attributes': convert_attributes( - attributes=cast(span.attributes, dict), # type: ignore - ), + 'attributes': convert_attributes(span.attributes), 'displayName': span.name, # "links": span.links, 'spanKind': trace_api.SpanKind(span.kind).name, @@ -73,8 +71,8 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: span_data['startTime'] = span.start_time span_data['endTime'] = span.end_time - # TODO: telemetry server URL must be dynamic, - # whatever tools notification says + # TODO: telemetry server URL must be dynamic, whatever tools + # notification says requests.post( 'http://localhost:4033/api/traces', data=json.dumps(span_data), diff --git a/py/pyproject.toml b/py/pyproject.toml index 2e315c54c..fbda6a470 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -21,15 +21,15 @@ version = "0.1.0" [dependency-groups] dev = [ "bpython>=0.25", - "ipython>=8.31.0", + "ipython>=8.32.0", "jupyter>=1.1.1", - "pytest-asyncio>=0.25.2", + "pytest-asyncio>=0.25.3", "pytest>=8.3.4", "pytest-cov>=6.0.0", - "datamodel-code-generator>=0.26.5", + "datamodel-code-generator>=0.27.3", ] -lint = ["mypy~=1.15", "ruff~=0.9"] +lint = ["mypy>=1.15", "ruff>=0.9"] [tool.hatch.build.targets.wheel] packages = [] @@ -97,6 +97,7 @@ exclude = [ ".vscode", "__pypackages__", "_build", + "bazel-*", "buck-out", "build", "dist", @@ -136,7 +137,6 @@ line-ending = "lf" quote-style = "single" skip-magic-trailing-comma = false - # Static type checking. [tool.mypy] disallow_incomplete_defs = true @@ -144,17 +144,21 @@ disallow_untyped_defs = true warn_unused_configs = true [tool.datamodel-codegen] -#strict-types = ["str", "int", "float", "bool", "bytes"] # Don't use; produces StrictStr, StrictInt, etc. #collapse-root-models = true # Don't use; produces Any as types. -disable-timestamp = true -enable-version-header = true -field-constraints = true -input = "../genkit-tools/genkit-schema.json" -input-file-type = "jsonschema" -output = "packages/genkit/src/genkit/core/schemas.py" -output-model-type = "pydantic_v2.BaseModel" +#strict-types = ["str", "int", "float", "bool", "bytes"] # Don't use; produces StrictStr, StrictInt, etc. +#use-subclass-enum = true +capitalize-enum-members = true +disable-timestamp = true +enable-version-header = true +field-constraints = true +input = "../genkit-tools/genkit-schema.json" +input-file-type = "jsonschema" +output = "packages/genkit/src/genkit/core/schemas.py" +output-model-type = "pydantic_v2.BaseModel" +#snake-case-field = true strict-nullable = true target-python-version = "3.12" use-schema-description = true use-standard-collections = true use-union-operator = true +use-unique-items-as-set = true diff --git a/py/uv.lock b/py/uv.lock index 08cc94480..0881701d7 100644 --- a/py/uv.lock +++ b/py/uv.lock @@ -425,7 +425,7 @@ wheels = [ [[package]] name = "datamodel-code-generator" -version = "0.26.5" +version = "0.27.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "argcomplete" }, @@ -435,12 +435,12 @@ dependencies = [ { name = "isort" }, { name = "jinja2" }, { name = "packaging" }, - { name = "pydantic", extra = ["email"], marker = "python_full_version < '4.0'" }, + { name = "pydantic" }, { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/63/e4/53153452235a387112df40f67aaf24072d4b5e33aa7bb385004f4c4baf38/datamodel_code_generator-0.26.5.tar.gz", hash = "sha256:c4a94a7dbf7972129882732d9bcee44c9ae090f57c82edd58d237b9d48c40dd0", size = 92586 } +sdist = { url = "https://files.pythonhosted.org/packages/53/17/53cec24a4edb2021c3b89a3c50c83a9a8dd0e506495d5f7ce486be4a24fa/datamodel_code_generator-0.27.3.tar.gz", hash = "sha256:01e928c00b800aec8d2ee77b5d4b47e1bc159a3a1c32f0f405df0a442d9ab5e7", size = 440612 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/d8/ead3e857d4048947fe92a731d6b1f257dcb267cc8b8918d3b72598c9b728/datamodel_code_generator-0.26.5-py3-none-any.whl", hash = "sha256:e32f986b9914a2b45093947043aa0192d704650be93151f78acf5c95676601ce", size = 114982 }, + { url = "https://files.pythonhosted.org/packages/08/28/a72154c09d09b4c331fee613bb43451a8c43860b52f349777db7935dde04/datamodel_code_generator-0.27.3-py3-none-any.whl", hash = "sha256:ddef49e66e2b90a4c9b238f6ce42dc5a2a23f6ab1b8370eaca08576777921e43", size = 116023 }, ] [[package]] @@ -490,15 +490,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1d/8f/c7f227eb42cfeaddce3eb0c96c60cbca37797fa7b34f8e1aeadf6c5c0983/Deprecated-1.2.15-py2.py3-none-any.whl", hash = "sha256:353bc4a8ac4bfc96800ddab349d89c25dec1079f65fd53acdcc1e0b975b21320", size = 9941 }, ] -[[package]] -name = "dnspython" -version = "2.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b5/4a/263763cb2ba3816dd94b08ad3a33d5fdae34ecb856678773cc40a3605829/dnspython-2.7.0.tar.gz", hash = "sha256:ce9c432eda0dc91cf618a5cedf1a4e142651196bbcd2c80e89ed5a907e5cfaf1", size = 345197 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/68/1b/e0a87d256e40e8c888847551b20a017a6b98139178505dc7ffb96f04e954/dnspython-2.7.0-py3-none-any.whl", hash = "sha256:b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86", size = 313632 }, -] - [[package]] name = "docstring-parser" version = "0.16" @@ -519,19 +510,6 @@ dependencies = [ [package.metadata] requires-dist = [{ name = "handlebarz", editable = "packages/handlebarz" }] -[[package]] -name = "email-validator" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dnspython", marker = "python_full_version < '4.0'" }, - { name = "idna", marker = "python_full_version < '4.0'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/48/ce/13508a1ec3f8bb981ae4ca79ea40384becc868bfae97fd1c942bb3a001b1/email_validator-2.2.0.tar.gz", hash = "sha256:cb690f344c617a714f22e66ae771445a1ceb46821152df8e165c5f9a364582b7", size = 48967 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/ee/bf0adb559ad3c786f12bcbc9296b3f5675f529199bef03e2df281fa1fadb/email_validator-2.2.0-py3-none-any.whl", hash = "sha256:561977c2d73ce3611850a06fa56b414621e0c8faa9d66f2611407d87465da631", size = 33521 }, -] - [[package]] name = "executing" version = "2.1.0" @@ -710,16 +688,16 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "bpython", specifier = ">=0.25" }, - { name = "datamodel-code-generator", specifier = ">=0.26.5" }, - { name = "ipython", specifier = ">=8.31.0" }, + { name = "datamodel-code-generator", specifier = ">=0.27.3" }, + { name = "ipython", specifier = ">=8.32.0" }, { name = "jupyter", specifier = ">=1.1.1" }, { name = "pytest", specifier = ">=8.3.4" }, - { name = "pytest-asyncio", specifier = ">=0.25.2" }, + { name = "pytest-asyncio", specifier = ">=0.25.3" }, { name = "pytest-cov", specifier = ">=6.0.0" }, ] lint = [ - { name = "mypy", specifier = "~=1.15" }, - { name = "ruff", specifier = "~=0.9" }, + { name = "mypy", specifier = ">=1.15" }, + { name = "ruff", specifier = ">=0.9" }, ] [[package]] @@ -1117,7 +1095,7 @@ wheels = [ [[package]] name = "ipython" -version = "8.31.0" +version = "8.32.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -1130,9 +1108,9 @@ dependencies = [ { name = "stack-data" }, { name = "traitlets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/01/35/6f90fdddff7a08b7b715fccbd2427b5212c9525cd043d26fdc45bee0708d/ipython-8.31.0.tar.gz", hash = "sha256:b6a2274606bec6166405ff05e54932ed6e5cfecaca1fc05f2cacde7bb074d70b", size = 5501011 } +sdist = { url = "https://files.pythonhosted.org/packages/36/80/4d2a072e0db7d250f134bc11676517299264ebe16d62a8619d49a78ced73/ipython-8.32.0.tar.gz", hash = "sha256:be2c91895b0b9ea7ba49d33b23e2040c352b33eb6a519cca7ce6e0c743444251", size = 5507441 } wheels = [ - { url = "https://files.pythonhosted.org/packages/04/60/d0feb6b6d9fe4ab89fe8fe5b47cbf6cd936bfd9f1e7ffa9d0015425aeed6/ipython-8.31.0-py3-none-any.whl", hash = "sha256:46ec58f8d3d076a61d128fe517a51eb730e3aaf0c184ea8c17d16e366660c6a6", size = 821583 }, + { url = "https://files.pythonhosted.org/packages/e7/e1/f4474a7ecdb7745a820f6f6039dc43c66add40f1bcc66485607d93571af6/ipython-8.32.0-py3-none-any.whl", hash = "sha256:cae85b0c61eff1fc48b0a8002de5958b6528fa9c8defb1894da63f42613708aa", size = 825524 }, ] [[package]] @@ -1926,11 +1904,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/26/82663c79010b28eddf29dcdd0ea723439535fa917fce5905885c0e9ba562/pydantic-2.10.5-py3-none-any.whl", hash = "sha256:4dd4e322dbe55472cb7ca7e73f4b63574eecccf2835ffa2af9021ce113c83c53", size = 431426 }, ] -[package.optional-dependencies] -email = [ - { name = "email-validator", marker = "python_full_version < '4.0'" }, -] - [[package]] name = "pydantic-core" version = "2.27.2" @@ -1996,14 +1969,14 @@ wheels = [ [[package]] name = "pytest-asyncio" -version = "0.25.2" +version = "0.25.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/72/df/adcc0d60f1053d74717d21d58c0048479e9cab51464ce0d2965b086bd0e2/pytest_asyncio-0.25.2.tar.gz", hash = "sha256:3f8ef9a98f45948ea91a0ed3dc4268b5326c0e7bce73892acc654df4262ad45f", size = 53950 } +sdist = { url = "https://files.pythonhosted.org/packages/f2/a8/ecbc8ede70921dd2f544ab1cadd3ff3bf842af27f87bbdea774c7baa1d38/pytest_asyncio-0.25.3.tar.gz", hash = "sha256:fc1da2cf9f125ada7e710b4ddad05518d4cee187ae9412e9ac9271003497f07a", size = 54239 } wheels = [ - { url = "https://files.pythonhosted.org/packages/61/d8/defa05ae50dcd6019a95527200d3b3980043df5aa445d40cb0ef9f7f98ab/pytest_asyncio-0.25.2-py3-none-any.whl", hash = "sha256:0d0bb693f7b99da304a0634afc0a4b19e49d5e0de2d670f38dc4bfa5727c5075", size = 19400 }, + { url = "https://files.pythonhosted.org/packages/67/17/3493c5624e48fd97156ebaec380dcaafee9506d7e2c46218ceebbb57d7de/pytest_asyncio-0.25.3-py3-none-any.whl", hash = "sha256:9e89518e0f9bd08928f97a3482fdc4e244df17529460bc038291ccaf8f85c7c3", size = 19467 }, ] [[package]] diff --git a/py/taplo.toml b/taplo.toml similarity index 100% rename from py/taplo.toml rename to taplo.toml