From 272835ae7eebd034f6008ea8eebb54871327f325 Mon Sep 17 00:00:00 2001 From: Jeremy Volkman Date: Mon, 29 Jan 2024 21:55:13 -0800 Subject: [PATCH] Tweaks for BCR (#69) Tweaks build rules and CI workflows to work better with BCR's expectations. --- .github/workflows/ci.yaml | 69 ++++++++++++----------- .github/workflows/release.yml | 18 +++--- BUILD.bazel | 41 ++++++++------ pycross/private/BUILD.bazel | 73 +++++++++++-------------- pycross/private/bzlmod/BUILD.bazel | 12 ++-- pycross/private/lock_attrs.bzl | 5 +- pycross/private/lock_file.bzl | 6 +- pycross/private/pdm.lock | 1 + pycross/private/pyproject.toml | 1 + pycross/private/toolchain_helpers.bzl | 8 +-- pycross/private/{debug.bzl => util.bzl} | 15 ++++- tests/smoke_workspace/.bazelrc | 2 + update_pypi_requirements_bzl.sh | 2 +- 13 files changed, 134 insertions(+), 119 deletions(-) create mode 120000 pycross/private/pdm.lock create mode 120000 pycross/private/pyproject.toml rename pycross/private/{debug.bzl => util.bzl} (66%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 10008f8b..d81bc174 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,40 +13,41 @@ concurrency: cancel-in-progress: true jobs: - main-tests: - # The type of runner that the job will run on + dev-tests: runs-on: ubuntu-latest - # Steps represent a sequence of tasks that will be executed as part of the job steps: - uses: actions/checkout@v2 - - name: .bazelrc.ci - run: | - python -c "import os; file = open('.bazelrc.ci', 'w'); file.write(os.environ['BAZELRC_CI']); file.close()" - shell: bash - env: - BAZELRC_CI: ${{secrets.BAZELRC_CI}} + - uses: DamianReeves/write-file-action@master + with: + path: .bazelrc.ci + contents: ${{secrets.BAZELRC_CI}} + write-mode: overwrite - - name: bazel test //docs/... //pycross/... + - name: Everything except //pycross/... run: | - bazel --bazelrc=.github/workflows/ci.bazelrc --bazelrc=.bazelrc test -- //pycross/... + bazel test -- //... -//pycross/... - smoke-test-default-python-versions: - name: workspace smoke test - Python - default version tests + smoke-test-multi-bazel: + name: Default Python version tests - Bazel ${{matrix.bazel-version}} runs-on: ubuntu-latest strategy: fail-fast: false + matrix: + bazel-version: ["6.x", "7.x"] + + env: + USE_BAZEL_VERSION: ${{matrix.bazel-version}} steps: - uses: actions/checkout@v2 - - name: .bazelrc.ci - run: | - python -c "import os; file = open('.bazelrc.ci', 'w'); file.write(os.environ['BAZELRC_CI']); file.close()" - shell: bash - env: - BAZELRC_CI: ${{secrets.BAZELRC_CI}} + - uses: DamianReeves/write-file-action@master + with: + path: .bazelrc.ci + contents: ${{secrets.BAZELRC_CI}} + write-mode: overwrite - name: run smoke tests - workspace working-directory: tests/smoke_workspace @@ -54,10 +55,15 @@ jobs: bazel test //... - name: run smoke tests - bzlmod - working-directory: tests/smoke_workspace + working-directory: tests/smoke_bzlmod run: | bazel test //... + - name: run pycross tests as consumer + working-directory: tests/smoke_bzlmod + run: | + bazel test @rules_pycross//pycross/... + workspace-smoke-test: name: workspace smoke test - Python ${{ matrix.python-version }} on ${{ startsWith(matrix.os, 'macos-') && 'macOS' || startsWith(matrix.os, 'windows-') && 'Windows' || 'Linux' }} runs-on: ubuntu-latest @@ -67,6 +73,7 @@ jobs: os: [ubuntu-latest, macos-latest] # This needs to stay in sync with tests/smoke_bzlmod/MODULE.bazel python-version: ["3.10.12", "3.11.6", "3.12.0"] + defaults: run: working-directory: tests/smoke_workspace @@ -74,12 +81,11 @@ jobs: steps: - uses: actions/checkout@v2 - - name: .bazelrc.ci - run: | - python -c "import os; file = open('.bazelrc.ci', 'w'); file.write(os.environ['BAZELRC_CI']); file.close()" - shell: bash - env: - BAZELRC_CI: ${{secrets.BAZELRC_CI}} + - uses: DamianReeves/write-file-action@master + with: + path: .bazelrc.ci + contents: ${{secrets.BAZELRC_CI}} + write-mode: overwrite - name: run smoke tests run: | @@ -101,12 +107,11 @@ jobs: steps: - uses: actions/checkout@v2 - - name: .bazelrc.ci - run: | - python -c "import os; file = open('.bazelrc.ci', 'w'); file.write(os.environ['BAZELRC_CI']); file.close()" - shell: bash - env: - BAZELRC_CI: ${{secrets.BAZELRC_CI}} + - uses: DamianReeves/write-file-action@master + with: + path: .bazelrc.ci + contents: ${{secrets.BAZELRC_CI}} + write-mode: overwrite - name: run smoke tests run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cab34de3..541e5a11 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,18 +12,20 @@ jobs: build: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v2 - - name: .bazelrc.ci - run: | - python -c "import os; file = open('.bazelrc.ci', 'w'); file.write(os.environ['BAZELRC_CI']); file.close()" - shell: bash - env: - BAZELRC_CI: ${{secrets.BAZELRC_CI}} + - uses: actions/checkout@v2 + + - uses: DamianReeves/write-file-action@master + with: + path: .bazelrc.ci + contents: ${{secrets.BAZELRC_CI}} + write-mode: overwrite + - name: bazel test //... run: bazel test -- //... + - name: Prepare release run: .github/workflows/release_prep.sh ${{ env.GITHUB_REF_NAME }} > release_notes.txt + - name: Release uses: softprops/action-gh-release@v1 with: diff --git a/BUILD.bazel b/BUILD.bazel index d59fe631..9e541149 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,20 +1,21 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -#load("@gazelle//:def.bzl", "gazelle", "gazelle_binary") +load("@bazel_skylib//rules:diff_test.bzl", "diff_test") +load("@gazelle//:def.bzl", "gazelle", "gazelle_binary") # gazelle:exclude tests # gazelle:exclude examples -#gazelle_binary( -# name = "gazelle_bin", -# languages = [ -# "@bazel_skylib_gazelle_plugin//bzl", -# ], -#) +gazelle_binary( + name = "gazelle_bin", + languages = [ + "@bazel_skylib_gazelle_plugin//bzl", + ], +) -#gazelle( -# name = "gazelle", -# gazelle = "gazelle_bin", -#) +gazelle( + name = "gazelle", + gazelle = "gazelle_bin", +) bzl_library( name = "internal_deps", @@ -26,10 +27,16 @@ bzl_library( ], ) -exports_files( - [ - "pdm.lock", - "pyproject.toml", - ], - visibility = ["//pycross:__subpackages__"], +diff_test( + name = "test_core_deps_diff", + failure_message = "Please run `bazel run //pycross/private:update_pycross_deps`", + file1 = "//pycross/private:updated-pycross_deps_core.lock.bzl", + file2 = "//pycross/private:pycross_deps_core.lock.bzl", +) + +diff_test( + name = "test_deps_diff", + failure_message = "Please run `bazel run //pycross/private:update_pycross_deps`", + file1 = "//pycross/private:updated-pycross_deps.lock.bzl", + file2 = "//pycross/private:pycross_deps.lock.bzl", ) diff --git a/pycross/private/BUILD.bazel b/pycross/private/BUILD.bazel index 54a793a8..7a69e426 100644 --- a/pycross/private/BUILD.bazel +++ b/pycross/private/BUILD.bazel @@ -4,12 +4,11 @@ load(":lock_file.bzl", "pycross_lock_file") load(":pdm_lock_model.bzl", "pycross_pdm_lock_model") load(":target_environment.bzl", "pycross_target_environment") -default_visibility = ["//pycross:__subpackages__"] +package(default_visibility = ["//pycross:__subpackages__"]) bzl_library( name = "cc_toolchain_util", srcs = ["cc_toolchain_util.bzl"], - visibility = default_visibility, deps = [ "@bazel_skylib//lib:collections", "@bazel_tools//tools/build_defs/cc:action_names.bzl", @@ -20,7 +19,6 @@ bzl_library( bzl_library( name = "internal_repo", srcs = ["internal_repo.bzl"], - visibility = default_visibility, deps = [ ":lock_attrs", ":repo_venv_utils", @@ -31,25 +29,23 @@ bzl_library( bzl_library( name = "lock_file", srcs = ["lock_file.bzl"], - visibility = default_visibility, deps = [ ":lock_attrs", ":providers", ":target_environment", + ":util", ], ) bzl_library( name = "pdm_lock_model", srcs = ["pdm_lock_model.bzl"], - visibility = default_visibility, deps = [":internal_repo"], ) bzl_library( name = "lock_repo", srcs = ["lock_repo.bzl"], - visibility = default_visibility, deps = [ ":lock_attrs", ":package_repo", @@ -60,13 +56,11 @@ bzl_library( bzl_library( name = "poetry_lock_model", srcs = ["poetry_lock_model.bzl"], - visibility = default_visibility, deps = [":internal_repo"], ) bzl_library( name = "pycross_deps_core_lock", - visibility = default_visibility, deps = [ "//pycross:defs", "@bazel_tools//tools/build_defs/repo:http.bzl", @@ -76,7 +70,6 @@ bzl_library( bzl_library( name = "pycross_deps_lock", - visibility = default_visibility, deps = [ "//pycross:defs", "@bazel_tools//tools/build_defs/repo:http.bzl", @@ -87,14 +80,12 @@ bzl_library( bzl_library( name = "pypi_file", srcs = ["pypi_file.bzl"], - visibility = default_visibility, deps = ["@bazel_tools//tools/build_defs/repo:utils.bzl"], ) bzl_library( name = "package_repo", srcs = ["package_repo.bzl"], - visibility = default_visibility, deps = [ ":internal_repo", "@bazel_skylib//lib:paths", @@ -104,7 +95,6 @@ bzl_library( bzl_library( name = "repo_venv_utils", srcs = ["repo_venv_utils.bzl"], - visibility = default_visibility, deps = [ "@bazel_skylib//lib:paths", ], @@ -113,7 +103,6 @@ bzl_library( bzl_library( name = "resolved_lock_repo", srcs = ["resolved_lock_repo.bzl"], - visibility = default_visibility, deps = [ ":internal_repo", ":lock_attrs", @@ -125,7 +114,6 @@ bzl_library( bzl_library( name = "target_environment", srcs = ["target_environment.bzl"], - visibility = default_visibility, deps = [ ":internal_repo", ], @@ -134,10 +122,10 @@ bzl_library( bzl_library( name = "toolchain_helpers", srcs = ["toolchain_helpers.bzl"], - visibility = default_visibility, deps = [ ":lock_attrs", ":target_environment", + ":util", "@rules_python//python:versions.bzl", ], ) @@ -145,7 +133,6 @@ bzl_library( bzl_library( name = "wheel_build", srcs = ["wheel_build.bzl"], - visibility = default_visibility, deps = [ ":cc_toolchain_util", ":providers", @@ -158,7 +145,6 @@ bzl_library( bzl_library( name = "wheel_library", srcs = ["wheel_library.bzl"], - visibility = default_visibility, deps = [ ":providers", "@bazel_skylib//lib:paths", @@ -169,7 +155,6 @@ bzl_library( bzl_library( name = "wheel_zipimport_library", srcs = ["wheel_zipimport_library.bzl"], - visibility = default_visibility, deps = [ "@bazel_skylib//lib:paths", "@rules_python//python:py_info_bzl", @@ -185,11 +170,22 @@ pycross_target_environment( visibility = ["//visibility:public"], ) +# NB: pdm.lock and pyproject.toml are symlinked into this directory from +# the root. This is so that this package does not need to depend on the +# root package, since the root utilizes dev dependencies which are not +# always present. pycross_pdm_lock_model( name = "rules_pycross_all_deps_lock_model", all_optional_groups = True, - lock_file = "//:pdm.lock", - project_file = "//:pyproject.toml", + lock_file = ":pdm.lock", + project_file = ":pyproject.toml", +) + +pycross_pdm_lock_model( + name = "rules_pycross_core_deps_lock_model", + lock_file = ":pdm.lock", + optional_groups = ["core"], + project_file = ":pyproject.toml", ) pycross_lock_file( @@ -200,13 +196,7 @@ pycross_lock_file( lock_model_file = ":rules_pycross_all_deps_lock_model", repo_prefix = "rules_pycross_internal_deps", target_environments = [":rules_pycross_deps_target_env"], -) - -pycross_pdm_lock_model( - name = "rules_pycross_core_deps_lock_model", - lock_file = "//:pdm.lock", - optional_groups = ["core"], - project_file = "//:pyproject.toml", + visibility = ["//:__subpackages__"], ) pycross_lock_file( @@ -218,10 +208,12 @@ pycross_lock_file( lock_model_file = ":rules_pycross_core_deps_lock_model", repo_prefix = "rules_pycross_internal_deps", target_environments = [":rules_pycross_deps_target_env"], + visibility = ["//:__subpackages__"], ) write_source_files( name = "update_pycross_deps", + diff_test = False, # Diff tests for these files are in the workspace root. files = { "pycross_deps.lock.bzl": ":updated-pycross_deps.lock.bzl", "pycross_deps_core.lock.bzl": ":updated-pycross_deps_core.lock.bzl", @@ -231,7 +223,6 @@ write_source_files( bzl_library( name = "pycross_deps.lock", srcs = ["pycross_deps.lock.bzl"], - visibility = ["//pycross:__subpackages__"], deps = [ "//pycross:defs", "@bazel_tools//tools/build_defs/repo:http.bzl", @@ -242,7 +233,6 @@ bzl_library( bzl_library( name = "pycross_deps_core.lock", srcs = ["pycross_deps_core.lock.bzl"], - visibility = ["//pycross:__subpackages__"], deps = [ "//pycross:defs", "@bazel_tools//tools/build_defs/repo:http.bzl", @@ -250,38 +240,41 @@ bzl_library( ], ) -bzl_library( - name = "debug", - srcs = ["debug.bzl"], - visibility = ["//pycross:__subpackages__"], -) - bzl_library( name = "lock_attrs", srcs = ["lock_attrs.bzl"], - visibility = ["//pycross:__subpackages__"], + deps = ["util"], ) bzl_library( name = "lock_file_repo", srcs = ["lock_file_repo.bzl"], - visibility = ["//pycross:__subpackages__"], ) bzl_library( name = "providers", srcs = ["providers.bzl"], - visibility = ["//pycross:__subpackages__"], ) bzl_library( name = "updated-pycross_deps.lock", srcs = ["updated-pycross_deps.lock.bzl"], - visibility = ["//pycross:__subpackages__"], ) bzl_library( name = "updated-pycross_deps_core.lock", srcs = ["updated-pycross_deps_core.lock.bzl"], - visibility = ["//pycross:__subpackages__"], +) + +bzl_library( + name = "util", + srcs = ["util.bzl"], +) + +exports_files( + [ + "pycross_deps_core.lock.bzl", + "pycross_deps.lock.bzl", + ], + visibility = ["//:__subpackages__"], ) diff --git a/pycross/private/bzlmod/BUILD.bazel b/pycross/private/bzlmod/BUILD.bazel index fa2b0c1e..faa679d0 100644 --- a/pycross/private/bzlmod/BUILD.bazel +++ b/pycross/private/bzlmod/BUILD.bazel @@ -1,4 +1,5 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("//pycross/private:util.bzl", "REPO_HTTP_DEPS") package(default_visibility = ["//pycross:__subpackages__"]) @@ -34,8 +35,7 @@ bzl_library( "//pycross/private:poetry_lock_model", "//pycross/private:pypi_file", "//pycross/private:resolved_lock_repo", - "@bazel_tools//tools/build_defs/repo:http.bzl", - ], + ] + REPO_HTTP_DEPS, ) bzl_library( @@ -61,10 +61,8 @@ bzl_library( ":tag_attrs", "//pycross/private:package_repo", "//pycross/private:pypi_file", - "@bazel_tools//tools/build_defs/repo:cache.bzl", - "@bazel_tools//tools/build_defs/repo:http.bzl", "@lock_import_repos_hub//:locks.bzl", - ], + ] + REPO_HTTP_DEPS, ) bzl_library( @@ -74,9 +72,7 @@ bzl_library( "//pycross/private:internal_repo", "//pycross/private:lock_file_repo", "//pycross/private:pypi_file", - "@bazel_tools//tools/build_defs/repo:cache.bzl", - "@bazel_tools//tools/build_defs/repo:http.bzl", - ], + ] + REPO_HTTP_DEPS, ) bzl_library( diff --git a/pycross/private/lock_attrs.bzl b/pycross/private/lock_attrs.bzl index 45667bb2..647d896b 100644 --- a/pycross/private/lock_attrs.bzl +++ b/pycross/private/lock_attrs.bzl @@ -1,7 +1,6 @@ """Common attr handling for things that generate lock files.""" -# Whether bzlmod is enabled. -_BZLMOD = str(Label("//:invalid")).startswith("@@") +load(":util.bzl", "BZLMOD") DEFAULT_MACOS_VERSION = "12.0" DEFAULT_GLIBC_VERSION = "2.25" @@ -211,7 +210,7 @@ def handle_render_attrs(attrs): # generated .bzl file. We can figure that out by comparing our workspace against the root workspace. if Label("@@//:invalid").workspace_name == Label("//:invalid").workspace_name: pycross_repo_name = "" - elif _BZLMOD: + elif BZLMOD: pycross_repo_name = "@@" + Label("//:invalid").workspace_name else: pycross_repo_name = "@" + Label("//:invalid").workspace_name diff --git a/pycross/private/lock_file.bzl b/pycross/private/lock_file.bzl index 562e70f4..f0990098 100644 --- a/pycross/private/lock_file.bzl +++ b/pycross/private/lock_file.bzl @@ -1,12 +1,10 @@ """Implementation of the pycross_lock_file rule.""" load(":lock_attrs.bzl", "RENDER_ATTRS", "RESOLVE_ATTRS", "handle_render_attrs", "handle_resolve_attrs") - -# Whether bzlmod is enabled. -_BZLMOD = str(Label("//:invalid")).startswith("@@") +load(":util.bzl", "BZLMOD") def fully_qualified_label(ctx, label): - prefix = "@@" if _BZLMOD else "@" + prefix = "@@" if BZLMOD else "@" return "%s%s//%s:%s" % (prefix, label.workspace_name or ctx.workspace_name, label.package, label.name) def _pycross_lock_file_impl(ctx): diff --git a/pycross/private/pdm.lock b/pycross/private/pdm.lock new file mode 120000 index 00000000..c30b80d0 --- /dev/null +++ b/pycross/private/pdm.lock @@ -0,0 +1 @@ +../../pdm.lock \ No newline at end of file diff --git a/pycross/private/pyproject.toml b/pycross/private/pyproject.toml new file mode 120000 index 00000000..00c904eb --- /dev/null +++ b/pycross/private/pyproject.toml @@ -0,0 +1 @@ +../../pyproject.toml \ No newline at end of file diff --git a/pycross/private/toolchain_helpers.bzl b/pycross/private/toolchain_helpers.bzl index a50b50c4..f19419c0 100644 --- a/pycross/private/toolchain_helpers.bzl +++ b/pycross/private/toolchain_helpers.bzl @@ -3,12 +3,10 @@ load("@rules_python//python:versions.bzl", "MINOR_MAPPING", "PLATFORMS", "TOOL_VERSIONS") load(":lock_attrs.bzl", "DEFAULT_GLIBC_VERSION", "DEFAULT_MACOS_VERSION") load(":target_environment.bzl", "repo_batch_create_target_environments") - -# Whether bzlmod is enabled. -_BZLMOD = str(Label("//:invalid")).startswith("@@") +load(":util.bzl", "BZLMOD") def _repo_label(repo_name, label): - if _BZLMOD: + if BZLMOD: return "@@{}{}".format(repo_name, label) else: return "@{}{}".format(repo_name, label) @@ -169,7 +167,7 @@ def _compute_toolchains( # These conditionals create a `interpreter_repo_pattern` which accepts a # platform name (e.g., x86_64-unknown-linux-gnu). - if _BZLMOD: + if BZLMOD: # With bzlmod we need to construct the canonical repository names for platform-specific interpreters. interpreter_repo_pattern = "@@{}python_{}_{{plat}}//:py3_runtime".format( _canonical_prefix(python_toolchains_repo_name), diff --git a/pycross/private/debug.bzl b/pycross/private/util.bzl similarity index 66% rename from pycross/private/debug.bzl rename to pycross/private/util.bzl index f9e7a169..5bb400ae 100644 --- a/pycross/private/debug.bzl +++ b/pycross/private/util.bzl @@ -1,4 +1,17 @@ -"""Debug utilities""" +"""Shared utilities""" + +# Whether we're using at least Bazel 7 +IS_BAZEL_7_OR_HIGHER = hasattr(native, "starlark_doc_extract") + +# Whether we're using bzlmod +BZLMOD = str(Label("//:invalid")).startswith("@@") + +# The http library seems to depend on cache.bzl as of Bazel 7 +REPO_HTTP_DEPS = [ + "@bazel_tools//tools/build_defs/repo:http.bzl", +] + [ + "@bazel_tools//tools/build_defs/repo:cache.bzl", +] if IS_BAZEL_7_OR_HIGHER else [] def trace_ctx(ctx, display_name = "ctx"): """Wraps a context object so that method calls are printed with their arguments. diff --git a/tests/smoke_workspace/.bazelrc b/tests/smoke_workspace/.bazelrc index 08c6f7e4..06083136 100644 --- a/tests/smoke_workspace/.bazelrc +++ b/tests/smoke_workspace/.bazelrc @@ -1,3 +1,5 @@ +common --noenable_bzlmod + build --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 build --incompatible_enable_cc_toolchain_resolution build --incompatible_strict_action_env diff --git a/update_pypi_requirements_bzl.sh b/update_pypi_requirements_bzl.sh index e2fe053a..b5472b5c 100755 --- a/update_pypi_requirements_bzl.sh +++ b/update_pypi_requirements_bzl.sh @@ -2,4 +2,4 @@ set -e pdm lock -bazel run //pycross/private:update_pycross_deps +bazel run :update_pycross_deps