diff --git a/.bazelrc b/.bazelrc index d226e65e2..687c13626 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1 +1,7 @@ build --host_platform=@io_tweag_rules_nixpkgs//nixpkgs/platforms:host +build --crosstool_top=@nixpkgs_config_cc//:toolchain +# Using toolchain resolution can lead to spurious dependencies on +# `@local_config_cc//:builtin_include_directory_paths`. This needs to be +# resolved before `--incompatible_enable_cc_toolchain_resolution` can be +# recommended for `nixpkgs_cc_configure_hermetic`. +# build --incompatible_enable_cc_toolchain_resolution diff --git a/CHANGELOG.md b/CHANGELOG.md index a1c6ddab2..d3d6ec9c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,25 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/). ### Changed +- The implementation of `nixpkgs_cc_configure` has been replaced by a more + hermetic version that no longer uses Bazel's builtin autodection toolchain + under the hood. The previous behavior is now available under the name + `nixpkgs_cc_configure_deprecated`, if required. + See [#128][#128]. - The values in the `nixopts` attribute to `nixpkgs_package` are now subject to location expansion. Any instance of `$(location LABEL)` in the `nixopts` attribute will be expanded to the file path of the file referenced by `LABEL`. To pass a plain `$` to Nix it must be escaped as `$$`. See [#132][#132]. +### Deprecated + +- The old implementation of `nixpkgs_cc_configure`, now available under the + name `nixpkgs_cc_configure_deprecated`, has been marked as deprecated in + favor of `nixpkgs_cc_configure` and will be replaced by it in future. + See [#128][#128]. + +[#128]: https://github.com/tweag/rules_nixpkgs/pull/128 [#132]: https://github.com/tweag/rules_nixpkgs/pull/132 ## [0.7.0] - 2020-04-20 diff --git a/README.md b/README.md index de7f269e1..7e1023abf 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Links: * [nixpkgs_git_repository](#nixpkgs_git_repository) * [nixpkgs_package](#nixpkgs_package) * [nixpkgs_cc_configure](#nixpkgs_cc_configure) +* [nixpkgs_cc_configure_deprecated](#nixpkgs_cc_configure_deprecated) * [nixpkgs_go_configure](#nixpkgs_go_configure) ## Setup @@ -328,15 +329,132 @@ filegroup( ### nixpkgs_cc_configure +Use a CC toolchain from Nixpkgs. No-op if not a nix-based platform. + +By default, Bazel auto-configures a CC toolchain from commands (e.g. +`gcc`) available in the environment. To make builds more hermetic, use +this rule to specify explicitly which commands the toolchain should use. + +Specifically, it builds a Nix derivation that provides the CC toolchain +tools in the `bin/` path and constructs a CC toolchain that uses those +tools. Tools that aren't found are replaced by `${coreutils}/bin/false`. +You can inspect the resulting `@_info//:CC_TOOLCHAIN_INFO` to see +which tools were discovered. + +This rule depends on [`rules_cc`](https://github.com/bazelbuild/rules_cc). + +Note: + +You need to configure `--crosstool_top=@//:toolchain` to activate this +toolchain. + +Example: + +```bzl +nixpkgs_cc_configure(repository = "@nixpkgs//:default.nix") +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Attributes
attribute_path +

String; optional

+

Obtain the toolchain from the Nix expression under this attribute path. Requires `nix_file` or `nix_file_content`.

+ 		nix_file +

String; optional

+

Obtain the toolchain from the Nix expression defined in this file. Specify only one of `nix_file` or `nix_file_content`.

+
nix_file_content +

String; optional

+

Obtain the toolchain from the given Nix expression. Specify only one of `nix_file` or `nix_file_content`.

+
nix_file_deps +

List of labels; optional

+

Additional files that the Nix expression depends on.

+
repository +

Label; optional

+

Provides ``. Specify one of `repositories` or `repository`.

+
repositories +

String-keyed label dict; optional

+

Provides `` and other repositories. Specify one of `repositories` or `repository`.

+
nixopts +

String list; optional

+

+ Extra flags to pass when calling Nix. Subject to location + expansion, any instance of $(location LABEL) will be + replaced by the path to the file ferenced by LABEL + relative to the workspace root. +

+
quiet +

Bool; optional

+

Whether to hide `nix-build` output.

+
fail_not_supported +

Bool; optional

+

Whether to fail if `nix-build` is not available.

+
+ +### nixpkgs_cc_configure_deprecated + Tells Bazel to use compilers and linkers from Nixpkgs for the CC toolchain. By default, Bazel autodetects a toolchain on the current `PATH`. Overriding this autodetection makes builds more hermetic and is considered a best practice. +Deprecated: + +Use `nixpkgs_cc_configure` instead. + +While this improves upon Bazel's autoconfigure toolchain by picking tools from +a Nix derivation rather than the environment, it is still not fully hermetic as +it is affected by the environment. In particular, system include directories +specified in the environment can leak in and affect the cache keys of targets +depending on the cc toolchain leading to cache misses. + Example: ```bzl -nixpkgs_cc_configure(repository = "@nixpkgs//:default.nix") +nixpkgs_cc_configure_deprecated(repository = "@nixpkgs//:default.nix") ``` diff --git a/WORKSPACE b/WORKSPACE index 85143ecc4..33cc5b44f 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -161,7 +161,12 @@ nixpkgs_package( repository = "@nixpkgs", ) -nixpkgs_cc_configure(repository = "@remote_nixpkgs") +nixpkgs_cc_configure( + # Use a different name to be able to distinguish this toolchain from the + # builtin one in the tests. + name = "nixpkgs_config_cc", + repository = "@remote_nixpkgs", +) nixpkgs_python_configure( python2_attribute_path = "python2", diff --git a/nixpkgs/BUILD.bazel b/nixpkgs/BUILD.bazel index 18f32ded3..680b55398 100644 --- a/nixpkgs/BUILD.bazel +++ b/nixpkgs/BUILD.bazel @@ -42,6 +42,8 @@ bzl_library( visibility = ["//visibility:public"], deps = [ ":bazel_tools", + "@bazel_skylib//lib:new_sets", "@bazel_skylib//lib:paths", + "@bazel_skylib//lib:sets", ], ) diff --git a/nixpkgs/nixpkgs.bzl b/nixpkgs/nixpkgs.bzl index 40308ed98..666948e3c 100644 --- a/nixpkgs/nixpkgs.bzl +++ b/nixpkgs/nixpkgs.bzl @@ -1,7 +1,15 @@ """Rules for importing Nixpkgs packages.""" +load("@bazel_skylib//lib:sets.bzl", "sets") +load("@bazel_skylib//lib:versions.bzl", "versions") load("@bazel_tools//tools/cpp:cc_configure.bzl", "cc_autoconf_impl") -load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "get_cpu_value") +load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "get_cpu_value", + "get_starlark_list", + "write_builtin_include_directory_paths", +) +load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") load(":private/location_expansion.bzl", "expand_location") def _nixpkgs_git_repository_impl(repository_ctx): @@ -257,6 +265,381 @@ def nixpkgs_package(*args, **kwargs): else: _nixpkgs_package(*args, **kwargs) +def _parse_cc_toolchain_info(content, filename): + """Parses the `CC_TOOLCHAIN_INFO` file generated by Nix. + + Attrs: + content: string, The content of the `CC_TOOLCHAIN_INFO` file. + filename: string, The path to the `CC_TOOLCHAIN_INFO` file, used for error reporting. + + Returns: + struct, The substitutions for `@bazel_tools//tools/cpp:BUILD.tpl`. + """ + + # Parse the content of CC_TOOLCHAIN_INFO. + # + # Each line has the form + # + # :::... + info = {} + for line in content.splitlines(): + fields = line.split(":") + if len(fields) == 0: + fail( + "Malformed CC_TOOLCHAIN_INFO '{}': Empty line encountered.".format(filename), + "cc_toolchain_info", + ) + info[fields[0]] = fields[1:] + + # Validate the keys in CC_TOOLCHAIN_INFO. + expected_keys = sets.make([ + "TOOL_NAMES", + "TOOL_PATHS", + "CXX_BUILTIN_INCLUDE_DIRECTORIES", + "COMPILE_FLAGS", + "CXX_FLAGS", + "LINK_FLAGS", + "LINK_LIBS", + "OPT_COMPILE_FLAGS", + "OPT_LINK_FLAGS", + "UNFILTERED_COMPILE_FLAGS", + "DBG_COMPILE_FLAGS", + "COVERAGE_COMPILE_FLAGS", + "COVERAGE_LINK_FLAGS", + "SUPPORTS_START_END_LIB", + "IS_CLANG", + ]) + actual_keys = sets.make(info.keys()) + missing_keys = sets.difference(expected_keys, actual_keys) + unexpected_keys = sets.difference(actual_keys, expected_keys) + if sets.length(missing_keys) > 0: + fail( + "Malformed CC_TOOLCHAIN_INFO '{}': Missing entries '{}'.".format( + filename, + "', '".join(sets.to_list(missing_keys)), + ), + "cc_toolchain_info", + ) + if sets.length(unexpected_keys) > 0: + fail( + "Malformed CC_TOOLCHAIN_INFO '{}': Unexpected entries '{}'.".format( + filename, + "', '".join(sets.to_list(unexpected_keys)), + ), + "cc_toolchain_info", + ) + + return struct( + tool_paths = { + tool: path + for (tool, path) in zip(info["TOOL_NAMES"], info["TOOL_PATHS"]) + }, + cxx_builtin_include_directories = info["CXX_BUILTIN_INCLUDE_DIRECTORIES"], + compile_flags = info["COMPILE_FLAGS"], + cxx_flags = info["CXX_FLAGS"], + link_flags = info["LINK_FLAGS"], + link_libs = info["LINK_LIBS"], + opt_compile_flags = info["OPT_COMPILE_FLAGS"], + opt_link_flags = info["OPT_LINK_FLAGS"], + unfiltered_compile_flags = info["UNFILTERED_COMPILE_FLAGS"], + dbg_compile_flags = info["DBG_COMPILE_FLAGS"], + coverage_compile_flags = info["COVERAGE_COMPILE_FLAGS"], + coverage_link_flags = info["COVERAGE_LINK_FLAGS"], + supports_start_end_lib = info["SUPPORTS_START_END_LIB"] == ["True"], + is_clang = info["IS_CLANG"] == ["True"], + ) + +def _nixpkgs_cc_toolchain_config_impl(repository_ctx): + cpu_value = get_cpu_value(repository_ctx) + darwin = cpu_value == "darwin" + + cc_toolchain_info_file = repository_ctx.path(repository_ctx.attr.cc_toolchain_info) + if not cc_toolchain_info_file.exists and not repository_ctx.attr.fail_not_supported: + return + info = _parse_cc_toolchain_info( + repository_ctx.read(cc_toolchain_info_file), + cc_toolchain_info_file, + ) + + # Generate the cc_toolchain workspace following the example from + # `@bazel_tools//tools/cpp:unix_cc_configure.bzl`. + # Uses the corresponding templates from `@bazel_tools` as well, see the + # private attributes of the `_nixpkgs_cc_toolchain_config` rule. + repository_ctx.symlink( + repository_ctx.path(repository_ctx.attr._unix_cc_toolchain_config), + "cc_toolchain_config.bzl", + ) + repository_ctx.symlink( + repository_ctx.path(repository_ctx.attr._armeabi_cc_toolchain_config), + "armeabi_cc_toolchain_config.bzl", + ) + + # A module map is required for clang starting from Bazel version 3.3.0. + # https://github.com/bazelbuild/bazel/commit/8b9f74649512ee17ac52815468bf3d7e5e71c9fa + needs_module_map = info.is_clang and versions.is_at_least("3.3.0", versions.get()) + if needs_module_map: + generate_system_module_map = [ + repository_ctx.path(repository_ctx.attr._generate_system_module_map), + ] + repository_ctx.file( + "module.modulemap", + _execute_or_fail( + repository_ctx, + generate_system_module_map + info.cxx_builtin_include_directories, + "Failed to generate system module map.", + ).stdout.strip(), + executable = False, + ) + cc_wrapper_src = ( + repository_ctx.attr._osx_cc_wrapper if darwin else repository_ctx.attr._linux_cc_wrapper + ) + repository_ctx.template( + "cc_wrapper.sh", + repository_ctx.path(cc_wrapper_src), + { + "%{cc}": info.tool_paths["gcc"], + "%{env}": "", + }, + ) + if darwin: + info.tool_paths["gcc"] = "cc_wrapper.sh" + info.tool_paths["ar"] = "/usr/bin/libtool" + write_builtin_include_directory_paths( + repository_ctx, + info.tool_paths["gcc"], + info.cxx_builtin_include_directories, + ) + repository_ctx.template( + "BUILD.bazel", + repository_ctx.path(repository_ctx.attr._build), + { + "%{cc_toolchain_identifier}": "local", + "%{name}": cpu_value, + "%{modulemap}": ("\":module.modulemap\"" if needs_module_map else "None"), + "%{supports_param_files}": "0" if darwin else "1", + "%{cc_compiler_deps}": get_starlark_list( + [":builtin_include_directory_paths"] + ( + [":cc_wrapper"] if darwin else [] + ), + ), + "%{compiler}": "compiler", + "%{abi_version}": "local", + "%{abi_libc_version}": "local", + "%{host_system_name}": "local", + "%{target_libc}": "macosx" if darwin else "local", + "%{target_cpu}": cpu_value, + "%{target_system_name}": "local", + "%{tool_paths}": ",\n ".join( + ['"%s": "%s"' % (k, v) for (k, v) in info.tool_paths.items()], + ), + "%{cxx_builtin_include_directories}": get_starlark_list(info.cxx_builtin_include_directories), + "%{compile_flags}": get_starlark_list(info.compile_flags), + "%{cxx_flags}": get_starlark_list(info.cxx_flags), + "%{link_flags}": get_starlark_list(info.link_flags), + "%{link_libs}": get_starlark_list(info.link_libs), + "%{opt_compile_flags}": get_starlark_list(info.opt_compile_flags), + "%{opt_link_flags}": get_starlark_list(info.opt_link_flags), + "%{unfiltered_compile_flags}": get_starlark_list(info.unfiltered_compile_flags), + "%{dbg_compile_flags}": get_starlark_list(info.dbg_compile_flags), + "%{coverage_compile_flags}": get_starlark_list(info.coverage_compile_flags), + "%{coverage_link_flags}": get_starlark_list(info.coverage_link_flags), + "%{supports_start_end_lib}": repr(info.supports_start_end_lib), + }, + ) + +_nixpkgs_cc_toolchain_config = repository_rule( + _nixpkgs_cc_toolchain_config_impl, + attrs = { + "cc_toolchain_info": attr.label(), + "fail_not_supported": attr.bool(), + "_unix_cc_toolchain_config": attr.label( + default = Label("@bazel_tools//tools/cpp:unix_cc_toolchain_config.bzl"), + ), + "_armeabi_cc_toolchain_config": attr.label( + default = Label("@bazel_tools//tools/cpp:armeabi_cc_toolchain_config.bzl"), + ), + "_generate_system_module_map": attr.label( + default = Label("@bazel_tools//tools/cpp:generate_system_module_map.sh"), + ), + "_osx_cc_wrapper": attr.label( + default = Label("@bazel_tools//tools/cpp:osx_cc_wrapper.sh.tpl"), + ), + "_linux_cc_wrapper": attr.label( + default = Label("@bazel_tools//tools/cpp:linux_cc_wrapper.sh.tpl"), + ), + "_build": attr.label( + default = Label("@bazel_tools//tools/cpp:BUILD.tpl"), + ), + }, +) + +def _nixpkgs_cc_toolchain_impl(repository_ctx): + cpu = get_cpu_value(repository_ctx) + repository_ctx.file( + "BUILD.bazel", + executable = False, + content = """\ +package(default_visibility = ["//visibility:public"]) + +toolchain( + name = "cc-toolchain-{cpu}", + toolchain = "@{cc_toolchain_config}//:cc-compiler-{cpu}", + toolchain_type = "@rules_cc//cc:toolchain_type", + exec_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:{os}", + "@io_tweag_rules_nixpkgs//nixpkgs/constraints:support_nix", + ], + target_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:{os}", + ], +) + +toolchain( + name = "cc-toolchain-armeabi-v7a", + toolchain = "@{cc_toolchain_config}//:cc-compiler-armeabi-v7a", + toolchain_type = "@rules_cc//cc:toolchain_type", + exec_compatible_with = [ + "@platforms//cpu:x86_64", + "@platforms//os:{os}", + "@io_tweag_rules_nixpkgs//nixpkgs/constraints:support_nix", + ], + target_compatible_with = [ + "@platforms//cpu:arm", + "@platforms//os:android", + ], +) +""".format( + cc_toolchain_config = repository_ctx.attr.cc_toolchain_config, + cpu = cpu, + os = "osx" if cpu == "darwin" else "linux", + ), + ) + +_nixpkgs_cc_toolchain = repository_rule( + _nixpkgs_cc_toolchain_impl, + attrs = { + "cc_toolchain_config": attr.string(), + }, +) + +def nixpkgs_cc_configure( + name = "local_config_cc", + attribute_path = "", + nix_file = None, + nix_file_content = "", + nix_file_deps = [], + repositories = {}, + repository = None, + nixopts = [], + quiet = False, + fail_not_supported = True): + """Use a CC toolchain from Nixpkgs. No-op if not a nix-based platform. + + By default, Bazel auto-configures a CC toolchain from commands (e.g. + `gcc`) available in the environment. To make builds more hermetic, use + this rule to specify explicitly which commands the toolchain should use. + + Specifically, it builds a Nix derivation that provides the CC toolchain + tools in the `bin/` path and constructs a CC toolchain that uses those + tools. Tools that aren't found are replaced by `${coreutils}/bin/false`. + You can inspect the resulting `@_info//:CC_TOOLCHAIN_INFO` to see + which tools were discovered. + + This rule depends on [`rules_cc`](https://github.com/bazelbuild/rules_cc). + + Note: + You need to configure `--crosstool_top=@//:toolchain` to activate this + toolchain. + + Attrs: + attribute_path: optional, string, Obtain the toolchain from the Nix expression under this attribute path. Requires `nix_file` or `nix_file_content`. + nix_file: optional, Label, Obtain the toolchain from the Nix expression defined in this file. Specify only one of `nix_file` or `nix_file_content`. + nix_file_content: optional, string, Obtain the toolchain from the given Nix expression. Specify only one of `nix_file` or `nix_file_content`. + nix_file_deps: optional, list of Label, Additional files that the Nix expression depends on. + repositories: dict of Label to string, Provides `` and other repositories. Specify one of `repositories` or `repository`. + repository: Label, Provides ``. Specify one of `repositories` or `repository`. + nixopts: optional, list of string, Extra flags to pass when calling Nix. Subject to location expansion, any instance of `$(location LABEL)` will be replaced by the path to the file ferenced by `LABEL` relative to the workspace root. + quiet: bool, Whether to hide `nix-build` output. + fail_not_supported: bool, Whether to fail if `nix-build` is not available. + """ + + nixopts = list(nixopts) + nix_file_deps = list(nix_file_deps) + + nix_expr = None + if nix_file and nix_file_content: + fail("Cannot specify both 'nix_file' and 'nix_file_content'.") + elif nix_file: + nix_expr = "import $(location {})".format(nix_file) + nix_file_deps.append(nix_file) + elif nix_file_content: + nix_expr = nix_file_content + + if attribute_path and nix_expr == None: + fail("'attribute_path' requires one of 'nix_file' or 'nix_file_content'", "attribute_path") + elif attribute_path: + nixopts.extend([ + "--argstr", + "ccType", + "ccTypeAttribute", + "--argstr", + "ccAttrPath", + attribute_path, + "--arg", + "ccAttrSet", + nix_expr, + ]) + elif nix_expr: + nixopts.extend([ + "--argstr", + "ccType", + "ccTypeExpression", + "--arg", + "ccExpr", + nix_expr, + ]) + else: + nixopts.extend([ + "--argstr", + "ccType", + "ccTypeDefault", + ]) + + # Invoke `toolchains/cc.nix` which generates `CC_TOOLCHAIN_INFO`. + nixpkgs_package( + name = "{}_info".format(name), + nix_file = "@io_tweag_rules_nixpkgs//nixpkgs:toolchains/cc.nix", + nix_file_deps = nix_file_deps, + build_file_content = "exports_files(['CC_TOOLCHAIN_INFO'])", + repositories = repositories, + repository = repository, + nixopts = nixopts, + quiet = quiet, + fail_not_supported = fail_not_supported, + ) + + # Generate the `cc_toolchain_config` workspace. + _nixpkgs_cc_toolchain_config( + name = "{}".format(name), + cc_toolchain_info = "@{}_info//:CC_TOOLCHAIN_INFO".format(name), + fail_not_supported = fail_not_supported, + ) + + # Generate the `cc_toolchain` workspace. + _nixpkgs_cc_toolchain( + name = "{}_toolchains".format(name), + cc_toolchain_config = name, + ) + + maybe( + native.bind, + name = "cc_toolchain", + actual = "@{}//:toolchain".format(name), + ) + native.register_toolchains("@{}_toolchains//:all".format(name)) + def _readlink(repository_ctx, path): return repository_ctx.path(path).realpath @@ -327,7 +710,7 @@ nixpkgs_cc_autoconf = repository_rule( ], ) -def nixpkgs_cc_configure( +def nixpkgs_cc_configure_deprecated( repository = None, repositories = {}, nix_file = None, @@ -336,6 +719,16 @@ def nixpkgs_cc_configure( nixopts = []): """Use a CC toolchain from Nixpkgs. No-op if not a nix-based platform. + Deprecated: + Use `nixpkgs_cc_configure` instead. + + While this improves upon Bazel's autoconfigure toolchain by picking tools + from a Nix derivation rather than the environment, it is still not fully + hermetic as it is affected by the environment. In particular, system + include directories specified in the environment can leak in and affect + the cache keys of targets depending on the cc toolchain leading to cache + misses. + By default, Bazel auto-configures a CC toolchain from commands (e.g. `gcc`) available in the environment. To make builds more hermetic, use this rule to specific explicitly which commands the toolchain should diff --git a/nixpkgs/toolchains/cc.nix b/nixpkgs/toolchains/cc.nix new file mode 100644 index 000000000..a234e3276 --- /dev/null +++ b/nixpkgs/toolchains/cc.nix @@ -0,0 +1,287 @@ +let + pkgs = import { config = {}; overlays = []; }; +in + +{ ccType +, ccAttrPath ? null, ccAttrSet ? null +, ccExpr ? null +}: + +let + darwinCC = + # Work around https://github.com/NixOS/nixpkgs/issues/42059. + # See also https://github.com/NixOS/nixpkgs/pull/41589. + pkgs.runCommand "bazel-nixpkgs-cc-wrapper" + { + buildInputs = [ pkgs.makeWrapper ]; + } + '' + mkdir -p $out/bin + + for i in ${pkgs.stdenv.cc}/bin/*; do + ln -sf $i $out/bin + done + + # Override cc + rm -f $out/bin/cc $out/bin/clang $out/bin/clang++ + makeWrapper ${pkgs.stdenv.cc}/bin/cc $out/bin/cc --add-flags \ + "-Wno-unused-command-line-argument \ + -isystem ${pkgs.llvmPackages.libcxx}/include/c++/v1 \ + -F${pkgs.darwin.apple_sdk.frameworks.CoreFoundation}/Library/Frameworks \ + -F${pkgs.darwin.apple_sdk.frameworks.CoreServices}/Library/Frameworks \ + -F${pkgs.darwin.apple_sdk.frameworks.Security}/Library/Frameworks \ + -F${pkgs.darwin.apple_sdk.frameworks.Foundation}/Library/Frameworks \ + -L${pkgs.libcxx}/lib \ + -L${pkgs.darwin.libobjc}/lib" + ''; + cc = + if ccType == "ccTypeAttribute" then + pkgs.lib.attrByPath (pkgs.lib.splitString "." ccAttrPath) null ccAttrSet + else if ccType == "ccTypeExpression" then + ccExpr + else + pkgs.buildEnv { + name = "bazel-nixpkgs-cc"; + # XXX: `gcov` is missing in `/bin`. + # It exists in `stdenv.cc.cc` but that collides with `stdenv.cc`. + paths = + if pkgs.stdenv.isDarwin then + [ (pkgs.overrideCC pkgs.stdenv darwinCC).cc pkgs.darwin.binutils ] + else + [ pkgs.stdenv.cc pkgs.binutils ]; + pathsToLink = [ "/bin" ]; + } + ; +in + pkgs.runCommand "bazel-nixpkgs-cc-toolchain" + { executable = false; + # Pointless to do this on a remote machine. + preferLocalBuild = true; + allowSubstitutes = false; + } + '' + # This constructs the substitutions for + # `@bazel_tools//tools/cpp:BUILD.tpl` following the example of + # `@bazel_tools//tools/cpp:unix_cc_configure.bzl` as of Bazel v2.1.0 git + # revision 0f4c498a270f05b3896d57055b6489e824821eda. + + # Determine toolchain tool paths. + # + # If a tool is not available then we use `bin/false` as a stand-in. + declare -A TOOLS=( [ar]=ar [cpp]=cpp [dwp]=dwp [gcc]=cc [gcov]=gcov [ld]=ld [nm]=nm [objcopy]=objcopy [objdump]=objdump [strip]=strip ) + TOOL_NAMES=(''${!TOOLS[@]}) + declare -A TOOL_PATHS=() + for tool_name in ''${!TOOLS[@]}; do + tool_path=${cc}/bin/''${TOOLS[$tool_name]} + if [[ -x $tool_path ]]; then + TOOL_PATHS[$tool_name]=$tool_path + else + TOOL_PATHS[$tool_name]=${pkgs.coreutils}/bin/false + fi + done + cc=''${TOOL_PATHS[gcc]} + + # Check whether a flag is supported by the compiler. + # + # The logic checks whether the flag causes an error message that contains + # the flag (or a pattern) verbatim. The assumption is that this will be a + # message of the kind `unknown argument: XYZ`. This logic is copied and + # adapted to bash from `@bazel_tools//tools/cpp:unix_cc_configure.bzl`. + is_compiler_option_supported() { + local option="$1" + local pattern="''${2-$1}" + { $cc "$option" -o /dev/null -c -x c++ - <<<"int main() {}" 2>&1 1>/dev/null || true; } \ + | grep -qe "$pattern" && return 1 || return 0 + } + is_linker_option_supported() { + local option="$1" + local pattern="''${2-$1}" + { $cc "$option" -o /dev/null -x c++ - <<<"int main() {}" 2>&1 1>/dev/null || true; } \ + | grep -qe "$pattern" && return 1 || return 0 + } + add_compiler_option_if_supported() { + if is_compiler_option_supported "$@"; then + echo "$1" + fi + } + add_linker_option_if_supported() { + if is_linker_option_supported "$@"; then + echo "$1" + fi + } + + # Determine default include directories. + # + # This is copied and adapted to bash from + # `@bazel_tools//tools/cpp:unix_cc_configure.bzl`. + IFS=$'\n' + include_dirs_for() { + $cc -E -x "$1" - -v "''${@:2}" 2>&1 \ + | sed -e '1,/^#include <...>/d;/^[^ ]/,$d;s/^ *//' -e 's: (framework directory)::g' \ + | tr '\n' '\0' \ + | xargs -0 realpath -ms + } + CXX_BUILTIN_INCLUDE_DIRECTORIES=($({ + include_dirs_for c + include_dirs_for c++ + if is_compiler_option_supported -fno-canonical-system-headers; then + include_dirs_for c -fno-canonical-system-headers + include_dirs_for c++ -std=c++0x -fno-canonical-system-headers + elif is_compiler_option_supported -no-canonical-prefixes; then + include_dirs_for c -no-canonical-prefixes + include_dirs_for c++ -std=c++0x -no-canonical-prefixes + fi + } 2>&1 | sort -u)) + unset IFS + + # Determine list of supported compiler and linker flags. + # + # This is copied and adapted to bash from + # `@bazel_tools//tools/cpp:unix_cc_configure.bzl`. + COMPILE_FLAGS=( + # Security hardening requires optimization. + # We need to undef it as some distributions now have it enabled by default. + -U_FORTIFY_SOURCE + -fstack-protector + # All warnings are enabled. Maybe enable -Werror as well? + -Wall + $( + # Enable a few more warnings that aren't part of -Wall. + add_compiler_option_if_supported -Wthread-safety + add_compiler_option_if_supported -Wself-assign + # Disable problematic warnings. + add_compiler_option_if_supported -Wunused-but-set-parameter + # has false positives + add_compiler_option_if_supported -Wno-free-nonheap-object + # Enable coloring even if there's no attached terminal. Bazel removes the + # escape sequences if --nocolor is specified. + add_compiler_option_if_supported -fcolor-diagnostics + ) + # Keep stack frames for debugging, even in opt mode. + -fno-omit-frame-pointer + ) + CXX_FLAGS=(-std=c++0x) + LINK_FLAGS=( + $( + if [[ -x ${cc}/bin/ld.gold ]]; then echo -fuse-ld=gold; fi + add_linker_option_if_supported -Wl,-no-as-needed -no-as-needed + add_linker_option_if_supported -Wl,-z,relro,-z,now -z + ) + ${ + if pkgs.stdenv.isDarwin + then "-undefined dynamic_lookup -headerpad_max_install_names" + else "-B${cc}/bin" + } + $( + # Have gcc return the exit code from ld. + add_compiler_option_if_supported -pass-exit-codes + ) + -lstdc++ + -lm + ) + LINK_LIBS=() + OPT_COMPILE_FLAGS=( + # No debug symbols. + # Maybe we should enable https://gcc.gnu.org/wiki/DebugFission for opt or + # even generally? However, that can't happen here, as it requires special + # handling in Bazel. + -g0 + + # Conservative choice for -O + # -O3 can increase binary size and even slow down the resulting binaries. + # Profile first and / or use FDO if you need better performance than this. + -O2 + + # Security hardening on by default. + # Conservative choice; -D_FORTIFY_SOURCE=2 may be unsafe in some cases. + -D_FORTIFY_SOURCE=1 + + # Disable assertions + -DNDEBUG + + # Removal of unused code and data at link time (can this increase binary + # size in some cases?). + -ffunction-sections + -fdata-sections + ) + OPT_LINK_FLAGS=( + ${ + if pkgs.stdenv.isDarwin + then "" + else "$(add_linker_option_if_supported -Wl,--gc-sections -gc-sections)" + } + ) + UNFILTERED_COMPILE_FLAGS=( + $( + if is_compiler_option_supported -fno-canonical-system-headers; then + echo -fno-canonical-system-headers + elif is_compiler_option_supported -no-canonical-prefixes; then + echo -no-canonical-prefixes + fi + ) + # Make C++ compilation deterministic. Use linkstamping instead of these + # compiler symbols. + -Wno-builtin-macro-redefined + -D__DATE__=\\\"redacted\\\" + -D__TIMESTAMP__=\\\"redacted\\\" + -D__TIME__=\\\"redacted\\\" + ) + DBG_COMPILE_FLAGS=(-g) + COVERAGE_COMPILE_FLAGS=( + ${ + if pkgs.stdenv.isDarwin then + "-fprofile-instr-generate -fcoverage-mapping" + else + "--coverage" + } + ) + COVERAGE_LINK_FLAGS=( + ${ + if pkgs.stdenv.isDarwin then + "-fprofile-instr-generate" + else + "--coverage" + } + ) + SUPPORTS_START_END_LIB=( + $( + if [[ -x ${cc}/bin/ld.gold ]]; then echo True; else echo False; fi + ) + ) + IS_CLANG=( + $( + ${cc}/bin/cc -v 2>&1 | grep -q clang && echo True || echo False + ) + ) + + # Write CC_TOOLCHAIN_INFO + # + # Each line has the following shape: + # :::... + # or + # + # I.e. each line is a colon-separated list of the key and the values. + mkdir -p $out + write_info() { + local -n flags=$1 + local output=( "$1" "''${flags[@]}" ) + IFS=: + echo "''${output[*]}" >>$out/CC_TOOLCHAIN_INFO + unset IFS + } + write_info TOOL_NAMES + write_info TOOL_PATHS + write_info CXX_BUILTIN_INCLUDE_DIRECTORIES + write_info COMPILE_FLAGS + write_info CXX_FLAGS + write_info LINK_FLAGS + write_info LINK_LIBS + write_info OPT_COMPILE_FLAGS + write_info OPT_LINK_FLAGS + write_info UNFILTERED_COMPILE_FLAGS + write_info DBG_COMPILE_FLAGS + write_info COVERAGE_COMPILE_FLAGS + write_info COVERAGE_LINK_FLAGS + write_info SUPPORTS_START_END_LIB + write_info IS_CLANG + '' diff --git a/tests/BUILD.bazel b/tests/BUILD.bazel index e782f747e..077c0c845 100644 --- a/tests/BUILD.bazel +++ b/tests/BUILD.bazel @@ -1,6 +1,7 @@ package(default_testonly = 1) load("@io_bazel_rules_go//go:def.bzl", "go_binary") +load(":cc-test.bzl", "cc_toolchain_test") load(":location_expansion_unit_test.bzl", "expand_location_unit_test_suite") expand_location_unit_test_suite() @@ -81,6 +82,11 @@ cc_binary( srcs = ["cc-test.cc"], ) +# Test that nixpkgs_cc_configure is selected. +cc_toolchain_test( + name = "cc-toolchain", +) + # Test nixpkgs_python_configure() by running some Python code. test_suite( name = "python-test", diff --git a/tests/cc-test.bzl b/tests/cc-test.bzl new file mode 100644 index 000000000..8494cc851 --- /dev/null +++ b/tests/cc-test.bzl @@ -0,0 +1,59 @@ +load("@bazel_skylib//lib:paths.bzl", "paths") +load("@bazel_tools//tools/cpp:toolchain_utils.bzl", "find_cpp_toolchain") + +def _cc_toolchain_test_impl(ctx): + cc = find_cpp_toolchain(ctx) + executable = ctx.actions.declare_file(ctx.attr.name + ".sh") + cc_toolchain_info = ctx.file._cc_toolchain_info + cc_toolchain_info_path = ctx.expand_location( + "$(rootpath {})".format(str(ctx.attr._cc_toolchain_info.label)), + [ctx.attr._cc_toolchain_info], + ) + ctx.actions.write(executable, content = """\ +# Find cc in CC_TOOLCHAIN_INFO +while IFS=: read -a line; do + if [[ ${{line[0]}} = TOOL_PATHS ]]; then + for item in ${{line[@]:1}}; do + if [[ $item = */bin/cc ]]; then + CC=$item + fi + done + fi +done <{cc_toolchain_info_path} +if [[ {cc} = */cc_wrapper.sh ]]; then + grep -q "$CC" "{cc}" || {{ + echo "Expected C compiler '$CC' in wrapper script '{cc}'." >&2 + exit 1 + }} +else + if [[ {cc} != $CC ]]; then + echo "Expected C compiler '$CC', but found '{cc}'." >&2 + exit 1 + fi +fi +""".format( + cc = cc.compiler_executable, + cc_toolchain_info_path = cc_toolchain_info_path, + )) + return [DefaultInfo( + executable = executable, + runfiles = ctx.runfiles( + files = [ctx.file._cc_toolchain_info], + transitive_files = cc.all_files, + ), + )] + +cc_toolchain_test = rule( + _cc_toolchain_test_impl, + attrs = { + "_cc_toolchain": attr.label( + default = Label("@rules_cc//cc:current_cc_toolchain"), + ), + "_cc_toolchain_info": attr.label( + allow_single_file = True, + default = Label("@nixpkgs_config_cc_info//:CC_TOOLCHAIN_INFO"), + ), + }, + test = True, + toolchains = ["@rules_cc//cc:toolchain_type"], +)