Merge pull request tweag#97 from tweag/python-toolchain

Adds nixpkgs_python_configure

.bazelrc

@@ -0,0 +1 @@
+build --host_platform=@io_tweag_rules_nixpkgs//nixpkgs/constraints:linux_x86_64_nixpkgs

WORKSPACE

@@ -6,6 +6,7 @@ load(
     "nixpkgs_git_repository",
     "nixpkgs_local_repository",
     "nixpkgs_package",
+    "nixpkgs_python_configure",
 )
 
 # For tests
@@ -38,9 +39,9 @@ nixpkgs_package(
 nixpkgs_package(
     name = "expr-test",
     nix_file_content = "let pkgs = import <nixpkgs> { config = {}; overlays = []; }; in pkgs.hello",
+    nix_file_deps = ["//:nixpkgs.json"],
     # Deliberately not @nixpkgs, to test whether explict file works.
     repositories = {"nixpkgs": "//:nixpkgs.nix"},
-    nix_file_deps = ["//:nixpkgs.json"],
 )
 
 nixpkgs_package(
@@ -81,8 +82,12 @@ nixpkgs_package(
     nix_file_content = """
 { packagePath }: (import <nixpkgs> { config = {}; overlays = []; }).${packagePath}
     """,
+    nixopts = [
+        "--argstr",
+        "packagePath",
+        "hello",
+    ],
     repository = "@nixpkgs",
-    nixopts = ["--argstr", "packagePath", "hello"],
 )
 
 nixpkgs_package(
@@ -111,3 +116,5 @@ nixpkgs_package(
 )
 
 nixpkgs_cc_configure(repository = "@remote_nixpkgs")
+
+nixpkgs_python_configure(repository = "@remote_nixpkgs")

nixpkgs.json

@@ -2,7 +2,6 @@
   "owner": "NixOS",
   "repo": "nixpkgs-channels",
   "branch": "nixpkgs-unstable",
-  "rev": "0620e0fdbf4d79df771afd28f741b2159f381d2b",
-  "sha256": "046l2c83s568c306hnm8nfdpdhmgnbzgid354hr7p0khq3jx3lhf"
+  "rev": "1ee040724a417d9e6e8e28dfd88a0c8c35070689",
+  "sha256": "0r8dv6p1vhxzv50j50bjvwrq5wi9sg35nkm12ga25pw1lzvv6yr9"
 }
-

nixpkgs.nix

@@ -1,5 +1,5 @@
 let
-  # nixpkgs-unstable as of 2019-04-25
+  # nixpkgs-unstable as of 2019-11-17
   spec = builtins.fromJSON (builtins.readFile ./nixpkgs.json);
   nixpkgs = fetchTarball {
     url = "https://github.com/${spec.owner}/${spec.repo}/archive/${spec.rev}.tar.gz";

nixpkgs/constraints/BUILD.bazel

@@ -0,0 +1,25 @@
+constraint_value(
+    name = "nixpkgs",
+    constraint_setting = "@bazel_tools//tools/cpp:cc_compiler",
+    visibility = ["//visibility:public"],
+)
+
+platform(
+    name = "linux_x86_64_nixpkgs",
+    constraint_values = [
+        "@bazel_tools//platforms:x86_64",
+        "@bazel_tools//platforms:linux",
+        ":nixpkgs",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+platform(
+    name = "darwin_x86_64_nixpkgs",
+    constraint_values = [
+        "@bazel_tools//platforms:x86_64",
+        "@bazel_tools//platforms:osx",
+        ":nixpkgs",
+    ],
+    visibility = ["//visibility:public"],
+)

nixpkgs/nixpkgs.bzl

@@ -65,6 +65,7 @@ def _nixpkgs_package_impl(repository_ctx):
 
     # Is nix supported on this platform?
     not_supported = not _is_supported_platform(repository_ctx)
+
     # Should we fail if Nix is not supported?
     fail_not_supported = repository_ctx.attr.fail_not_supported
 
@@ -308,6 +309,128 @@ def nixpkgs_cc_configure(
     native.bind(name = "cc_toolchain", actual = "@local_config_cc//:toolchain")
     native.register_toolchains("@local_config_cc//:all")
 
+def _nixpkgs_python_toolchain_impl(repository_ctx):
+    cpu = get_cpu_value(repository_ctx)
+    repository_ctx.file("BUILD.bazel", executable = False, content = """
+load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")
+py_runtime_pair(
+    name = "py_runtime_pair",
+    py2_runtime = {python2_runtime},
+    py3_runtime = {python3_runtime},
+)
+toolchain(
+    name = "toolchain",
+    toolchain = ":py_runtime_pair",
+    toolchain_type = "@bazel_tools//tools/python:toolchain_type",
+    exec_compatible_with = [
+        "@bazel_tools//platforms:x86_64",
+        "@bazel_tools//platforms:{os}",
+        "@io_tweag_rules_nixpkgs//nixpkgs/constraints:nixpkgs",
+    ],
+    target_compatible_with = [
+        "@bazel_tools//platforms:x86_64",
+        "@bazel_tools//platforms:{os}",
+    ],
+)
+""".format(
+        python2_runtime = _label_string(repository_ctx.attr.python2_runtime),
+        python3_runtime = _label_string(repository_ctx.attr.python3_runtime),
+        os = {"darwin": "osx"}.get(cpu, "linux"),
+    ))
+
+_nixpkgs_python_toolchain = repository_rule(
+    _nixpkgs_python_toolchain_impl,
+    attrs = {
+        # Using attr.string instead of attr.label, so that the repository rule
+        # does not explicitly depend on the nixpkgs_package instances. This is
+        # necessary, so that builds don't fail on platforms without nixpkgs.
+        "python2_runtime": attr.string(),
+        "python3_runtime": attr.string(),
+    },
+)
+
+_python_build_file_content = """
+py_runtime(
+    name = "runtime",
+    files = glob(["**"]),
+    interpreter = "{bin_path}",
+    python_version = "{version}",
+    visibility = ["//visibility:public"],
+)
+"""
+
+def nixpkgs_python_configure(
+        name = "nixpkgs_python_toolchain",
+        python2_attribute_path = None,
+        python2_bin_path = "bin/python",
+        python3_attribute_path = "python3",
+        python3_bin_path = "bin/python",
+        repository = None,
+        repositories = {},
+        nix_file = None,
+        nix_file_content = None,
+        nix_file_deps = None,
+        nixopts = [],
+        fail_not_supported = True):
+    """Define and register a Python toolchain provided by nixpkgs.
+
+    Creates `nixpkgs_package`s for Python 2 or 3 `py_runtime` instances and a
+    corresponding `py_runtime_pair` and `toolchain`. The toolchain is
+    automatically registered and uses the constraint:
+      "@io_tweag_rules_nixpkgs//nixpkgs/constraints:nixpkgs"
+
+    Attrs:
+      name: The name-prefix for the created external repositories.
+      python2_attribute_path: The nixpkgs attribute path for python2.
+      python2_bin_path: The path to the interpreter within the package.
+      python3_attribute_path: The nixpkgs attribute path for python3.
+      python3_bin_path: The path to the interpreter within the package.
+      ...: See `nixpkgs_package` for the remaining attributes.
+    """
+    python2_specified = python2_attribute_path and python2_bin_path
+    python3_specified = python3_attribute_path and python3_bin_path
+    if not python2_specified and not python3_specified:
+        fail("At least one of python2 or python3 has to be specified.")
+    kwargs = dict(
+        repository = repository,
+        repositories = repositories,
+        nix_file = nix_file,
+        nix_file_content = nix_file_content,
+        nix_file_deps = nix_file_deps,
+        nixopts = nixopts,
+        fail_not_supported = fail_not_supported,
+    )
+    python2_runtime = None
+    if python2_attribute_path:
+        python2_runtime = "@%s_python2//:runtime" % name
+        nixpkgs_package(
+            name = name + "_python2",
+            attribute_path = python2_attribute_path,
+            build_file_content = _python_build_file_content.format(
+                bin_path = python2_bin_path,
+                version = "PY2",
+            ),
+            **kwargs
+        )
+    python3_runtime = None
+    if python3_attribute_path:
+        python3_runtime = "@%s_python3//:runtime" % name
+        nixpkgs_package(
+            name = name + "_python3",
+            attribute_path = python3_attribute_path,
+            build_file_content = _python_build_file_content.format(
+                bin_path = python3_bin_path,
+                version = "PY3",
+            ),
+            **kwargs
+        )
+    _nixpkgs_python_toolchain(
+        name = name,
+        python2_runtime = python2_runtime,
+        python3_runtime = python3_runtime,
+    )
+    native.register_toolchains("@%s//:toolchain" % name)
+
 def _execute_or_fail(repository_ctx, arguments, failure_message = "", *args, **kwargs):
     """Call repository_ctx.execute() and fail if non-zero return code."""
     result = repository_ctx.execute(arguments, *args, **kwargs)
@@ -373,3 +496,10 @@ def _cp(repository_ctx, src, dest = None):
         ])
     repository_ctx.template(dest, src, executable = False)
     return repository_ctx.path(dest)
+
+def _label_string(label):
+    """Convert the given (optional) Label to a string."""
+    if not label:
+        return "None"
+    else:
+        return '"%s"' % label

tests/BUILD.bazel

@@ -57,3 +57,11 @@ cc_binary(
     name = "cc-test",
     srcs = ["cc-test.cc"],
 )
+
+# Test nixpkgs_python_configure() by running some Python code.
+py_test(
+    name = "python-test",
+    srcs = ["python-test.py"],
+    python_version = "PY3",
+    srcs_version = "PY3",
+)

tests/python-test.py

@@ -0,0 +1,19 @@
+import os
+import sys
+
+_failure_message = """\
+Python interpreter is not provided by the toolchain.
+Expected: '{expected}'
+Actual:   '{actual}'.
+"""
+
+if __name__ == "__main__":
+    runfiles_dir = os.environ["RUNFILES_DIR"]
+    python_bin = os.path.join(
+        runfiles_dir, "nixpkgs_python_toolchain_python3", "bin", "python")
+    if not sys.executable == python_bin:
+        print(_failure_message.format(
+            expected = python_bin,
+            actual = sys.executable,
+        ), file=sys.stderr)
+        sys.exit(1)