Merge branch 'master' of https://github.com/ray-project/ray

Signed-off-by: Balaji Veeramani <[email protected]>

.bazelrc

@@ -150,30 +150,32 @@ test:asan-buildkite --test_env=LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libasan.so.
 # CI configuration:
 aquery:ci --color=no
 aquery:ci --noshow_progress
+
+test:ci-base --test_output=errors
+test:ci-base --test_verbose_timeout_warnings
+test:ci-base --flaky_test_attempts=3
+
 build:ci --color=yes
 build:ci --curses=no
 build:ci --keep_going
 build:ci --progress_report_interval=100
 build:ci --show_progress_rate_limit=15
 build:ci --ui_actions_shown=1024
 build:ci --show_timestamps
-test:ci --flaky_test_attempts=3
 # Disable test result caching because py_test under Bazel can import from outside of sandbox, but Bazel only looks at
 # declared dependencies to determine if a result should be cached. More details at:
 # https://github.com/bazelbuild/bazel/issues/7091, https://github.com/bazelbuild/rules_python/issues/382
+test:ci --config=ci-base
 test:ci --nocache_test_results
 test:ci --spawn_strategy=local
-test:ci --test_output=errors
 test:ci --experimental_ui_max_stdouterr_bytes=-1
-test:ci --test_verbose_timeout_warnings
+
+test:ci-debug --config=ci-base
 test:ci-debug -c dbg
 test:ci-debug --copt="-g"
-test:ci-debug --flaky_test_attempts=3
 # Disable test result caching for the same reason above.
 test:ci-debug --nocache_test_results
 test:ci-debug --spawn_strategy=local
-test:ci-debug --test_output=errors
-test:ci-debug --test_verbose_timeout_warnings
 
 aquery:get-toolchain --include_commandline=false
 aquery:get-toolchain --noimplicit_deps

.buildkite/cicd.rayci.yml

@@ -3,9 +3,10 @@ steps:
   - label: ":coral: reef: ci+release tooling tests"
     key: reef-tests
     commands:
-      - bazel run //ci/ray_ci:test_in_docker -- //ci/ray_ci/... //release/... ci
+      - bazel run //ci/ray_ci:test_in_docker --
+          //ci/ray_ci/... //release/... //ci/pipeline/... ci
           --only-tags=release_unit,ci_unit
-          --parallelism-per-worker 2
+          --cache-test-results --parallelism-per-worker 2
           --build-name oss-ci-base_test
           --build-type skip
     instance_type: small

.buildkite/core.rayci.yml

@@ -275,30 +275,30 @@ steps:
     instance_type: medium
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --build-type clang
-        --parallelism-per-worker 2
+        --cache-test-results --parallelism-per-worker 2
 
   - label: ":ray: core: cpp asan tests"
     tags: core_cpp
     instance_type: medium
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --build-type asan-clang
-        --parallelism-per-worker 2
+        --cache-test-results --parallelism-per-worker 2
 
   - label: ":ray: core: cpp ubsan tests"
     tags: core_cpp
     instance_type: large
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --build-type ubsan
         --except-tags no_ubsan
-        --parallelism-per-worker 2
+        --cache-test-results --parallelism-per-worker 2
 
   - label: ":ray: core: cpp tsan tests"
     tags: core_cpp
     instance_type: medium
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //:all //src/... core --build-type tsan-clang
         --except-tags no_tsan
-        --parallelism-per-worker 2
+        --cache-test-results --parallelism-per-worker 2
 
   - label: ":ray: core: flaky tests"
     key: core_flaky_tests

.buildkite/hooks/post-command

@@ -16,6 +16,7 @@ if [ -d "/tmp/artifacts/test-summaries" ] && [ "$(ls -A /tmp/artifacts/test-summ
     # Only upload annotations if there are at least 2 files in the directory:
     # 1 header and 1 failed test.
     if [ "$(find /tmp/artifacts/test-summaries -maxdepth 1 -name '*.txt' | wc -l)" -ge 2 ]; then
+      echo "Test summaries for ${BUILDKITE_JOB_ID} ${BUILDKITE_LABEL}" | buildkite-agent annotate --job "${BUILDKITE_JOB_ID}" --style error --context "${BUILDKITE_JOB_ID}"
       cat /tmp/artifacts/test-summaries/*.txt | head -n 20 | buildkite-agent annotate --job "${BUILDKITE_JOB_ID}" --append --style error --context "${BUILDKITE_JOB_ID}"
     fi
 

BUILD.bazel

@@ -118,18 +118,16 @@ ray_cc_library(
         "src/ray/rpc/rpc_chaos.cc",
         "src/ray/rpc/server_call.cc",
     ],
-    hdrs = glob([
-        "src/ray/rpc/rpc_chaos.h",
+    hdrs = [
         "src/ray/rpc/client_call.h",
         "src/ray/rpc/common.h",
         "src/ray/rpc/grpc_client.h",
-        "src/ray/rpc/retryable_grpc_client.h",
         "src/ray/rpc/grpc_server.h",
         "src/ray/rpc/metrics_agent_client.h",
+        "src/ray/rpc/retryable_grpc_client.h",
+        "src/ray/rpc/rpc_chaos.h",
         "src/ray/rpc/server_call.h",
-        "src/ray/rpc/grpc_util.h",
-        "src/ray/raylet_client/*.h",
-    ]),
+    ] + glob(["src/ray/raylet_client/*.h"]),
     deps = [
         ":stats_metric",
         "//src/ray/common:asio",
@@ -487,6 +485,7 @@ ray_cc_binary(
     deps = [
         ":raylet_lib",
         "//src/ray/util",
+        "//src/ray/util:cmd_line_utils",
         "@com_github_gflags_gflags//:gflags",
     ],
 )
@@ -722,6 +721,8 @@ ray_cc_library(
         "//src/ray/protobuf:common_cc_proto",
         "//src/ray/protobuf:runtime_env_agent_cc_proto",
         "//src/ray/util",
+        "//src/ray/util:cmd_line_utils",
+        "//src/ray/util:container_util",
         "//src/ray/util:throttler",
         "@boost//:asio",
         "@boost//:beast",
@@ -792,6 +793,7 @@ ray_cc_library(
         ":worker_rpc",
         "//src/ray/protobuf:worker_cc_proto",
         "//src/ray/util",
+        "//src/ray/util:container_util",
         "//src/ray/util:shared_lru",
         "@boost//:circular_buffer",
         "@boost//:fiber",
@@ -1977,6 +1979,7 @@ ray_cc_library(
         ":pubsub_lib",
         ":ray_common",
         ":redis_store_client",
+        "//src/ray/util:container_util",
         "//src/ray/util:sequencer",
         "//src/ray/protobuf:usage_cc_proto",
     ],
@@ -2232,6 +2235,7 @@ ray_cc_library(
     ],
     deps = [
         ":redis_client",
+        "//src/ray/util:container_util",
         "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/synchronization",
     ],

ci/pipeline/BUILD.bazel

@@ -0,0 +1,20 @@
+load("@rules_python//python:defs.bzl", "py_test")
+load("@py_deps_buildkite//:requirements.bzl", ci_require = "requirement")
+
+py_test(
+    name = "test_conditional_testing",
+    size = "small",
+    srcs = ["test_conditional_testing.py"],
+    data = [
+        ":determine_tests_to_run.py",
+    ],
+    exec_compatible_with = ["//:hermetic_python"],
+    tags = [
+        "ci_unit",
+        "team:ci",
+    ],
+    deps = [
+        ci_require("bazel-runfiles"),
+        ci_require("pytest"),
+    ],
+)

ci/pipeline/determine_tests_to_run.py

@@ -1,5 +1,4 @@
-# Script used for checking changes for incremental testing cases
-from __future__ import absolute_import, division, print_function
+#!/usr/bin/env python3
 
 import argparse
 import os

ci/pipeline/test_conditional_testing.py

@@ -0,0 +1,84 @@
+import os
+import subprocess
+import sys
+import tempfile
+
+import runfiles
+import pytest
+
+_REPO_NAME = "com_github_ray_project_ray"
+_runfiles = runfiles.Create()
+
+
+def test_conditional_testing_pull_request():
+    script = _runfiles.Rlocation(_REPO_NAME + "/ci/pipeline/determine_tests_to_run.py")
+
+    test_cases = [
+        (["ci/pipeline/test_conditional_testing.py"], {"lint", "tools"}),
+        (
+            ["python/ray/data/__init__.py"],
+            {"lint", "data", "linux_wheels", "macos_wheels", "ml", "train"},
+        ),
+    ]
+
+    with tempfile.TemporaryDirectory() as origin, tempfile.TemporaryDirectory() as workdir:
+        subprocess.check_call(["git", "init", "--bare"], cwd=origin)
+        subprocess.check_call(["git", "init"], cwd=workdir)
+        subprocess.check_call(
+            ["git", "config", "user.email", "[email protected]"], cwd=workdir
+        )
+        subprocess.check_call(
+            ["git", "config", "user.name", "Ray CI Test"], cwd=workdir
+        )
+        subprocess.check_call(["git", "remote", "add", "origin", origin], cwd=workdir)
+
+        with open(os.path.join(workdir, "README.md"), "w") as f:
+            f.write("# README\n")
+        subprocess.check_call(["git", "add", "README.md"], cwd=workdir)
+        subprocess.check_call(["git", "commit", "-m", "init with readme"], cwd=workdir)
+        subprocess.check_call(["git", "push", "origin", "master"], cwd=workdir)
+
+        for test_case in test_cases:
+            subprocess.check_call(
+                ["git", "checkout", "-B", "pr01", "master"], cwd=workdir
+            )
+
+            add_files = test_case[0]
+            for f in add_files:
+                dirname = os.path.dirname(f)
+                if dirname:
+                    os.makedirs(os.path.join(workdir, dirname), exist_ok=True)
+                with open(os.path.join(workdir, f), "w") as f:
+                    f.write("...\n")
+
+            subprocess.check_call(["git", "add", "."], cwd=workdir)
+            subprocess.check_call(
+                ["git", "commit", "-m", "add test files"], cwd=workdir
+            )
+            commit = (
+                subprocess.check_output(
+                    ["git", "show", "HEAD", "-q", "--format=%H"], cwd=workdir
+                )
+                .decode()
+                .strip()
+            )
+
+            envs = os.environ.copy()
+            envs["BUILDKITE"] = "true"
+            envs["BUILDKITE_PULL_REQUEST_BASE_BRANCH"] = "master"
+            envs["BUILDKITE_PULL_REQUEST"] = "true"
+            envs["BUILDKITE_COMMIT"] = commit
+
+            output = (
+                subprocess.check_output([sys.executable, script], env=envs, cwd=workdir)
+                .decode()
+                .strip()
+            )
+            tags = output.split()
+
+            want = test_case[1]
+            assert want == set(tags), f"want {want}, got {tags}"
+
+
+if __name__ == "__main__":
+    sys.exit(pytest.main(["-vv", __file__]))

ci/ray_ci/test_linux_tester_container.py

@@ -90,7 +90,10 @@ def _mock_popen(input: List[str]) -> None:
         return_value=None,
     ):
         LinuxTesterContainer(
-            "team", network="host", build_type="debug", test_envs=["ENV_01", "ENV_02"]
+            "team",
+            network="host",
+            build_type="debug",
+            test_envs=["ENV_01", "ENV_02"],
         )._run_tests_in_docker(["t1", "t2"], [0, 1], "/tmp", ["v=k"], "flag")
         input_str = inputs[-1]
         assert "--env ENV_01 --env ENV_02 --env BUILDKITE" in input_str
@@ -111,6 +114,12 @@ def _mock_popen(input: List[str]) -> None:
         assert "--gpus" not in input_str
         assert f"--runs_per_test {RUN_PER_FLAKY_TEST} " in input_str
 
+        LinuxTesterContainer("team")._run_tests_in_docker(
+            ["t1", "t2"], [], "/tmp", ["v=k"], cache_test_results=True
+        )
+        input_str = inputs[-1]
+        assert "--cache_test_results=auto" in input_str.split()
+
 
 def test_run_script_in_docker() -> None:
     def _mock_check_output(input: List[str]) -> bytes:
@@ -181,6 +190,7 @@ def _mock_run_tests_in_docker(
         test_envs: List[str],
         test_arg: Optional[str] = None,
         run_flaky_tests: Optional[bool] = False,
+        cache_test_results: Optional[bool] = False,
     ) -> MockPopen:
         return MockPopen(test_targets)
 

ci/ray_ci/tester.py

@@ -74,6 +74,13 @@
     type=str,
     help=("Only include tests with the given tags."),
 )
+@click.option(
+    "--cache-test-results",
+    is_flag=True,
+    show_default=True,
+    default=False,
+    help=("If cache and use test results in bazel cache."),
+)
 @click.option(
     "--run-flaky-tests",
     is_flag=True,
@@ -185,6 +192,7 @@ def main(
     operating_system: str,
     except_tags: str,
     only_tags: str,
+    cache_test_results: bool,
     run_flaky_tests: bool,
     run_high_impact_tests: bool,
     skip_ray_installation: bool,
@@ -249,6 +257,7 @@ def main(
         test_arg,
         is_bisect_run=bisect_run_test_target is not None,
         run_flaky_tests=run_flaky_tests,
+        cache_test_results=cache_test_results,
     )
     sys.exit(0 if success else 42)
 

ci/ray_ci/tester_container.py

@@ -74,6 +74,7 @@ def run_tests(
         test_arg: Optional[str] = None,
         is_bisect_run: bool = False,
         run_flaky_tests: bool = False,
+        cache_test_results: bool = False,
     ) -> bool:
         """
         Run tests parallelly in docker.  Return whether all tests pass.
@@ -101,8 +102,9 @@ def run_tests(
                 gpu_ids[i],
                 bazel_log_dir_host,
                 self.test_envs,
-                test_arg,
-                run_flaky_tests,
+                test_arg=test_arg,
+                run_flaky_tests=run_flaky_tests,
+                cache_test_results=cache_test_results,
             )
             for i in range(len(chunks))
         ]
@@ -218,6 +220,7 @@ def _run_tests_in_docker(
         test_envs: List[str],
         test_arg: Optional[str] = None,
         run_flaky_tests: bool = False,
+        cache_test_results: bool = False,
     ) -> subprocess.Popen:
         logger.info("Running tests: %s", test_targets)
         commands = [
@@ -252,6 +255,8 @@ def _run_tests_in_docker(
             test_cmd += f"--test_env {env} "
         if test_arg:
             test_cmd += f"--test_arg {test_arg} "
+        if cache_test_results:
+            test_cmd += "--cache_test_results=auto "
         if run_flaky_tests:
             test_cmd += f"--runs_per_test {RUN_PER_FLAKY_TEST} "
         test_cmd += f"{' '.join(test_targets)}"

ci/run/bazel_export_options

@@ -10,7 +10,6 @@ event_json_flag=--build_event_json_file=$(mktemp /tmp/bazel_event_logs/bazel_log
 logs_archive_flag=--test_env=RAY_TEST_FAILURE_LOGS_ARCHIVE_DIR="${artifact_mount_path}"/failed_test_logs
 summary_directory_flag=--test_env=RAY_TEST_SUMMARY_DIR="${artifact_mount_path}"/test-summaries
 summary_directory_host_flag=--test_env=RAY_TEST_SUMMARY_DIR_HOST=/tmp/artifacts/test-summaries
-buildkite_env_vars="--test_env=BUILDKITE_COMMIT --test_env=BUILDKITE_LABEL --test_env=BUILDKITE_JOB_ID"
 mujoco_env_var="--test_env=LD_LIBRARY_PATH=/root/.mujoco/mujoco210/bin"
 
-echo "${event_json_flag} ${logs_archive_flag} ${summary_directory_flag} ${summary_directory_host_flag} ${buildkite_env_vars} ${mujoco_env_var}"
+echo "${event_json_flag} ${logs_archive_flag} ${summary_directory_flag} ${summary_directory_host_flag} ${mujoco_env_var}"

cpp/BUILD.bazel

@@ -60,6 +60,7 @@ cc_library(
         "//:global_state_accessor_lib",
         "//:ray_common",
         "//src/ray/util",
+        "//src/ray/util:cmd_line_utils",
         "@boost//:callable_traits",
         "@boost//:dll",
         "@com_google_absl//absl/flags:flag",

cpp/src/ray/util/process_helper.cc

@@ -18,6 +18,7 @@
 #include <string>
 
 #include "ray/common/ray_config.h"
+#include "ray/util/cmd_line_utils.h"
 #include "ray/util/process.h"
 #include "ray/util/util.h"
 #include "src/ray/protobuf/gcs.pb.h"