[rllib] only run rllib gpu tests on rllib changes (ray-project#46509)

since the tests are currently flaky and slow Signed-off-by: Lonnie Liu <[email protected]>
jkpjkpjkp · Jul 9, 2024 · a09e5ea · a09e5ea
1 parent 6b98284
commit a09e5ea
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 6 deletions.
diff --git a/.buildkite/rllib.rayci.yml b/.buildkite/rllib.rayci.yml
@@ -105,9 +105,8 @@ steps:
 
   - label: ":brain: rllib: gpu tests"
     tags: 
-      - rllib
+      - rllib_gpu
       - gpu
-      - skip-on-premerge
     parallelism: 3
     instance_type: gpu
     commands:
@@ -198,12 +197,13 @@ steps:
 
   - label: ":brain: rllib: multi-gpu tests"
     tags: 
-      - rllib
+      - rllib_gpu
       - gpu
     parallelism: 2
     instance_type: gpu-large
     commands:
-      - bazel run //ci/ray_ci:test_in_docker -- //rllib/... rllib 
+      - bazel run //ci/ray_ci:test_in_docker -- //rllib/... rllib
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}"
         --parallelism-per-worker 2
         --gpus 4
         --build-name rllibgpubuild
@@ -213,9 +213,9 @@ steps:
   - label: ":brain: rllib: flaky multi-gpu tests"
     key: rllib_flaky_gpu_tests
     tags: 
-      - rllib
-      - skip-on-premerge
+      - rllib_gpu
       - gpu
+      - skip-on-premerge
     instance_type: gpu-large
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //rllib/... rllib --run-flaky-tests

diff --git a/ci/pipeline/determine_tests_to_run.py b/ci/pipeline/determine_tests_to_run.py
@@ -64,6 +64,7 @@ def get_commit_range():
     # Set to 1 for any changes to Ray Tune or python source files that are
     # NOT related to Serve, Dashboard or Train.
     RAY_CI_RLLIB_AFFECTED = 0
+    RAY_CI_RLLIB_GPU_AFFECTED = 0
     # Whether all RLlib tests should be run.
     # Set to 1 only when a source file in `ray/rllib` has been changed.
     RAY_CI_RLLIB_DIRECTLY_AFFECTED = 0
@@ -162,6 +163,7 @@ def get_commit_range():
                 or changed_file == ".buildkite/rllib.rayci.yml"
             ):
                 RAY_CI_RLLIB_AFFECTED = 1
+                RAY_CI_RLLIB_GPU_AFFECTED = 1
                 RAY_CI_RLLIB_DIRECTLY_AFFECTED = 1
                 RAY_CI_LINUX_WHEELS_AFFECTED = 1
                 RAY_CI_MACOS_WHEELS_AFFECTED = 1
@@ -404,6 +406,7 @@ def get_commit_range():
             "RAY_CI_TUNE_AFFECTED={}".format(RAY_CI_TUNE_AFFECTED),
             "RAY_CI_TRAIN_AFFECTED={}".format(RAY_CI_TRAIN_AFFECTED),
             "RAY_CI_RLLIB_AFFECTED={}".format(RAY_CI_RLLIB_AFFECTED),
+            "RAY_CI_RLLIB_GPU_AFFECTED={}".format(RAY_CI_RLLIB_GPU_AFFECTED),
             "RAY_CI_RLLIB_DIRECTLY_AFFECTED={}".format(RAY_CI_RLLIB_DIRECTLY_AFFECTED),
             "RAY_CI_RLLIB_CONTRIB_AFFECTED={}".format(RAY_CI_RLLIB_CONTRIB_AFFECTED),
             "RAY_CI_SERVE_AFFECTED={}".format(RAY_CI_SERVE_AFFECTED),