diff --git a/.buildkite/rllib.rayci.yml b/.buildkite/rllib.rayci.yml
index 1a22fed8e1501..cb690839d4bde 100644
--- a/.buildkite/rllib.rayci.yml
+++ b/.buildkite/rllib.rayci.yml
@@ -105,9 +105,8 @@ steps:
 
   - label: ":brain: rllib: gpu tests"
     tags: 
-      - rllib
+      - rllib_gpu
       - gpu
-      - skip-on-premerge
     parallelism: 3
     instance_type: gpu
     commands:
@@ -198,12 +197,13 @@ steps:
 
   - label: ":brain: rllib: multi-gpu tests"
     tags: 
-      - rllib
+      - rllib_gpu
       - gpu
     parallelism: 2
     instance_type: gpu-large
     commands:
-      - bazel run //ci/ray_ci:test_in_docker -- //rllib/... rllib 
+      - bazel run //ci/ray_ci:test_in_docker -- //rllib/... rllib
+        --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}"
         --parallelism-per-worker 2
         --gpus 4
         --build-name rllibgpubuild
@@ -213,9 +213,9 @@ steps:
   - label: ":brain: rllib: flaky multi-gpu tests"
     key: rllib_flaky_gpu_tests
     tags: 
-      - rllib
-      - skip-on-premerge
+      - rllib_gpu
       - gpu
+      - skip-on-premerge
     instance_type: gpu-large
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //rllib/... rllib --run-flaky-tests
diff --git a/ci/pipeline/determine_tests_to_run.py b/ci/pipeline/determine_tests_to_run.py
index a01acef5e4bc9..c92a3474229a0 100644
--- a/ci/pipeline/determine_tests_to_run.py
+++ b/ci/pipeline/determine_tests_to_run.py
@@ -64,6 +64,7 @@ def get_commit_range():
     # Set to 1 for any changes to Ray Tune or python source files that are
     # NOT related to Serve, Dashboard or Train.
     RAY_CI_RLLIB_AFFECTED = 0
+    RAY_CI_RLLIB_GPU_AFFECTED = 0
     # Whether all RLlib tests should be run.
     # Set to 1 only when a source file in `ray/rllib` has been changed.
     RAY_CI_RLLIB_DIRECTLY_AFFECTED = 0
@@ -162,6 +163,7 @@ def get_commit_range():
                 or changed_file == ".buildkite/rllib.rayci.yml"
             ):
                 RAY_CI_RLLIB_AFFECTED = 1
+                RAY_CI_RLLIB_GPU_AFFECTED = 1
                 RAY_CI_RLLIB_DIRECTLY_AFFECTED = 1
                 RAY_CI_LINUX_WHEELS_AFFECTED = 1
                 RAY_CI_MACOS_WHEELS_AFFECTED = 1
@@ -404,6 +406,7 @@ def get_commit_range():
             "RAY_CI_TUNE_AFFECTED={}".format(RAY_CI_TUNE_AFFECTED),
             "RAY_CI_TRAIN_AFFECTED={}".format(RAY_CI_TRAIN_AFFECTED),
             "RAY_CI_RLLIB_AFFECTED={}".format(RAY_CI_RLLIB_AFFECTED),
+            "RAY_CI_RLLIB_GPU_AFFECTED={}".format(RAY_CI_RLLIB_GPU_AFFECTED),
             "RAY_CI_RLLIB_DIRECTLY_AFFECTED={}".format(RAY_CI_RLLIB_DIRECTLY_AFFECTED),
             "RAY_CI_RLLIB_CONTRIB_AFFECTED={}".format(RAY_CI_RLLIB_CONTRIB_AFFECTED),
             "RAY_CI_SERVE_AFFECTED={}".format(RAY_CI_SERVE_AFFECTED),