diff --git a/bin/all-py.Dockerfile b/bin/all-py.Dockerfile
index afb357425..c1c411151 100644
--- a/bin/all-py.Dockerfile
+++ b/bin/all-py.Dockerfile
@@ -26,6 +26,7 @@ ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/root/.mujoco/mujoco210/bin"
 COPY . /usr/local/gymnasium/
 WORKDIR /usr/local/gymnasium/
 
-RUN pip install .[all,testing] --no-cache-dir
+# Test with PyTorch CPU build, since CUDA is not available in CI anyway
+RUN pip install .[all,testing] --no-cache-dir --extra-index-url https://download.pytorch.org/whl/cpu
 
 ENTRYPOINT ["/usr/local/gymnasium/bin/docker_entrypoint"]
diff --git a/tests/envs/test_rendering.py b/tests/envs/test_rendering.py
index 2bec341c6..b2604c5ff 100644
--- a/tests/envs/test_rendering.py
+++ b/tests/envs/test_rendering.py
@@ -62,13 +62,19 @@ def test_render_modes(spec):
         if mode != "human":
             new_env = spec.make(render_mode=mode)
 
-            new_env.reset()
-            rendered = new_env.render()
-            check_rendered(rendered, mode)
+            try:
+                new_env.reset()
+                rendered = new_env.render()
+                check_rendered(rendered, mode)
 
-            new_env.step(new_env.action_space.sample())
-            rendered = new_env.render()
-            check_rendered(rendered, mode)
-
-            new_env.close()
+                new_env.step(new_env.action_space.sample())
+                rendered = new_env.render()
+                check_rendered(rendered, mode)
+            except Exception as e:
+                if "gladLoadGL error" in str(e):
+                    pytest.skip("OpenGL not available")
+                else:
+                    raise
+            finally:
+                new_env.close()
     env.close()
diff --git a/tests/envs/utils.py b/tests/envs/utils.py
index e643d22b7..a16133d76 100644
--- a/tests/envs/utils.py
+++ b/tests/envs/utils.py
@@ -22,6 +22,7 @@ def try_make_env(env_spec: EnvSpec) -> Optional[gym.Env]:
             return env_spec.make(disable_env_checker=True).unwrapped
         except (
             ImportError,
+            AttributeError,
             gym.error.DependencyNotInstalled,
             gym.error.MissingArgument,
         ) as e: