Skip to content

Commit

Permalink
Rewrite env tests (#2867)
Browse files Browse the repository at this point in the history
  • Loading branch information
pseudo-rnd-thoughts authored Jun 16, 2022
1 parent 71f11a0 commit ffbf971
Show file tree
Hide file tree
Showing 37 changed files with 447 additions and 635 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@ jobs:
--build-arg PYTHON_VERSION=${{ matrix.python-version }} \
--tag gym-docker .
- name: Run tests
run: docker run gym-docker pytest --import-mode=append
run: docker run gym-docker pytest
5 changes: 4 additions & 1 deletion gym/envs/mujoco/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
from gym.envs.mujoco.mujoco_env import MujocoEnv # isort:skip
from gym.envs.mujoco.mujoco_rendering import ( # isort:skip
RenderContextOffscreen,
Viewer,
)

# ^^^^^ so that user gets the correct error
# message if mujoco is not installed correctly
Expand All @@ -9,7 +13,6 @@
from gym.envs.mujoco.humanoidstandup import HumanoidStandupEnv
from gym.envs.mujoco.inverted_double_pendulum import InvertedDoublePendulumEnv
from gym.envs.mujoco.inverted_pendulum import InvertedPendulumEnv
from gym.envs.mujoco.mujoco_rendering import RenderContextOffscreen, Viewer
from gym.envs.mujoco.pusher import PusherEnv
from gym.envs.mujoco.reacher import ReacherEnv
from gym.envs.mujoco.swimmer import SwimmerEnv
Expand Down
1 change: 1 addition & 0 deletions tests/envs/register_during_make_env.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import gym


# todo, remove, use a generic testing environment
class RegisterDuringMakeEnv(gym.Env):
"""Used in `test_registration.py` to check if `env.make` can import and register an env"""

Expand Down
70 changes: 0 additions & 70 deletions tests/envs/spec_list.py

This file was deleted.

113 changes: 62 additions & 51 deletions tests/envs/test_action_dim_check.py
Original file line number Diff line number Diff line change
@@ -1,93 +1,102 @@
from typing import List

import numpy as np
import pytest

import gym
from gym import Env
from gym import spaces
from gym.envs.registration import EnvSpec
from gym.spaces.box import Box
from gym.spaces.discrete import Discrete
from tests.envs.spec_list import (
SKIP_MUJOCO_V3_WARNING_MESSAGE,
skip_mujoco_v3,
spec_list,
)

ENVIRONMENT_IDS = ("HalfCheetah-v2",)

from tests.envs.utils import all_testing_initialised_envs, mujoco_testing_env_specs

def filters_envs_action_space_type(
env_spec_list: List[EnvSpec], action_space: type
) -> List[Env]:
"""Make environments of specific action_space type.

This function returns a filtered list of environment from the spec_list that matches the action_space type.
Args:
env_spec_list (list): list of registered environments' specification
action_space (gym.spaces.Space): action_space type
@pytest.mark.parametrize(
"env_spec",
mujoco_testing_env_specs,
ids=[env_spec.id for env_spec in mujoco_testing_env_specs],
)
def test_mujoco_action_dimensions(env_spec: EnvSpec):
"""Test that for all mujoco environment, mis-dimensioned actions, an error is raised.
Types of mis-dimensioned actions:
* Too few actions
* Too many actions
* Too few dimensions
* Too many dimensions
* Incorrect shape
"""
filtered_envs = []
for spec in env_spec_list:
env = gym.make(spec.id)
if isinstance(env.action_space, action_space):
filtered_envs.append(env)
return filtered_envs


@pytest.mark.skipif(skip_mujoco_v3, reason=SKIP_MUJOCO_V3_WARNING_MESSAGE)
@pytest.mark.parametrize("environment_id", ENVIRONMENT_IDS)
def test_serialize_deserialize(environment_id):
env = gym.make(environment_id)
env = env_spec.make(disable_env_checker=True)
env.reset()

# Too few actions
with pytest.raises(ValueError, match="Action dimension mismatch"):
env.step([0.1])
env.step(env.action_space.sample()[1:])

# Too many actions
with pytest.raises(ValueError, match="Action dimension mismatch"):
env.step(np.append(env.action_space.sample(), 0))

# Too few dimensions
with pytest.raises(ValueError, match="Action dimension mismatch"):
env.step(0.1)

# Too many dimensions
with pytest.raises(ValueError, match="Action dimension mismatch"):
env.step(np.expand_dims(env.action_space.sample(), 0))

# Incorrect shape
with pytest.raises(ValueError, match="Action dimension mismatch"):
env.step(np.expand_dims(env.action_space.sample(), 1))

env.close()


@pytest.mark.parametrize("env", filters_envs_action_space_type(spec_list, Discrete))
def test_discrete_actions_out_of_bound(env):
@pytest.mark.parametrize(
"env",
filter(
lambda env: isinstance(env.action_space, spaces.Discrete),
all_testing_initialised_envs,
),
)
def test_discrete_actions_out_of_bound(env: gym.Env):
"""Test out of bound actions in Discrete action_space.
In discrete action_space environments, `out-of-bound`
actions are not allowed and should raise an exception.
Args:
env (gym.Env): the gym environment
"""
env.reset()

action_space = env.action_space
upper_bound = action_space.start + action_space.n - 1
assert isinstance(env.action_space, spaces.Discrete)
upper_bound = env.action_space.start + env.action_space.n - 1

env.reset()
with pytest.raises(Exception):
env.step(upper_bound + 1)

env.close()


OOB_VALUE = 100


@pytest.mark.parametrize(
("env", "seed"),
[(env, 42) for env in filters_envs_action_space_type(spec_list, Box)],
"env", filter(lambda env: isinstance(env, spaces.Box), all_testing_initialised_envs)
)
def test_box_actions_out_of_bound(env, seed):
def test_box_actions_out_of_bound(env: gym.Env):
"""Test out of bound actions in Box action_space.
Environments with Box actions spaces perform clipping inside `step`.
The expected behaviour is that an action `out-of-bound` has the same effect
of an action with value exactly at the upper (or lower) bound.
Args:
env (gym.Env): the gym environment
seed (int): seed value for determinism
"""
OOB_VALUE = 100
env.reset(seed=42)

env.reset(seed=seed)

oob_env = gym.make(env.spec.id)
oob_env.reset(seed=seed)
oob_env = gym.make(env.spec.id, disable_env_checker=True)
oob_env.reset(seed=42)

assert isinstance(env.action_space, spaces.Box)
dtype = env.action_space.dtype

upper_bounds = env.action_space.high
lower_bounds = env.action_space.low

Expand All @@ -113,3 +122,5 @@ def test_box_actions_out_of_bound(env, seed):
oob_obs, _, _, _ = oob_env.step(oob_action)

assert np.alltrue(obs == oob_obs)

env.close()
136 changes: 0 additions & 136 deletions tests/envs/test_atari_legacy_env_specs.py

This file was deleted.

Loading

0 comments on commit ffbf971

Please sign in to comment.