diff --git a/docker/Dockerfile.softlearning.base.cpu b/docker/Dockerfile.softlearning.base.cpu
index 7a9ed593..35eda83f 100644
--- a/docker/Dockerfile.softlearning.base.cpu
+++ b/docker/Dockerfile.softlearning.base.cpu
@@ -133,6 +133,9 @@ COPY ./requirements.txt /tmp/
 RUN --mount=type=secret,id=mjkey,dst=/root/.mujoco/mjkey.txt \
     conda env update -f /tmp/environment.yml \
     && conda clean --all -y
+RUN source activate softlearning \
+    && pip install setuptools==65.5.0 wheel==0.38.0 \
+    && pip install gym==0.18.0
 
 RUN echo "conda activate softlearning" >> ~/.bashrc \
     && echo "cd ~/softlearning" >> ~/.bashrc
diff --git a/requirements.txt b/requirements.txt
index ad8423a3..532c6819 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -40,7 +40,6 @@ googleapis-common-protos==1.52.0
 gpustat==0.6.0
 grpcio==1.32.0
 gtimer==1.0.0b5
-gym==0.18.0
 h5py==2.10.0
 hiredis==1.1.0
 idna==2.10