diff --git a/docker/Dockerfile.softlearning.base.cpu b/docker/Dockerfile.softlearning.base.cpu
index 7a9ed593..9a847449 100644
--- a/docker/Dockerfile.softlearning.base.cpu
+++ b/docker/Dockerfile.softlearning.base.cpu
@@ -130,9 +130,11 @@ COPY ./requirements.txt /tmp/
 # NOTE: Fetch `mjkey.txt` from secret mount to avoid writing it to the build
 # history. For details, see:
 # https://docs.docker.com/develop/develop-images/build_enhancements/#new-docker-build-secret-information
-RUN --mount=type=secret,id=mjkey,dst=/root/.mujoco/mjkey.txt \
-    conda env update -f /tmp/environment.yml \
+RUN conda env update -f /tmp/environment.yml \
     && conda clean --all -y
+RUN source activate softlearning \
+    && pip install setuptools==65.5.0 wheel==0.38.0 \
+    && pip install gym==0.18.0
 
 RUN echo "conda activate softlearning" >> ~/.bashrc \
     && echo "cd ~/softlearning" >> ~/.bashrc
diff --git a/requirements.txt b/requirements.txt
index ad8423a3..532c6819 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -40,7 +40,6 @@ googleapis-common-protos==1.52.0
 gpustat==0.6.0
 grpcio==1.32.0
 gtimer==1.0.0b5
-gym==0.18.0
 h5py==2.10.0
 hiredis==1.1.0
 idna==2.10