From f3632de754bff069f07c975d1fada642d7ae1907 Mon Sep 17 00:00:00 2001
From: Katherine Yang <katheriney@nvidia.com>
Date: Thu, 3 Aug 2023 19:17:53 -0700
Subject: [PATCH] add enviroment specific installation of shm

---
 qa/L0_backend_python/env/test.sh              | 136 +-------------
 .../setup_python_enviroment.sh                | 170 ++++++++++++++++++
 qa/L0_backend_python/test.sh                  | 106 +++++------
 3 files changed, 224 insertions(+), 188 deletions(-)
 create mode 100755 qa/L0_backend_python/setup_python_enviroment.sh

diff --git a/qa/L0_backend_python/env/test.sh b/qa/L0_backend_python/env/test.sh
index b1db62ea784..aeba2691cf9 100755
--- a/qa/L0_backend_python/env/test.sh
+++ b/qa/L0_backend_python/env/test.sh
@@ -42,125 +42,6 @@ rm -rf *.tar.gz
 install_build_deps
 install_conda
 
-if [ ${PYTHON_ENV_VERSION} != "10" ]; then
-    # Test other python versions
-    conda update -n base -c defaults conda -y
-    rm $SERVER_LOG
-    # Create a model with python 3.8 version
-    # Successful execution of the Python model indicates that the environment has
-    # been setup correctly.
-
-    create_conda_env "3.8" "python-3-8"
-    conda install -c conda-forge libstdcxx-ng=12 -y
-    conda install numpy=1.23.4 -y
-    conda install tensorflow=2.10.0 -y
-    PY38_VERSION_STRING="Python version is 3.8, NumPy version is 1.23.4, and Tensorflow version is 2.10.0"
-
-    create_python_backend_stub
-    conda-pack -o python3.8.tar.gz
-    path_to_conda_pack="$PWD/python-3-8"
-    mkdir -p $path_to_conda_pack
-    tar -xzf python3.8.tar.gz -C $path_to_conda_pack
-    mkdir -p models/python_3_8/1/
-    cp ../../python_models/python_version/config.pbtxt ./models/python_3_8
-    (cd models/python_3_8 && \
-            sed -i "s/^name:.*/name: \"python_3_8\"/" config.pbtxt && \
-            echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt)
-    cp ../../python_models/python_version/model.py ./models/python_3_8/1/
-    cp python_backend/builddir/triton_python_backend_stub ./models/python_3_8
-    # copy the stub out to /opt/tritonserver/backends/python/3-8
-    mkdir -p /opt/tritonserver/backends/python/3-8
-    cp python_backend/builddir/triton_python_backend_stub /opt/tritonserver/backends/python/3-8/triton_python_backend_stub
-    conda deactivate
-
-    # Create a model with python 3.9 version
-    # Successful execution of the Python model indicates that the environment has
-    # been setup correctly.
-    create_conda_env "3.9" "python-3-9"
-    conda install -c conda-forge libstdcxx-ng=12 -y
-    conda install numpy=1.23.4 -y
-    conda install tensorflow=2.10.0 -y
-    PY39_VERSION_STRING="Python version is 3.9, NumPy version is 1.23.4, and Tensorflow version is 2.10.0"
-    create_python_backend_stub
-    conda-pack -o python3.9.tar.gz
-    path_to_conda_pack="$PWD/python-3-9"
-    mkdir -p $path_to_conda_pack
-    tar -xzf python3.9.tar.gz -C $path_to_conda_pack
-    mkdir -p models/python_3_9/1/
-    cp ../../python_models/python_version/config.pbtxt ./models/python_3_9
-    (cd models/python_3_9 && \
-            sed -i "s/^name:.*/name: \"python_3_9\"/" config.pbtxt && \
-            echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt)
-    cp ../../python_models/python_version/model.py ./models/python_3_9/1/
-    cp python_backend/builddir/triton_python_backend_stub ./models/python_3_9
-    # copy the stub out to /opt/tritonserver/backends/python/3-9
-    mkdir -p /opt/tritonserver/backends/python/3-9
-    cp python_backend/builddir/triton_python_backend_stub /opt/tritonserver/backends/python/3-9/triton_python_backend_stub
-    conda deactivate
-
-    # Create a model with python 3.11 version
-    # Successful execution of the Python model indicates that the environment has
-    # been setup correctly.
-    create_conda_env "3.11" "python-3-11"
-    # tensorflow needs to be installed before numpy so pip does not mess up conda
-    # environment
-    pip install tensorflow==2.12.0
-    conda install -c conda-forge libstdcxx-ng=12 -y
-    conda install numpy=1.23.5 -y
-    PY311_VERSION_STRING="Python version is 3.11, NumPy version is 1.23.5, and Tensorflow version is 2.12.0"
-
-    create_python_backend_stub
-    conda-pack -o python3.11.tar.gz
-    path_to_conda_pack=`pwd`/python3.11.tar.gz
-    path_to_conda_pack="$PWD/python-3-11"
-    mkdir -p $path_to_conda_pack
-    tar -xzf python3.11.tar.gz -C $path_to_conda_pack
-    mkdir -p models/python_3_11/1/
-    cp ../../python_models/python_version/config.pbtxt ./models/python_3_11
-    (cd models/python_3_11 && \
-            sed -i "s/^name:.*/name: \"python_3_11\"/" config.pbtxt && \
-            echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt)
-    cp ../../python_models/python_version/model.py ./models/python_3_11/1/
-    cp python_backend/builddir/triton_python_backend_stub ./models/python_3_11
-    # copy the stub out to /opt/tritonserver/backends/python/3-11
-    mkdir -p /opt/tritonserver/backends/python/3-11
-    cp python_backend/builddir/triton_python_backend_stub /opt/tritonserver/backends/python/3-11/triton_python_backend_stub
-    conda deactivate
-
-    # copy the stub out to /opt/tritonserver/backends/python/3-10
-    mkdir -p /opt/tritonserver/backends/python/3-10
-    cp /opt/tritonserver/backends/python/triton_python_backend_stub /opt/tritonserver/backends/python/3-10/triton_python_backend_stub
-
-    rm -rf ./miniconda
-
-    run_server
-    if [ "$SERVER_PID" == "0" ]; then
-        echo -e "\n***\n*** Failed to start $SERVER\n***"
-        cat $SERVER_LOG
-        exit 1
-    fi
-
-    kill $SERVER_PID
-    wait $SERVER_PID
-
-    set +e
-    for EXPECTED_VERSION_STRING in "$PY38_VERSION_STRING" "$PY39_VERSION_STRING" "$PY311_VERSION_STRING"; do
-        grep "$EXPECTED_VERSION_STRING" $SERVER_LOG
-        if [ $? -ne 0 ]; then
-            cat $SERVER_LOG
-            echo -e "\n***\n*** $EXPECTED_VERSION_STRING was not found in Triton logs. \n***"
-            RET=1
-         fi
-    done
-    if [ $RET -eq 0 ]; then
-        echo -e "\n***\n*** Finished creating environments.\n***"
-    else
-        cat $SERVER_LOG
-    echo -e "\n***\n*** Failed to create python environments.\n***"
-    fi
-
-    exit $RET
-fi
 # Tensorflow 2.1.0 only works with Python 3.4 - 3.7. Successful execution of
 # the Python model indicates that the environment has been setup correctly.
 # Create a model with python 3.7 version
@@ -176,8 +57,8 @@ path_to_conda_pack=`pwd`/python3.7.tar.gz
 mkdir -p models/python_3_7/1/
 cp ../../python_models/python_version/config.pbtxt ./models/python_3_7
 (cd models/python_3_7 && \
-        sed -i "s/^name:.*/name: \"python_3_7\"/" config.pbtxt && \
-        echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt)
+          sed -i "s/^name:.*/name: \"python_3_7\"/" config.pbtxt && \
+          echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt)
 cp ../../python_models/python_version/model.py ./models/python_3_7/1/
 cp python_backend/builddir/triton_python_backend_stub ./models/python_3_7
 conda deactivate
@@ -198,8 +79,8 @@ create_python_backend_stub
 mkdir -p models/python_3_7_1/1/
 cp ../../python_models/python_version/config.pbtxt ./models/python_3_7_1
 (cd models/python_3_7_1 && \
-        sed -i "s/^name:.*/name: \"python_3_7_1\"/" config.pbtxt && \
-        echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt)
+          sed -i "s/^name:.*/name: \"python_3_7_1\"/" config.pbtxt && \
+          echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt)
 cp ../../python_models/python_version/model.py ./models/python_3_7_1/1/
 # Copy activate script to folder
 cp $path_to_conda_pack/lib/python3.7/site-packages/conda_pack/scripts/posix/activate $path_to_conda_pack/bin/.
@@ -223,8 +104,8 @@ mkdir -p models/python_3_6/1/
 cp ../../python_models/python_version/config.pbtxt ./models/python_3_6
 cp python3.6.tar.gz models/python_3_6/python_3_6_environment.tar.gz
 (cd models/python_3_6 && \
-        sed -i "s/^name:.*/name: \"python_3_6\"/" config.pbtxt && \
-        echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}" >> config.pbtxt)
+          sed -i "s/^name:.*/name: \"python_3_6\"/" config.pbtxt && \
+          echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}" >> config.pbtxt)
 cp ../../python_models/python_version/model.py ./models/python_3_6/1/
 cp python_backend/builddir/triton_python_backend_stub ./models/python_3_6
 conda deactivate
@@ -243,10 +124,11 @@ mkdir -p models/python_3_10/1/
 cp ../../python_models/python_version/config.pbtxt ./models/python_3_10
 cp python3.10.tar.gz models/python_3_10/python_3_10_environment.tar.gz
 (cd models/python_3_10 && \
-        sed -i "s/^name:.*/name: \"python_3_10\"/" config.pbtxt && \
-        echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}" >> config.pbtxt)
+          sed -i "s/^name:.*/name: \"python_3_10\"/" config.pbtxt && \
+          echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}" >> config.pbtxt)
 cp ../../python_models/python_version/model.py ./models/python_3_10/1/
 conda deactivate
+rm -rf ./miniconda
 
 run_server
 if [ "$SERVER_PID" == "0" ]; then
diff --git a/qa/L0_backend_python/setup_python_enviroment.sh b/qa/L0_backend_python/setup_python_enviroment.sh
new file mode 100755
index 00000000000..30d5631fb29
--- /dev/null
+++ b/qa/L0_backend_python/setup_python_enviroment.sh
@@ -0,0 +1,170 @@
+#!/bin/bash
+# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#  * Neither the name of NVIDIA CORPORATION nor the names of its
+#    contributors may be used to endorse or promote products derived
+#    from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
+# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+RET=0
+set -e
+if [ ${PYTHON_ENV_VERSION} = "10" ]; then
+    echo No need to set up anything for default python3.${PYTHON_ENV_VERSION}
+    exit $RET
+fi
+
+source common.sh
+source ../common/util.sh
+
+SERVER=/opt/tritonserver/bin/tritonserver
+BASE_SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1 --disable-auto-complete-config"
+PYTHON_BACKEND_BRANCH=$PYTHON_BACKEND_REPO_TAG
+SERVER_ARGS=$BASE_SERVER_ARGS
+SERVER_LOG="./inference_server.log"
+export PYTHON_ENV_VERSION=${PYTHON_ENV_VERSION:="10"}
+RET=0
+EXPECTED_VERSION_STRINGS=""
+
+rm -fr ./models
+rm -rf *.tar.gz
+install_build_deps
+install_conda
+
+# Test other python versions
+conda update -n base -c defaults conda -y
+# Create a model with python 3.8 version
+# Successful execution of the Python model indicates that the environment has
+# been setup correctly.
+if [ ${PYTHON_ENV_VERSION} = "8" ]; then
+    create_conda_env "3.8" "python-3-8"
+    conda install -c conda-forge libstdcxx-ng=12 -y
+    conda install numpy=1.23.4 -y
+    conda install tensorflow=2.10.0 -y
+    EXPECTED_VERSION_STRING="Python version is 3.8, NumPy version is 1.23.4, and Tensorflow version is 2.10.0"
+    create_python_backend_stub
+    conda-pack -o python3.8.tar.gz
+    path_to_conda_pack="$PWD/python-3-8"
+    mkdir -p $path_to_conda_pack
+    tar -xzf python3.8.tar.gz -C $path_to_conda_pack
+    mkdir -p models/python_3_8/1/
+    cp ../python_models/python_version/config.pbtxt ./models/python_3_8
+    (cd models/python_3_8 && \
+            sed -i "s/^name:.*/name: \"python_3_8\"/" config.pbtxt && \
+            echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt)
+    cp ../python_models/python_version/model.py ./models/python_3_8/1/
+    cp python_backend/builddir/triton_python_backend_stub ./models/python_3_8
+fi
+
+# Create a model with python 3.9 version
+# Successful execution of the Python model indicates that the environment has
+# been setup correctly.
+if [ ${PYTHON_ENV_VERSION} = "9" ]; then
+    create_conda_env "3.9" "python-3-9"
+    conda install -c conda-forge libstdcxx-ng=12 -y
+    conda install numpy=1.23.4 -y
+    conda install tensorflow=2.10.0 -y
+    EXPECTED_VERSION_STRING="Python version is 3.9, NumPy version is 1.23.4, and Tensorflow version is 2.10.0"
+    create_python_backend_stub
+    conda-pack -o python3.9.tar.gz
+    path_to_conda_pack="$PWD/python-3-9"
+    mkdir -p $path_to_conda_pack
+    tar -xzf python3.9.tar.gz -C $path_to_conda_pack
+    mkdir -p models/python_3_9/1/
+    cp ../python_models/python_version/config.pbtxt ./models/python_3_9
+    (cd models/python_3_9 && \
+            sed -i "s/^name:.*/name: \"python_3_9\"/" config.pbtxt && \
+            echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt)
+    cp ../python_models/python_version/model.py ./models/python_3_9/1/
+    cp python_backend/builddir/triton_python_backend_stub ./models/python_3_9
+fi
+
+# Create a model with python 3.11 version
+# Successful execution of the Python model indicates that the environment has
+# been setup correctly.
+if [ ${PYTHON_ENV_VERSION} = "11" ]; then
+    create_conda_env "3.11" "python-3-11"
+    # tensorflow needs to be installed before numpy so pip does not mess up conda
+    # environment
+    pip install tensorflow==2.12.0
+    conda install -c conda-forge libstdcxx-ng=12 -y
+    conda install numpy=1.23.5 -y
+    EXPECTED_VERSION_STRING="Python version is 3.11, NumPy version is 1.23.5, and Tensorflow version is 2.12.0"
+    create_python_backend_stub
+    conda-pack -o python3.11.tar.gz
+    path_to_conda_pack="$PWD/python-3-11"
+    mkdir -p $path_to_conda_pack
+    tar -xzf python3.11.tar.gz -C $path_to_conda_pack
+    mkdir -p models/python_3_11/1/
+    cp ../python_models/python_version/config.pbtxt ./models/python_3_11
+    (cd models/python_3_11 && \
+            sed -i "s/^name:.*/name: \"python_3_11\"/" config.pbtxt && \
+            echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt)
+    cp ../python_models/python_version/model.py ./models/python_3_11/1/
+    cp python_backend/builddir/triton_python_backend_stub ./models/python_3_11
+fi
+# copy the stub out to /opt/tritonserver/backends/python/triton_python_backend_stub
+cp python_backend/builddir/triton_python_backend_stub /opt/tritonserver/backends/python/triton_python_backend_stub
+conda deactivate
+rm -rf ./miniconda
+
+# test that
+set +e
+run_server
+if [ "$SERVER_PID" == "0" ]; then
+    echo -e "\n***\n*** Failed to start $SERVER\n***"
+    cat $SERVER_LOG
+    exit 1
+fi
+
+kill $SERVER_PID
+wait $SERVER_PID
+
+grep "$EXPECTED_VERSION_STRING" $SERVER_LOG
+if [ $? -ne 0 ]; then
+    cat $SERVER_LOG
+    echo -e "\n***\n*** $EXPECTED_VERSION_STRING was not found in Triton logs. \n***"
+    RET=1
+fi
+
+echo "python environment 3.${PYTHON_ENV_VERSION}"
+# Set up environment and stub for each test
+add-apt-repository ppa:deadsnakes/ppa -y
+apt-get update && apt-get -y install \
+                            "python3.${PYTHON_ENV_VERSION}-dev" \
+                            "python3.${PYTHON_ENV_VERSION}-distutils" \
+                            libboost-dev
+rm -f /usr/bin/python3 && \
+ln -s "/usr/bin/python3.${PYTHON_ENV_VERSION}" /usr/bin/python3
+pip3 install --upgrade install requests numpy virtualenv
+find /opt/tritonserver/qa/pkgs/ -maxdepth 1 -type f -name \
+    "tritonclient-*linux*.whl" | xargs printf -- '%s[all]' | \
+    xargs pip3 install --upgrade
+
+# Build triton-shm-monitor for the test
+cd python_backend && rm -rf install build && mkdir build && cd build && \
+    cmake -DCMAKE_INSTALL_PREFIX:PATH=$PWD/install \
+        -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} \
+        -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG} \
+        -DTRITON_BACKEND_REPO_TAG:STRING=${TRITON_BACKEND_REPO_TAG} .. && \
+    make -j16 triton-shm-monitor install
+cp $PWD/install/backends/python/triton_shm_monitor.cpython-* /opt/tritonserver/qa/common/.
+
+exit $RET
diff --git a/qa/L0_backend_python/test.sh b/qa/L0_backend_python/test.sh
index 36bcacc36a5..8bd361842d5 100755
--- a/qa/L0_backend_python/test.sh
+++ b/qa/L0_backend_python/test.sh
@@ -133,38 +133,14 @@ mkdir -p models/dlpack_identity/1/
 cp ../python_models/dlpack_identity/model.py ./models/dlpack_identity/1/
 cp ../python_models/dlpack_identity/config.pbtxt ./models/dlpack_identity
 
-# Env test should be run first so we can get the stubs for each of the
-# environments
-# Disable env test for Jetson since cloud storage repos are not supported
-if [ "$TEST_JETSON" == "0" ]; then
-    # In 'env' test we use miniconda for dependency management. No need to run
-    # the test in a virtual environment.
-    (cd env && bash -ex test.sh)
-    if [ $? -ne 0 ]; then
-        echo "Subtest env FAILED"
-        RET=1
-    fi
-fi
-
-echo "python environment 3.${PYTHON_ENV_VERSION}"
-# Set up environment and stub for each test
-add-apt-repository ppa:deadsnakes/ppa -y
-apt-get update && apt-get -y install \
-                            "python3.${PYTHON_ENV_VERSION}-dev" \
-                            "python3.${PYTHON_ENV_VERSION}-distutils" \
-                            numpy
-rm -f /usr/bin/python3 && \
-ln -s "/usr/bin/python3.${PYTHON_ENV_VERSION}" /usr/bin/python3
-
-PYTHON_STUB_LOCATION=/opt/tritonserver/backends/python/3-${PYTHON_ENV_VERSION}/triton_python_backend_stub
-cp ${PYTHON_STUB_LOCATION} /opt/tritonserver/backends/python/triton_python_backend_stub
+(bash -ex setup_python_enviroment.sh)
 
 # Skip torch install on Jetson since it is already installed.
 if [ "$TEST_JETSON" == "0" ]; then
-pip3 install torch==1.13.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
+  pip3 install torch==1.13.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
 else
-# GPU tensor tests are disabled on jetson
-EXPECTED_NUM_TESTS=9
+  # GPU tensor tests are disabled on jetson
+  EXPECTED_NUM_TESTS=9
 fi
 
 prev_num_pages=`get_shm_pages`
@@ -239,37 +215,37 @@ set -e
 # Test KIND_GPU
 # Disable env test for Jetson since GPU Tensors are not supported
 if [ "$TEST_JETSON" == "0" ]; then
-rm -rf models/
-mkdir -p models/add_sub_gpu/1/
-cp ../python_models/add_sub/model.py ./models/add_sub_gpu/1/
-cp ../python_models/add_sub_gpu/config.pbtxt ./models/add_sub_gpu/
-
-prev_num_pages=`get_shm_pages`
-run_server
-if [ "$SERVER_PID" == "0" ]; then
-    cat $SERVER_LOG
-    echo -e "\n***\n*** Failed to start $SERVER\n***"
-    exit 1
-fi
-
-if [ $? -ne 0 ]; then
-    cat $SERVER_LOG
-    echo -e "\n***\n*** KIND_GPU model test failed \n***"
-    RET=1
-fi
-
-kill $SERVER_PID
-wait $SERVER_PID
-
-current_num_pages=`get_shm_pages`
-if [ $current_num_pages -ne $prev_num_pages ]; then
-    cat $CLIENT_LOG
-    ls /dev/shm
-    echo -e "\n***\n*** Test Failed. Shared memory pages where not cleaned properly.
-Shared memory pages before starting triton equals to $prev_num_pages
-and shared memory pages after starting triton equals to $current_num_pages \n***"
-    exit 1
-fi
+  rm -rf models/
+  mkdir -p models/add_sub_gpu/1/
+  cp ../python_models/add_sub/model.py ./models/add_sub_gpu/1/
+  cp ../python_models/add_sub_gpu/config.pbtxt ./models/add_sub_gpu/
+
+  prev_num_pages=`get_shm_pages`
+  run_server
+  if [ "$SERVER_PID" == "0" ]; then
+      cat $SERVER_LOG
+      echo -e "\n***\n*** Failed to start $SERVER\n***"
+      exit 1
+  fi
+
+  if [ $? -ne 0 ]; then
+      cat $SERVER_LOG
+      echo -e "\n***\n*** KIND_GPU model test failed \n***"
+      RET=1
+  fi
+
+  kill $SERVER_PID
+  wait $SERVER_PID
+
+  current_num_pages=`get_shm_pages`
+  if [ $current_num_pages -ne $prev_num_pages ]; then
+      cat $CLIENT_LOG
+      ls /dev/shm
+      echo -e "\n***\n*** Test Failed. Shared memory pages where not cleaned properly.
+  Shared memory pages before starting triton equals to $prev_num_pages
+  and shared memory pages after starting triton equals to $current_num_pages \n***"
+      exit 1
+  fi
 fi
 
 # Test Multi file models
@@ -278,7 +254,7 @@ mkdir -p models/multi_file/1/
 cp ../python_models/multi_file/*.py ./models/multi_file/1/
 cp ../python_models/identity_fp32/config.pbtxt ./models/multi_file/
 (cd models/multi_file && \
-        sed -i "s/^name:.*/name: \"multi_file\"/" config.pbtxt)
+          sed -i "s/^name:.*/name: \"multi_file\"/" config.pbtxt)
 
 prev_num_pages=`get_shm_pages`
 run_server
@@ -402,6 +378,7 @@ and shared memory pages after starting triton equals to $current_num_pages \n***
     exit 1
 fi
 
+# Disable env test for Jetson since cloud storage repos are not supported
 # Disable ensemble, io and bls tests for Jetson since GPU Tensors are not supported
 # Disable variants test for Jetson since already built without GPU Tensor support
 # Disable decoupled test because it uses GPU tensors
@@ -422,6 +399,14 @@ if [ "$TEST_JETSON" == "0" ]; then
         deactivate
         rm -fr venv
     done
+
+    # In 'env' test we use miniconda for dependency management. No need to run
+    # the test in a virtual environment.
+    (cd env && bash -ex test.sh)
+    if [ $? -ne 0 ]; then
+        echo "Subtest env FAILED"
+        RET=1
+    fi
 fi
 
 SUBTESTS="lifecycle restart model_control examples argument_validation logging custom_metrics"
@@ -442,7 +427,6 @@ for TEST in ${SUBTESTS}; do
     rm -fr venv
 done
 
-
 if [ $RET -eq 0 ]; then
   echo -e "\n***\n*** Test Passed\n***"
 else