diff --git a/qa/L0_backend_python/env/test.sh b/qa/L0_backend_python/env/test.sh index b1db62ea784..aeba2691cf9 100755 --- a/qa/L0_backend_python/env/test.sh +++ b/qa/L0_backend_python/env/test.sh @@ -42,125 +42,6 @@ rm -rf *.tar.gz install_build_deps install_conda -if [ ${PYTHON_ENV_VERSION} != "10" ]; then - # Test other python versions - conda update -n base -c defaults conda -y - rm $SERVER_LOG - # Create a model with python 3.8 version - # Successful execution of the Python model indicates that the environment has - # been setup correctly. - - create_conda_env "3.8" "python-3-8" - conda install -c conda-forge libstdcxx-ng=12 -y - conda install numpy=1.23.4 -y - conda install tensorflow=2.10.0 -y - PY38_VERSION_STRING="Python version is 3.8, NumPy version is 1.23.4, and Tensorflow version is 2.10.0" - - create_python_backend_stub - conda-pack -o python3.8.tar.gz - path_to_conda_pack="$PWD/python-3-8" - mkdir -p $path_to_conda_pack - tar -xzf python3.8.tar.gz -C $path_to_conda_pack - mkdir -p models/python_3_8/1/ - cp ../../python_models/python_version/config.pbtxt ./models/python_3_8 - (cd models/python_3_8 && \ - sed -i "s/^name:.*/name: \"python_3_8\"/" config.pbtxt && \ - echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) - cp ../../python_models/python_version/model.py ./models/python_3_8/1/ - cp python_backend/builddir/triton_python_backend_stub ./models/python_3_8 - # copy the stub out to /opt/tritonserver/backends/python/3-8 - mkdir -p /opt/tritonserver/backends/python/3-8 - cp python_backend/builddir/triton_python_backend_stub /opt/tritonserver/backends/python/3-8/triton_python_backend_stub - conda deactivate - - # Create a model with python 3.9 version - # Successful execution of the Python model indicates that the environment has - # been setup correctly. - create_conda_env "3.9" "python-3-9" - conda install -c conda-forge libstdcxx-ng=12 -y - conda install numpy=1.23.4 -y - conda install tensorflow=2.10.0 -y - PY39_VERSION_STRING="Python version is 3.9, NumPy version is 1.23.4, and Tensorflow version is 2.10.0" - create_python_backend_stub - conda-pack -o python3.9.tar.gz - path_to_conda_pack="$PWD/python-3-9" - mkdir -p $path_to_conda_pack - tar -xzf python3.9.tar.gz -C $path_to_conda_pack - mkdir -p models/python_3_9/1/ - cp ../../python_models/python_version/config.pbtxt ./models/python_3_9 - (cd models/python_3_9 && \ - sed -i "s/^name:.*/name: \"python_3_9\"/" config.pbtxt && \ - echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) - cp ../../python_models/python_version/model.py ./models/python_3_9/1/ - cp python_backend/builddir/triton_python_backend_stub ./models/python_3_9 - # copy the stub out to /opt/tritonserver/backends/python/3-9 - mkdir -p /opt/tritonserver/backends/python/3-9 - cp python_backend/builddir/triton_python_backend_stub /opt/tritonserver/backends/python/3-9/triton_python_backend_stub - conda deactivate - - # Create a model with python 3.11 version - # Successful execution of the Python model indicates that the environment has - # been setup correctly. - create_conda_env "3.11" "python-3-11" - # tensorflow needs to be installed before numpy so pip does not mess up conda - # environment - pip install tensorflow==2.12.0 - conda install -c conda-forge libstdcxx-ng=12 -y - conda install numpy=1.23.5 -y - PY311_VERSION_STRING="Python version is 3.11, NumPy version is 1.23.5, and Tensorflow version is 2.12.0" - - create_python_backend_stub - conda-pack -o python3.11.tar.gz - path_to_conda_pack=`pwd`/python3.11.tar.gz - path_to_conda_pack="$PWD/python-3-11" - mkdir -p $path_to_conda_pack - tar -xzf python3.11.tar.gz -C $path_to_conda_pack - mkdir -p models/python_3_11/1/ - cp ../../python_models/python_version/config.pbtxt ./models/python_3_11 - (cd models/python_3_11 && \ - sed -i "s/^name:.*/name: \"python_3_11\"/" config.pbtxt && \ - echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) - cp ../../python_models/python_version/model.py ./models/python_3_11/1/ - cp python_backend/builddir/triton_python_backend_stub ./models/python_3_11 - # copy the stub out to /opt/tritonserver/backends/python/3-11 - mkdir -p /opt/tritonserver/backends/python/3-11 - cp python_backend/builddir/triton_python_backend_stub /opt/tritonserver/backends/python/3-11/triton_python_backend_stub - conda deactivate - - # copy the stub out to /opt/tritonserver/backends/python/3-10 - mkdir -p /opt/tritonserver/backends/python/3-10 - cp /opt/tritonserver/backends/python/triton_python_backend_stub /opt/tritonserver/backends/python/3-10/triton_python_backend_stub - - rm -rf ./miniconda - - run_server - if [ "$SERVER_PID" == "0" ]; then - echo -e "\n***\n*** Failed to start $SERVER\n***" - cat $SERVER_LOG - exit 1 - fi - - kill $SERVER_PID - wait $SERVER_PID - - set +e - for EXPECTED_VERSION_STRING in "$PY38_VERSION_STRING" "$PY39_VERSION_STRING" "$PY311_VERSION_STRING"; do - grep "$EXPECTED_VERSION_STRING" $SERVER_LOG - if [ $? -ne 0 ]; then - cat $SERVER_LOG - echo -e "\n***\n*** $EXPECTED_VERSION_STRING was not found in Triton logs. \n***" - RET=1 - fi - done - if [ $RET -eq 0 ]; then - echo -e "\n***\n*** Finished creating environments.\n***" - else - cat $SERVER_LOG - echo -e "\n***\n*** Failed to create python environments.\n***" - fi - - exit $RET -fi # Tensorflow 2.1.0 only works with Python 3.4 - 3.7. Successful execution of # the Python model indicates that the environment has been setup correctly. # Create a model with python 3.7 version @@ -176,8 +57,8 @@ path_to_conda_pack=`pwd`/python3.7.tar.gz mkdir -p models/python_3_7/1/ cp ../../python_models/python_version/config.pbtxt ./models/python_3_7 (cd models/python_3_7 && \ - sed -i "s/^name:.*/name: \"python_3_7\"/" config.pbtxt && \ - echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) + sed -i "s/^name:.*/name: \"python_3_7\"/" config.pbtxt && \ + echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) cp ../../python_models/python_version/model.py ./models/python_3_7/1/ cp python_backend/builddir/triton_python_backend_stub ./models/python_3_7 conda deactivate @@ -198,8 +79,8 @@ create_python_backend_stub mkdir -p models/python_3_7_1/1/ cp ../../python_models/python_version/config.pbtxt ./models/python_3_7_1 (cd models/python_3_7_1 && \ - sed -i "s/^name:.*/name: \"python_3_7_1\"/" config.pbtxt && \ - echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) + sed -i "s/^name:.*/name: \"python_3_7_1\"/" config.pbtxt && \ + echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) cp ../../python_models/python_version/model.py ./models/python_3_7_1/1/ # Copy activate script to folder cp $path_to_conda_pack/lib/python3.7/site-packages/conda_pack/scripts/posix/activate $path_to_conda_pack/bin/. @@ -223,8 +104,8 @@ mkdir -p models/python_3_6/1/ cp ../../python_models/python_version/config.pbtxt ./models/python_3_6 cp python3.6.tar.gz models/python_3_6/python_3_6_environment.tar.gz (cd models/python_3_6 && \ - sed -i "s/^name:.*/name: \"python_3_6\"/" config.pbtxt && \ - echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}" >> config.pbtxt) + sed -i "s/^name:.*/name: \"python_3_6\"/" config.pbtxt && \ + echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}" >> config.pbtxt) cp ../../python_models/python_version/model.py ./models/python_3_6/1/ cp python_backend/builddir/triton_python_backend_stub ./models/python_3_6 conda deactivate @@ -243,10 +124,11 @@ mkdir -p models/python_3_10/1/ cp ../../python_models/python_version/config.pbtxt ./models/python_3_10 cp python3.10.tar.gz models/python_3_10/python_3_10_environment.tar.gz (cd models/python_3_10 && \ - sed -i "s/^name:.*/name: \"python_3_10\"/" config.pbtxt && \ - echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}" >> config.pbtxt) + sed -i "s/^name:.*/name: \"python_3_10\"/" config.pbtxt && \ + echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}" >> config.pbtxt) cp ../../python_models/python_version/model.py ./models/python_3_10/1/ conda deactivate +rm -rf ./miniconda run_server if [ "$SERVER_PID" == "0" ]; then diff --git a/qa/L0_backend_python/setup_python_enviroment.sh b/qa/L0_backend_python/setup_python_enviroment.sh new file mode 100755 index 00000000000..30d5631fb29 --- /dev/null +++ b/qa/L0_backend_python/setup_python_enviroment.sh @@ -0,0 +1,170 @@ +#!/bin/bash +# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +RET=0 +set -e +if [ ${PYTHON_ENV_VERSION} = "10" ]; then + echo No need to set up anything for default python3.${PYTHON_ENV_VERSION} + exit $RET +fi + +source common.sh +source ../common/util.sh + +SERVER=/opt/tritonserver/bin/tritonserver +BASE_SERVER_ARGS="--model-repository=`pwd`/models --log-verbose=1 --disable-auto-complete-config" +PYTHON_BACKEND_BRANCH=$PYTHON_BACKEND_REPO_TAG +SERVER_ARGS=$BASE_SERVER_ARGS +SERVER_LOG="./inference_server.log" +export PYTHON_ENV_VERSION=${PYTHON_ENV_VERSION:="10"} +RET=0 +EXPECTED_VERSION_STRINGS="" + +rm -fr ./models +rm -rf *.tar.gz +install_build_deps +install_conda + +# Test other python versions +conda update -n base -c defaults conda -y +# Create a model with python 3.8 version +# Successful execution of the Python model indicates that the environment has +# been setup correctly. +if [ ${PYTHON_ENV_VERSION} = "8" ]; then + create_conda_env "3.8" "python-3-8" + conda install -c conda-forge libstdcxx-ng=12 -y + conda install numpy=1.23.4 -y + conda install tensorflow=2.10.0 -y + EXPECTED_VERSION_STRING="Python version is 3.8, NumPy version is 1.23.4, and Tensorflow version is 2.10.0" + create_python_backend_stub + conda-pack -o python3.8.tar.gz + path_to_conda_pack="$PWD/python-3-8" + mkdir -p $path_to_conda_pack + tar -xzf python3.8.tar.gz -C $path_to_conda_pack + mkdir -p models/python_3_8/1/ + cp ../python_models/python_version/config.pbtxt ./models/python_3_8 + (cd models/python_3_8 && \ + sed -i "s/^name:.*/name: \"python_3_8\"/" config.pbtxt && \ + echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) + cp ../python_models/python_version/model.py ./models/python_3_8/1/ + cp python_backend/builddir/triton_python_backend_stub ./models/python_3_8 +fi + +# Create a model with python 3.9 version +# Successful execution of the Python model indicates that the environment has +# been setup correctly. +if [ ${PYTHON_ENV_VERSION} = "9" ]; then + create_conda_env "3.9" "python-3-9" + conda install -c conda-forge libstdcxx-ng=12 -y + conda install numpy=1.23.4 -y + conda install tensorflow=2.10.0 -y + EXPECTED_VERSION_STRING="Python version is 3.9, NumPy version is 1.23.4, and Tensorflow version is 2.10.0" + create_python_backend_stub + conda-pack -o python3.9.tar.gz + path_to_conda_pack="$PWD/python-3-9" + mkdir -p $path_to_conda_pack + tar -xzf python3.9.tar.gz -C $path_to_conda_pack + mkdir -p models/python_3_9/1/ + cp ../python_models/python_version/config.pbtxt ./models/python_3_9 + (cd models/python_3_9 && \ + sed -i "s/^name:.*/name: \"python_3_9\"/" config.pbtxt && \ + echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) + cp ../python_models/python_version/model.py ./models/python_3_9/1/ + cp python_backend/builddir/triton_python_backend_stub ./models/python_3_9 +fi + +# Create a model with python 3.11 version +# Successful execution of the Python model indicates that the environment has +# been setup correctly. +if [ ${PYTHON_ENV_VERSION} = "11" ]; then + create_conda_env "3.11" "python-3-11" + # tensorflow needs to be installed before numpy so pip does not mess up conda + # environment + pip install tensorflow==2.12.0 + conda install -c conda-forge libstdcxx-ng=12 -y + conda install numpy=1.23.5 -y + EXPECTED_VERSION_STRING="Python version is 3.11, NumPy version is 1.23.5, and Tensorflow version is 2.12.0" + create_python_backend_stub + conda-pack -o python3.11.tar.gz + path_to_conda_pack="$PWD/python-3-11" + mkdir -p $path_to_conda_pack + tar -xzf python3.11.tar.gz -C $path_to_conda_pack + mkdir -p models/python_3_11/1/ + cp ../python_models/python_version/config.pbtxt ./models/python_3_11 + (cd models/python_3_11 && \ + sed -i "s/^name:.*/name: \"python_3_11\"/" config.pbtxt && \ + echo "parameters: {key: \"EXECUTION_ENV_PATH\", value: {string_value: \"$path_to_conda_pack\"}}">> config.pbtxt) + cp ../python_models/python_version/model.py ./models/python_3_11/1/ + cp python_backend/builddir/triton_python_backend_stub ./models/python_3_11 +fi +# copy the stub out to /opt/tritonserver/backends/python/triton_python_backend_stub +cp python_backend/builddir/triton_python_backend_stub /opt/tritonserver/backends/python/triton_python_backend_stub +conda deactivate +rm -rf ./miniconda + +# test that +set +e +run_server +if [ "$SERVER_PID" == "0" ]; then + echo -e "\n***\n*** Failed to start $SERVER\n***" + cat $SERVER_LOG + exit 1 +fi + +kill $SERVER_PID +wait $SERVER_PID + +grep "$EXPECTED_VERSION_STRING" $SERVER_LOG +if [ $? -ne 0 ]; then + cat $SERVER_LOG + echo -e "\n***\n*** $EXPECTED_VERSION_STRING was not found in Triton logs. \n***" + RET=1 +fi + +echo "python environment 3.${PYTHON_ENV_VERSION}" +# Set up environment and stub for each test +add-apt-repository ppa:deadsnakes/ppa -y +apt-get update && apt-get -y install \ + "python3.${PYTHON_ENV_VERSION}-dev" \ + "python3.${PYTHON_ENV_VERSION}-distutils" \ + libboost-dev +rm -f /usr/bin/python3 && \ +ln -s "/usr/bin/python3.${PYTHON_ENV_VERSION}" /usr/bin/python3 +pip3 install --upgrade install requests numpy virtualenv +find /opt/tritonserver/qa/pkgs/ -maxdepth 1 -type f -name \ + "tritonclient-*linux*.whl" | xargs printf -- '%s[all]' | \ + xargs pip3 install --upgrade + +# Build triton-shm-monitor for the test +cd python_backend && rm -rf install build && mkdir build && cd build && \ + cmake -DCMAKE_INSTALL_PREFIX:PATH=$PWD/install \ + -DTRITON_COMMON_REPO_TAG:STRING=${TRITON_COMMON_REPO_TAG} \ + -DTRITON_CORE_REPO_TAG:STRING=${TRITON_CORE_REPO_TAG} \ + -DTRITON_BACKEND_REPO_TAG:STRING=${TRITON_BACKEND_REPO_TAG} .. && \ + make -j16 triton-shm-monitor install +cp $PWD/install/backends/python/triton_shm_monitor.cpython-* /opt/tritonserver/qa/common/. + +exit $RET diff --git a/qa/L0_backend_python/test.sh b/qa/L0_backend_python/test.sh index 36bcacc36a5..8bd361842d5 100755 --- a/qa/L0_backend_python/test.sh +++ b/qa/L0_backend_python/test.sh @@ -133,38 +133,14 @@ mkdir -p models/dlpack_identity/1/ cp ../python_models/dlpack_identity/model.py ./models/dlpack_identity/1/ cp ../python_models/dlpack_identity/config.pbtxt ./models/dlpack_identity -# Env test should be run first so we can get the stubs for each of the -# environments -# Disable env test for Jetson since cloud storage repos are not supported -if [ "$TEST_JETSON" == "0" ]; then - # In 'env' test we use miniconda for dependency management. No need to run - # the test in a virtual environment. - (cd env && bash -ex test.sh) - if [ $? -ne 0 ]; then - echo "Subtest env FAILED" - RET=1 - fi -fi - -echo "python environment 3.${PYTHON_ENV_VERSION}" -# Set up environment and stub for each test -add-apt-repository ppa:deadsnakes/ppa -y -apt-get update && apt-get -y install \ - "python3.${PYTHON_ENV_VERSION}-dev" \ - "python3.${PYTHON_ENV_VERSION}-distutils" \ - numpy -rm -f /usr/bin/python3 && \ -ln -s "/usr/bin/python3.${PYTHON_ENV_VERSION}" /usr/bin/python3 - -PYTHON_STUB_LOCATION=/opt/tritonserver/backends/python/3-${PYTHON_ENV_VERSION}/triton_python_backend_stub -cp ${PYTHON_STUB_LOCATION} /opt/tritonserver/backends/python/triton_python_backend_stub +(bash -ex setup_python_enviroment.sh) # Skip torch install on Jetson since it is already installed. if [ "$TEST_JETSON" == "0" ]; then -pip3 install torch==1.13.0+cpu -f https://download.pytorch.org/whl/torch_stable.html + pip3 install torch==1.13.0+cpu -f https://download.pytorch.org/whl/torch_stable.html else -# GPU tensor tests are disabled on jetson -EXPECTED_NUM_TESTS=9 + # GPU tensor tests are disabled on jetson + EXPECTED_NUM_TESTS=9 fi prev_num_pages=`get_shm_pages` @@ -239,37 +215,37 @@ set -e # Test KIND_GPU # Disable env test for Jetson since GPU Tensors are not supported if [ "$TEST_JETSON" == "0" ]; then -rm -rf models/ -mkdir -p models/add_sub_gpu/1/ -cp ../python_models/add_sub/model.py ./models/add_sub_gpu/1/ -cp ../python_models/add_sub_gpu/config.pbtxt ./models/add_sub_gpu/ - -prev_num_pages=`get_shm_pages` -run_server -if [ "$SERVER_PID" == "0" ]; then - cat $SERVER_LOG - echo -e "\n***\n*** Failed to start $SERVER\n***" - exit 1 -fi - -if [ $? -ne 0 ]; then - cat $SERVER_LOG - echo -e "\n***\n*** KIND_GPU model test failed \n***" - RET=1 -fi - -kill $SERVER_PID -wait $SERVER_PID - -current_num_pages=`get_shm_pages` -if [ $current_num_pages -ne $prev_num_pages ]; then - cat $CLIENT_LOG - ls /dev/shm - echo -e "\n***\n*** Test Failed. Shared memory pages where not cleaned properly. -Shared memory pages before starting triton equals to $prev_num_pages -and shared memory pages after starting triton equals to $current_num_pages \n***" - exit 1 -fi + rm -rf models/ + mkdir -p models/add_sub_gpu/1/ + cp ../python_models/add_sub/model.py ./models/add_sub_gpu/1/ + cp ../python_models/add_sub_gpu/config.pbtxt ./models/add_sub_gpu/ + + prev_num_pages=`get_shm_pages` + run_server + if [ "$SERVER_PID" == "0" ]; then + cat $SERVER_LOG + echo -e "\n***\n*** Failed to start $SERVER\n***" + exit 1 + fi + + if [ $? -ne 0 ]; then + cat $SERVER_LOG + echo -e "\n***\n*** KIND_GPU model test failed \n***" + RET=1 + fi + + kill $SERVER_PID + wait $SERVER_PID + + current_num_pages=`get_shm_pages` + if [ $current_num_pages -ne $prev_num_pages ]; then + cat $CLIENT_LOG + ls /dev/shm + echo -e "\n***\n*** Test Failed. Shared memory pages where not cleaned properly. + Shared memory pages before starting triton equals to $prev_num_pages + and shared memory pages after starting triton equals to $current_num_pages \n***" + exit 1 + fi fi # Test Multi file models @@ -278,7 +254,7 @@ mkdir -p models/multi_file/1/ cp ../python_models/multi_file/*.py ./models/multi_file/1/ cp ../python_models/identity_fp32/config.pbtxt ./models/multi_file/ (cd models/multi_file && \ - sed -i "s/^name:.*/name: \"multi_file\"/" config.pbtxt) + sed -i "s/^name:.*/name: \"multi_file\"/" config.pbtxt) prev_num_pages=`get_shm_pages` run_server @@ -402,6 +378,7 @@ and shared memory pages after starting triton equals to $current_num_pages \n*** exit 1 fi +# Disable env test for Jetson since cloud storage repos are not supported # Disable ensemble, io and bls tests for Jetson since GPU Tensors are not supported # Disable variants test for Jetson since already built without GPU Tensor support # Disable decoupled test because it uses GPU tensors @@ -422,6 +399,14 @@ if [ "$TEST_JETSON" == "0" ]; then deactivate rm -fr venv done + + # In 'env' test we use miniconda for dependency management. No need to run + # the test in a virtual environment. + (cd env && bash -ex test.sh) + if [ $? -ne 0 ]; then + echo "Subtest env FAILED" + RET=1 + fi fi SUBTESTS="lifecycle restart model_control examples argument_validation logging custom_metrics" @@ -442,7 +427,6 @@ for TEST in ${SUBTESTS}; do rm -fr venv done - if [ $RET -eq 0 ]; then echo -e "\n***\n*** Test Passed\n***" else