diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ef3d92ae..d76038692 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ## Online Documentation -[rocAL Documentation](https://github.com/ROCmSoftwarePlatform/rocAL) +[rocAL Documentation](https://github.com/ROCm/rocAL) ## rocAL 1.0.0 (unreleased) @@ -28,12 +28,12 @@ * Linux distribution + Ubuntu - `20.04` / `22.04` -* ROCm: rocm-core - `5.4.0.50400-72` +* ROCm: rocm-core - `6.0.60002-1` * Protobuf - [V3.12.4](https://github.com/protocolbuffers/protobuf/releases/tag/v3.12.4) * OpenCV - [4.6.0](https://github.com/opencv/opencv/releases/tag/4.6.0) -* RPP - [1.2.0](https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp/releases/tag/1.2.0) +* RPP - [1.4.0](https://github.com/ROCms/rpp/releases/tag/1.4.0) * FFMPEG - [n4.4.2](https://github.com/FFmpeg/FFmpeg/releases/tag/n4.4.2) -* MIVisionX - [master](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX) +* MIVisionX - [master](https://github.com/ROCm/MIVisionX) * Dependencies for all the above packages * rocAL Setup Script - `V1.0.2` diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f6dea8a8..93766916f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,7 +142,7 @@ install(FILES docs/README.md DESTINATION ${CMAKE_INSTALL_DATADIR}/doc/rocal) enable_testing() include(CTest) -add_subdirectory(tests/cpp_api_tests) +add_subdirectory(tests/cpp_api) # set package information set(CPACK_PACKAGE_VERSION ${VERSION}) @@ -152,7 +152,7 @@ set(CPACK_PACKAGE_CONTACT "rocAL Support ") set(CPACK_PACKAGE_VENDOR "AMD Radeon") set(CPACK_PACKAGE_ICON "${CMAKE_SOURCE_DIR}/docs/data/rocAL_logo.png") set(CPACK_PACKAGE_GROUP "Development/Tools") -set(CPACK_PACKAGE_HOMEPAGE "https://github.com/ROCmSoftwarePlatform/rocAL") +set(CPACK_PACKAGE_HOMEPAGE "https://github.com/ROCm/rocAL") set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "AMD rocAL is a comprehensive Augmentation Library The AMD ROCm Augmentation Library (rocAL) is designed to efficiently decode and process images \ diff --git a/README.md b/README.md index 21d95b031..caa9e926e 100644 --- a/README.md +++ b/README.md @@ -77,8 +77,8 @@ rocAL can be currently used to perform the following operations either with rand + RedHat - `8` / `9` + SLES - `15-SP4` * [ROCm](https://rocmdocs.amd.com/en/latest/deploy/linux/installer/install.html) with --usecase=graphics,rocm -* [AMD RPP](https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp) - MIVisionX Component -* [AMD OpenVX™](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/master/amd_openvx) and AMD OpenVX™ Extensions: `VX_RPP` and `AMD Media` - MIVisionX Components +* [AMD RPP](https://github.com/ROCm/rpp) - MIVisionX Component +* [AMD OpenVX™](https://github.com/ROCm/MIVisionX/tree/master/amd_openvx) and AMD OpenVX™ Extensions: `VX_RPP` and `AMD Media` - MIVisionX Components * [Turbo JPEG](https://libjpeg-turbo.org/) - Version 2.0.6.2 from `https://github.com/rrawther/libjpeg-turbo.git` * [Half-precision floating-point](https://half.sourceforge.net) library - Version `1.12.0` or higher * [Google Protobuf](https://developers.google.com/protocol-buffers) - Version `3.12.4` or higher @@ -131,7 +131,7 @@ For the convenience of the developer, we here provide the setup script which wil + Clone rocAL source code ``` - git clone https://github.com/ROCmSoftwarePlatform/rocAL.git + git clone https://github.com/ROCm/rocAL.git cd rocAL ``` **Note:** rocAL supports **CPU** and two **GPU** backends: **OPENCL**/**HIP**: @@ -144,6 +144,7 @@ For the convenience of the developer, we here provide the setup script which wil ``` + run the below commands to build rocAL + **Note:** Use `-DPYTHONVERSION=3.x` with `cmake` for using a specific Python3 version if required. ``` mkdir build-hip cd build-hip @@ -153,7 +154,7 @@ For the convenience of the developer, we here provide the setup script which wil sudo make install ``` - + run tests - [test option instructions](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/wiki/CTest) + + run tests - [test option instructions](https://github.com/ROCm/MIVisionX/wiki/CTest) ``` make test ``` @@ -174,13 +175,13 @@ For the convenience of the developer, we here provide the setup script which wil + RedHat - `8` / `9` + SLES - `15-SP4` * ROCm: rocm-core - `5.7.0.50700-6` -* RPP - [1.2.0](https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp/releases/tag/1.2.0) -* MIVisionX - [master](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX) +* RPP - [1.2.0](https://github.com/ROCm/rpp/releases/tag/1.2.0) +* MIVisionX - [master](https://github.com/ROCm/MIVisionX) * Protobuf - [V3.12.4](https://github.com/protocolbuffers/protobuf/releases/tag/v3.12.4) * OpenCV - [4.6.0](https://github.com/opencv/opencv/releases/tag/4.6.0) * FFMPEG - [n4.4.2](https://github.com/FFmpeg/FFmpeg/releases/tag/n4.4.2) * RapidJSON- [master](https://github.com/Tencent/rapidjson) * PyBind11 - [v2.10.4](https://github.com/pybind/pybind11) -* CuPy - [v12.2.0](https://github.com/ROCmSoftwarePlatform/cupy/releases/tag/v12.0.0) +* CuPy - [v12.2.0](https://github.com/ROCm/cupy/releases/tag/v12.0.0) * rocAL Setup Script - `V1.0.2` * Dependencies for all the above packages diff --git a/apps/README.md b/apps/README.md deleted file mode 100644 index f0bc10718..000000000 --- a/apps/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Applications - -rocAL has several applications built on top of AMD optimized libraries that can be used as prototypes or used as models to develop products. - -## Prerequisites -* [rocAL](https://github.com/ROCmSoftwarePlatform/rocAL) - -## Image Augmentation - -This sample [application](./image_augmentation#image-augmentation-application) demonstrates the basic usage of rocAL's C API to load JPEG images from the disk and modify them in different possible ways and displays the output images. - -

diff --git a/copyright.txt b/copyright.txt index c6c3df3a2..f0c87eebd 100644 --- a/copyright.txt +++ b/copyright.txt @@ -1,6 +1,6 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: rocAL -Source: https://github.com/ROCmSoftwarePlatform/rocAL.git +Source: https://github.com/ROCm/rocAL.git Files: * Copyright: 2022 - 2023 Advanced Micro Devices, Inc. diff --git a/docker/README.md b/docker/README.md index 91399d4e5..aa204cffe 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,6 +1,6 @@ # rocAL Docker -Docker is a set of platform as a service (PaaS) products that use OS-level virtualization to deliver software in packages called containers. [Read More](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/wiki/Docker) +Docker is a set of platform as a service (PaaS) products that use OS-level virtualization to deliver software in packages called containers. [Read More](https://github.com/ROCm/MIVisionX/wiki/Docker) ## Build - dockerfiles diff --git a/docker/rocal-on-rhel-09.dockerfile b/docker/rocal-on-rhel-09.dockerfile index c831c8038..7e8028255 100644 --- a/docker/rocal-on-rhel-09.dockerfile +++ b/docker/rocal-on-rhel-09.dockerfile @@ -26,12 +26,12 @@ RUN apt-get update -y && apt-get -y install autoconf automake libbz2-dev libssl- RUN apt-get -y install sqlite3 libsqlite3-dev libtool build-essential RUN git clone -b v3.21.9 https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && \ ./autogen.sh && ./configure && make -j8 && make check -j8 && sudo make install && sudo ldconfig && cd -RUN git clone -b 0.99 https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp.git && cd rpp && mkdir build && cd build && \ +RUN git clone -b 0.99 https://github.com/ROCm/rpp.git && cd rpp && mkdir build && cd build && \ cmake -DBACKEND=HIP ../ && make -j4 && sudo make install && cd ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE # Install MIVisionX -RUN git clone https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX.git && \ - mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../MIVisionX && make -j8 && make install \ No newline at end of file +RUN git clone https://github.com/ROCm/MIVisionX.git && \ + mkdir build && cd build && cmake -DBACKEND=HIP ../MIVisionX && make -j8 && make install \ No newline at end of file diff --git a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile index a8d3ab5ef..66f5319a9 100644 --- a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile +++ b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile @@ -58,14 +58,14 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ cmake ../ && make -j4 && sudo make install && cd ../../ && \ pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ - pip install numpy==1.24.2 scipy==1.9.3 cython==0.29.* git+https://github.com/ROCmSoftwarePlatform/hipify_torch.git && \ + pip install numpy==1.24.2 scipy==1.9.3 cython==0.29.* git+https://github.com/ROCm/hipify_torch.git && \ env CC=$MPI_HOME/bin/mpicc python -m pip install mpi4py && \ - git clone https://github.com/ROCmSoftwarePlatform/cupy.git && cd cupy && git submodule update --init && \ + git clone https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv # install MIVisionX RUN git clone https://github.com/ROCm/MIVisionX.git && cd MIVisionX && \ - mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../ && make -j8 && make install + mkdir build && cd build && cmake -DBACKEND=HIP ../ && make -j8 && make install ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE diff --git a/docker/rocal-on-ubuntu-20.dockerfile b/docker/rocal-on-ubuntu-20.dockerfile index 24a25ed47..e2954678d 100644 --- a/docker/rocal-on-ubuntu-20.dockerfile +++ b/docker/rocal-on-ubuntu-20.dockerfile @@ -54,14 +54,14 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ cmake ../ && make -j4 && sudo make install && cd ../../ && \ pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ - pip install numpy==1.24.2 scipy==1.9.3 cython==0.29.* git+https://github.com/ROCmSoftwarePlatform/hipify_torch.git && \ + pip install numpy==1.24.2 scipy==1.9.3 cython==0.29.* git+https://github.com/ROCm/hipify_torch.git && \ env CC=$MPI_HOME/bin/mpicc python -m pip install mpi4py && \ git clone -b rocm6.1_internal_testing https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv # install MIVisionX RUN git clone https://github.com/ROCm/MIVisionX.git && cd MIVisionX && \ - mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../ && make -j8 && make install + mkdir build && cd build && cmake -DBACKEND=HIP ../ && make -j8 && make install ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE diff --git a/docker/rocal-on-ubuntu-22.dockerfile b/docker/rocal-on-ubuntu-22.dockerfile index a28403457..2d4c97d06 100644 --- a/docker/rocal-on-ubuntu-22.dockerfile +++ b/docker/rocal-on-ubuntu-22.dockerfile @@ -57,14 +57,14 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ cmake ../ && make -j4 && sudo make install && cd ../../ && \ pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ - pip install numpy==1.24.2 scipy==1.9.3 cython==0.29.* git+https://github.com/ROCmSoftwarePlatform/hipify_torch.git && \ + pip install numpy==1.24.2 scipy==1.9.3 cython==0.29.* git+https://github.com/ROCm/hipify_torch.git && \ env CC=$MPI_HOME/bin/mpicc python -m pip install mpi4py && \ git clone -b rocm6.1_internal_testing https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv # Install MIVisionX RUN git clone https://github.com/ROCm/MIVisionX && cd MIVisionX && \ - mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../ && make -j8 && make install + mkdir build && cd build && cmake -DBACKEND=HIP ../ && make -j8 && make install ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE diff --git a/docker/rocal-with-pytorch.dockerfile b/docker/rocal-with-pytorch.dockerfile index f955e9146..547b79854 100644 --- a/docker/rocal-with-pytorch.dockerfile +++ b/docker/rocal-with-pytorch.dockerfile @@ -45,14 +45,14 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ cmake ../ && make -j4 && sudo make install && cd ../../ && \ pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ - pip install numpy==1.24.2 scipy==1.9.3 cython==0.29.* git+https://github.com/ROCmSoftwarePlatform/hipify_torch.git && \ + pip install numpy==1.24.2 scipy==1.9.3 cython==0.29.* git+https://github.com/ROCm/hipify_torch.git && \ env CC=$MPI_HOME/bin/mpicc python -m pip install mpi4py && \ git clone -b rocm6.1_internal_testing https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv # install MIVisionX RUN git clone https://github.com/ROCm/MIVisionX.git && cd MIVisionX && \ - mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../ && make -j8 && make install && cd + mkdir build && cd build && cmake -DBACKEND=HIP ../ && make -j8 && make install && cd ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE diff --git a/docker/rocal-with-tensorflow.dockerfile b/docker/rocal-with-tensorflow.dockerfile new file mode 100644 index 000000000..337367c43 --- /dev/null +++ b/docker/rocal-with-tensorflow.dockerfile @@ -0,0 +1,73 @@ +ARG TENSORFLOW_VERSION=latest +FROM rocm/tensorflow:${TENSORFLOW_VERSION} + +ENV ROCAL_DEPS_ROOT=/rocAL-deps +WORKDIR $ROCAL_DEPS_ROOT + +RUN apt-get update -y + +# install rocAL base dependencies +RUN DEBIAN_FRONTEND=noninteractive apt-get -y install gcc g++ cmake pkg-config git + +# install OpenCV +RUN DEBIAN_FRONTEND=noninteractive apt-get -y install build-essential libgtk2.0-dev libavcodec-dev libavformat-dev libswscale-dev python-dev python-numpy \ + libtbb2 libtbb-dev libjpeg-dev libpng-dev libtiff-dev libdc1394-22-dev unzip && \ + mkdir OpenCV && cd OpenCV && wget https://github.com/opencv/opencv/archive/4.6.0.zip && unzip 4.6.0.zip && \ + mkdir build && cd build && cmake -DWITH_GTK=ON -DWITH_JPEG=ON -DBUILD_JPEG=ON -DWITH_OPENCL=OFF ../opencv-4.6.0 && make -j8 && sudo make install && sudo ldconfig && cd + +# install FFMPEG +ENV PKG_CONFIG_PATH="/usr/local/lib/pkgconfig/" +RUN DEBIAN_FRONTEND=noninteractive apt-get -y install autoconf automake build-essential cmake git-core libass-dev libfreetype6-dev libsdl2-dev libtool libva-dev \ + libvdpau-dev libvorbis-dev libxcb1-dev libxcb-shm0-dev libxcb-xfixes0-dev pkg-config texinfo wget zlib1g-dev \ + nasm yasm libx264-dev libx265-dev libnuma-dev libfdk-aac-dev && \ + wget https://github.com/FFmpeg/FFmpeg/archive/refs/tags/n4.4.2.zip && unzip n4.4.2.zip && cd FFmpeg-n4.4.2/ && sudo ldconfig && \ + ./configure --enable-shared --disable-static --enable-libx264 --enable-libx265 --enable-libfdk-aac --enable-libass --enable-gpl --enable-nonfree && \ + make -j8 && sudo make install && cd + +# install rocAL neural net dependency +RUN DEBIAN_FRONTEND=noninteractive apt-get -y install rocblas rocblas-dev miopen-hip miopen-hip-dev migraphx && \ + mkdir neuralNet && cd neuralNet && wget https://sourceforge.net/projects/half/files/half/1.12.0/half-1.12.0.zip && \ + unzip half-1.12.0.zip -d half-files && sudo mkdir -p /usr/local/include/half && sudo cp half-files/include/half.hpp /usr/local/include/half && cd + +# install rocAL dependency +RUN DEBIAN_FRONTEND=noninteractive apt-get -y install wget libbz2-dev libssl-dev python-dev python3-dev libgflags-dev libgoogle-glog-dev liblmdb-dev nasm yasm libjsoncpp-dev clang && \ + git clone -b 3.0.1 https://github.com/libjpeg-turbo/libjpeg-turbo.git && cd libjpeg-turbo && mkdir build && cd build && \ + cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_STATIC=FALSE -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib -DWITH_JPEG8=TRUE ../ && \ + make -j4 && sudo make install && cd && \ + git clone https://github.com/ROCm/rpp.git && cd rpp && mkdir build && cd build && \ + cmake -DBACKEND=HIP ../ && make -j4 && sudo make install && cd ../../ && \ + git clone -b v3.12.4 https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && \ + ./autogen.sh && ./configure && make -j8 && make check -j8 && sudo make install && sudo ldconfig && cd +ENV CUPY_INSTALL_USE_HIP=1 +ENV ROCM_HOME=/opt/rocm +RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g++ hipblas hipsparse rocrand hipfft rocfft rocthrust-dev hipcub-dev python3-dev && \ + git clone https://github.com/Tencent/rapidjson.git && cd rapidjson && mkdir build && cd build && \ + cmake ../ && make -j4 && sudo make install && cd ../../ && \ + pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ + cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ + pip install numpy==1.24.2 scipy==1.9.3 cython==0.29.* git+https://github.com/ROCm/hipify_torch.git && \ + env CC=$MPI_HOME/bin/mpicc python -m pip install mpi4py && \ + git clone -b rocm6.1_internal_testing https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ + pip install -e . --no-cache-dir -vvvv + +# install MIVisionX +RUN git clone https://github.com/ROCm/MIVisionX.git && cd MIVisionX && \ + mkdir build && cd build && cmake -DBACKEND=HIP ../ && make -j8 && make install && cd + +ENV ROCAL_WORKSPACE=/workspace +WORKDIR $ROCAL_WORKSPACE + +# Install rocAL +RUN git clone -b develop https://github.com/ROCm/rocAL && \ + mkdir build && cd build && cmake -DPYTHONVERSION=3.9 ../rocAL && make -j8 && cmake --build . --target PyPackageInstall && make install + +WORKDIR /workspace + +#install tensorflow dependencies - Compile protos and Install TensorFlow Object Detection API. +RUN cd ~/models/research/ && \ +protoc object_detection/protos/*.proto --python_out=. && \ +cp object_detection/packages/tf2/setup.py . && \ +python -m pip install . + +#upgrade pip +RUN pip3 install --upgrade pip \ No newline at end of file diff --git a/docs/README.md b/docs/README.md index ede6f142f..7fa4f21bb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -24,11 +24,11 @@ These pipelines are programmable by the user using both C++ and Python APIs. ## Prerequisites -Refer [rocAL Prerequisites](https://github.com/ROCmSoftwarePlatform/rocAL#prerequisites) +Refer [rocAL Prerequisites](https://github.com/ROCm/rocAL#prerequisites) ## Build instructions -Refer [rocAL build instructions](https://github.com/ROCmSoftwarePlatform/rocAL#build-instructions) +Refer [rocAL build instructions](https://github.com/ROCm/rocAL#build-instructions) ## rocAL Python @@ -81,12 +81,11 @@ amd.rocal.types are enums exported from C++ API to python. Some examples include * From the above classes, any hybrid iterator pipeline can be created by adding augmentations. * see example [PyTorch Simple Example](./examples). Requires PyTorch. -### installing rocAL python plugin (Python 3.6) +### installing rocAL python plugin (Python 3.9+) * Build and install RPP -* Build and install MIVisionX which installs rocAL c++ lib -* Go to the [rocal_pybind](https://github.com/ROCm/rocAL/tree/develop/rocAL_pybind) folder -* sudo ./run.sh +* Build and install MIVisionX +* Build and install [rocAL](https://github.com/ROCm/rocAL/) ### Steps to run MLPerf Resnet50 classification training with rocAL on a system with MI50+ and ROCm @@ -110,7 +109,7 @@ git clone -b mlperf-v1.1-rocal https://github.com/rrawther/MLPerf-mGPU ### MIVisionX Pytorch Docker -* Refer to the [docker](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX#docker) page for prerequisites and information on building the docker +* Refer to the [docker](https://github.com/ROCm/MIVisionX#docker) page for prerequisites and information on building the docker * Step 1: Run the docker image* ``` bash diff --git a/docs/data/amd-epyc-black-resize.png b/docs/data/amd-epyc-black-resize.png new file mode 100644 index 000000000..a79b38ca9 Binary files /dev/null and b/docs/data/amd-epyc-black-resize.png differ diff --git a/docs/data/rocm-black-resize.png b/docs/data/rocm-black-resize.png new file mode 100644 index 000000000..34829a6ad Binary files /dev/null and b/docs/data/rocm-black-resize.png differ diff --git a/docs/examples.md b/docs/examples.md index 065c48b28..7e9089c38 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -2,6 +2,6 @@ Use the links below to see more examples: -- [Image Processing](https://github.com/ROCmSoftwarePlatform/rocAL/tree/master/docs/examples/image_processing) +- [Image Processing](https://github.com/ROCm/rocAL/tree/master/docs/examples/image_processing) -- [Pytorch](https://github.com/ROCmSoftwarePlatform/rocAL/tree/master/docs/examples/pytorch) +- [Pytorch](https://github.com/ROCm/rocAL/tree/master/docs/examples/pytorch) diff --git a/docs/examples.rst b/docs/examples.rst index 1baa5e5bf..10607f2a3 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -12,4 +12,6 @@ Use the links below to see more examples: * `Image Processing `_ * `Pytorch `_ +* `Tensorflow `_ +* `Jupyter Notebooks `_ diff --git a/rocAL_pybind/examples/rocAL_example_doc/Augmentation_examples.ipynb b/docs/examples/notebooks/augmentation_examples.ipynb similarity index 100% rename from rocAL_pybind/examples/rocAL_example_doc/Augmentation_examples.ipynb rename to docs/examples/notebooks/augmentation_examples.ipynb diff --git a/rocAL_pybind/examples/rocAL_example_doc/Classification_training_flowerdataset.ipynb b/docs/examples/notebooks/classification_training_flowerdataset.ipynb similarity index 100% rename from rocAL_pybind/examples/rocAL_example_doc/Classification_training_flowerdataset.ipynb rename to docs/examples/notebooks/classification_training_flowerdataset.ipynb diff --git a/rocAL_pybind/examples/rocAL_example_doc/create_classification_flower_dataset.py b/docs/examples/notebooks/create_classification_flower_dataset.py similarity index 100% rename from rocAL_pybind/examples/rocAL_example_doc/create_classification_flower_dataset.py rename to docs/examples/notebooks/create_classification_flower_dataset.py diff --git a/docs/examples/image_processing/decoder_examples.ipynb b/docs/examples/notebooks/decoder_examples.ipynb similarity index 99% rename from docs/examples/image_processing/decoder_examples.ipynb rename to docs/examples/notebooks/decoder_examples.ipynb index cb1bef27e..1545f0314 100644 --- a/docs/examples/image_processing/decoder_examples.ipynb +++ b/docs/examples/notebooks/decoder_examples.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ diff --git a/rocAL_pybind/examples/rocAL_example_doc/resize_implementation.ipynb b/docs/examples/notebooks/resize_implementation.ipynb similarity index 100% rename from rocAL_pybind/examples/rocAL_example_doc/resize_implementation.ipynb rename to docs/examples/notebooks/resize_implementation.ipynb diff --git a/rocAL_pybind/examples/rocAL_example_doc/tf_dataloader.ipynb b/docs/examples/notebooks/tf_dataloader.ipynb similarity index 98% rename from rocAL_pybind/examples/rocAL_example_doc/tf_dataloader.ipynb rename to docs/examples/notebooks/tf_dataloader.ipynb index dfdf74c30..5305cec9a 100644 --- a/rocAL_pybind/examples/rocAL_example_doc/tf_dataloader.ipynb +++ b/docs/examples/notebooks/tf_dataloader.ipynb @@ -63,7 +63,7 @@ " \"image/class/label\": \"image/class/label\",\n", " \"image/filename\": \"image/filename\",\n", "}\n", - "path = \"OUTPUT_FOLDER/TF_READER/CLASSIFICATION/\"" + "path = \"output_folder/tf_reader/classification/\"" ] }, { @@ -171,7 +171,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.16" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/rocAL_pybind/examples/rocAL_example_doc/Video_decoder.ipynb b/docs/examples/notebooks/video_decoder.ipynb similarity index 100% rename from rocAL_pybind/examples/rocAL_example_doc/Video_decoder.ipynb rename to docs/examples/notebooks/video_decoder.ipynb diff --git a/rocAL_pybind/examples/rocAL_example_doc/VideoReader_with_label.ipynb b/docs/examples/notebooks/video_reader_with_label.ipynb similarity index 100% rename from rocAL_pybind/examples/rocAL_example_doc/VideoReader_with_label.ipynb rename to docs/examples/notebooks/video_reader_with_label.ipynb diff --git a/docs/examples/pytorch/README.md b/docs/examples/pytorch/README.md deleted file mode 100644 index 9ae0ded87..000000000 --- a/docs/examples/pytorch/README.md +++ /dev/null @@ -1,10 +0,0 @@ -* This example shows how to run training using pytorch and ToyNet with 2 classes -* Use a dataset with 2 classes - -To run the sample: -* Install rocal_pybind - -``` -python3 test_training.py -``` -* rocal device can be cpu/gpu. diff --git a/docs/examples/pytorch/toynet_training/README.md b/docs/examples/pytorch/toynet_training/README.md new file mode 100644 index 000000000..0f0c63c4a --- /dev/null +++ b/docs/examples/pytorch/toynet_training/README.md @@ -0,0 +1,15 @@ +* This example shows how to run training using pytorch and ToyNet with 2 classes +* Use a dataset with 2 classes + +### Building the required Pytorch Rocm docker +* Use the instructions in the [docker section](https://github.com/ROCm/rocAL/docker) to build the required [Pytorch docker](https://github.com/ROCm/rocAL/docker/rocal-with-pytorch.dockerfile) +* Upgrade pip to the latest version. +* Run requirements.sh to install the required packages. + +### To run the sample: +* Install rocal_pybind + +``` +python3 train.py +``` +* rocal device can be cpu/gpu. diff --git a/docs/examples/pytorch/test_training.py b/docs/examples/pytorch/toynet_training/train.py similarity index 100% rename from docs/examples/pytorch/test_training.py rename to docs/examples/pytorch/toynet_training/train.py diff --git a/docs/examples/tf/pets_training/README.md b/docs/examples/tf/pets_training/README.md new file mode 100644 index 000000000..f3fc184b3 --- /dev/null +++ b/docs/examples/tf/pets_training/README.md @@ -0,0 +1,15 @@ +## Running Pets Training Example + +### Building the required TF Rocm docker +* Use the instructions in the [docker section](https://github.com/ROCm/rocAL/docker) to build the required [Tensorflow docker](https://github.com/ROCm/rocAL/docker/rocal-with-tensorflow.dockerfile) +* Upgrade pip to the latest version. + +### Running the training + +* For first run, to setup dataset, edit "train.py" and set "DATASET_DOWNLOAD_AND_PREPROCESS = True" +* For subsequent runs, after the dataset has already been downloaded and preprocessed, set "DATASET_DOWNLOAD_AND_PREPROCESS = False" + +To run this example for the first run or subsequent runs, just execute: +``` +python3 train.py +``` diff --git a/rocAL_pybind/examples/tf_petsTrainingExample/download_and_preprocess_dataset.sh b/docs/examples/tf/pets_training/download_and_preprocess_dataset.sh similarity index 100% rename from rocAL_pybind/examples/tf_petsTrainingExample/download_and_preprocess_dataset.sh rename to docs/examples/tf/pets_training/download_and_preprocess_dataset.sh diff --git a/rocAL_pybind/examples/tf_petsTrainingExample/requirements.sh b/docs/examples/tf/pets_training/requirements.sh similarity index 100% rename from rocAL_pybind/examples/tf_petsTrainingExample/requirements.sh rename to docs/examples/tf/pets_training/requirements.sh diff --git a/rocAL_pybind/examples/tf_petsTrainingExample/train_withROCAL_withTFRecordReader.py b/docs/examples/tf/pets_training/train.py similarity index 100% rename from rocAL_pybind/examples/tf_petsTrainingExample/train_withROCAL_withTFRecordReader.py rename to docs/examples/tf/pets_training/train.py diff --git a/docs/how-to/framework.rst b/docs/how-to/framework.rst index 9dc4923d8..eafc2eb34 100644 --- a/docs/how-to/framework.rst +++ b/docs/how-to/framework.rst @@ -27,7 +27,7 @@ Create Data-loading Pipeline Follow these steps: -1. Import libraries for `rocAL `_. +1. Import libraries for `rocAL `_. .. code-block:: python :caption: Import libraries @@ -38,7 +38,7 @@ Follow these steps: import amd.rocal.types as types -2. See a rocAL pipeline for PyTorch below. It reads data from the dataset using a fileReader and uses image_slice to decode the raw images. The other required augmentation operations are also defined in the `pipeline `_. +2. See a rocAL pipeline for PyTorch below. It reads data from the dataset using a fileReader and uses image_slice to decode the raw images. The other required augmentation operations are also defined in the `pipeline `_. .. code-block:: python :caption: Pipeline for PyTorch @@ -80,7 +80,7 @@ Follow these steps: import torch.optim as optim -4. Call the training pipeline with rocAL classification data `loader `_. +4. Call the training pipeline with rocAL classification data `loader `_. .. code-block:: python :caption: Call the training pipeline @@ -93,7 +93,7 @@ Follow these steps: train_loader = ROCALClassificationIterator(pipe_train, device=”cpu” if self.rocal_cpu else “cuda”, device_id = self.local_rank) -5. Run the `training script `_. +5. Run the `training script `_. .. code-block:: python :caption: Run the training pipeline @@ -130,7 +130,7 @@ Create Data-loading Pipeline Follow these steps: -1. Import libraries for `rocAL_pybind `_. +1. Import libraries for `rocAL_pybind `_. .. code-block:: python :caption: Import libraries @@ -141,7 +141,7 @@ Follow these steps: import amd.rocal.types as types -2. See a rocAL pipeline for TensorFlow below. It reads data from the TFRecords using TFRecord Reader and uses ``fn.decoders.image`` to decode the raw `images `_. +2. See a rocAL pipeline for TensorFlow below. It reads data from the TFRecords using TFRecord Reader and uses ``fn.decoders.image`` to decode the raw `images `_. .. code-block:: python :caption: Pipeline for TensorFlow @@ -170,7 +170,7 @@ Follow these steps: trainPipe.build() -3. Import libraries for `TensorFlow `_. +3. Import libraries for `TensorFlow `_. .. code-block:: python :caption: Import libraries for TensorFlow @@ -191,16 +191,16 @@ Follow these steps: train_label_one_hot_list = get_label_one_hot(train_label_ndArray) -4. To see and run a sample training script, refer to `rocAL TensorFlow example `_. +4. To see and run a sample training script, refer to `rocAL TensorFlow example `_. -.. _ml-perf: +.. __resnet50: -Run MLPerf Resnet50 classification training with rocAL +Run Resnet50 classification training with rocAL ======================================================= #. Ensure you have downloaded ``ILSVRC2012_img_val.tar`` (6.3GB) and ``ILSVRC2012_img_train.tar`` (138 GB) files and unzip into ``train`` and ``val`` folders -#. Build `MIVisionX Pytorch docker `_ +#. Build `rocAL Pytorch docker `_ * Run the docker image @@ -209,26 +209,13 @@ Run MLPerf Resnet50 classification training with rocAL sudo docker run -it -v :/data -v /:/dockerx -w /dockerx --privileged --device=/dev/kfd --device=/dev/dri --group-add video --shm-size=4g --ipc="host" --network=host .. note:: - Refer to the `docker `_ page for prerequisites and information on building the docker image. + Refer to the `docker `_ page for prerequisites and information on building the docker image. Optional: Map localhost directory on the docker image * Option to map the localhost directory with imagenet dataset folder to be accessed on the docker image. * Usage: ``-v {LOCAL_HOST_DIRECTORY_PATH}:{DOCKER_DIRECTORY_PATH}`` -#. Install rocAL ``python_pybind`` plugin as described above -#. Clone `MLPerf `_ repo and checkout ``mlperf-v1.1-rocal`` branch +#. To see and run a sample training script, refer to `rocAL Imagenet example `_. -.. code-block:: shell - - git clone -b mlperf-v1.1-rocal https://github.com/rrawther/MLPerf-mGPU - -#. Modify ``RN50_AMP_LARS_8GPUS_NCHW.sh`` or ``RN50_AMP_LARS_8GPUS_NHWC.sh`` to reflect correct path for imagenet directory -#. Run appropriate script as needed: - -.. code-block:: shell - - ./RN50_AMP_LARS_8GPUS_NCHW.sh - (or) - ./RN50_AMP_LARS_8GPUS_NHWC.sh diff --git a/docs/how-to/overview.rst b/docs/how-to/overview.rst index fc3d674ad..b134f0dde 100644 --- a/docs/how-to/overview.rst +++ b/docs/how-to/overview.rst @@ -109,4 +109,4 @@ Decoders Description ====================== ======================================== To see examples demonstrating the usage of decoders and readers, see -`MIVisionX rocAL Python Binding Examples `_. +`rocAL Python Examples `_. diff --git a/docs/how-to/using-with-cpp.rst b/docs/how-to/using-with-cpp.rst index 50704c60b..cb3a02188 100644 --- a/docs/how-to/using-with-cpp.rst +++ b/docs/how-to/using-with-cpp.rst @@ -92,7 +92,7 @@ See `rocalRelease example `_. +The example below shows how to create a pipeline, read JPEG images, perform certain augmentations on them, and show the output using OpenCV by utilizing `C++ API `_. .. code-block:: cpp :caption: Example Image Augmentation @@ -131,4 +131,4 @@ The example below shows how to create a pipeline, read JPEG images, perform cert } -To see a sample image augmentation application in C++, see `Image Augmentation `_. +To see a sample image augmentation application in C++, see `Image Augmentation `_. diff --git a/docs/how-to/using-with-python.rst b/docs/how-to/using-with-python.rst index e4b7abd33..d5257ac69 100644 --- a/docs/how-to/using-with-python.rst +++ b/docs/how-to/using-with-python.rst @@ -95,7 +95,7 @@ Given below is an example of a file reader, which takes a folder of images as in ## 4.1.2 Defining the Pipeline -To define a pipeline, see https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/rocAL_pybind/amd/rocal/pipeline.py#L29. +To define a pipeline, see https://github.com/ROCm/rocAL/blob/master/rocAL_pybind/amd/rocal/pipeline.py#L29. ``` class Pipeline(object): @@ -188,11 +188,7 @@ Building the Pipeline Building the pipeline ensures that all operators are validated with the corresponding inputs and outputs. -<<<<<<< HEAD:docs/user_guide/ch4.md -To build the pipeline, see https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/rocAL_pybind/examples/rocAL_api_python_unittest.py#L166 -======= -To build the pipeline, see `https://github.com/ROCm/rocAL/blob/master/rocAL_pybind/examples/rocAL_api_python_unittest.py#L166` ->>>>>>> 80a9d60 (Documentation - reorg for diataxis (#102)):docs/how-to/using-with-python.rst +To build the pipeline, see `https://github.com/ROCm/rocAL/blob/master/tests/python_api/unit_test.py#L166` .. code-block:: python :caption: Build the Pipeline @@ -207,14 +203,10 @@ Running the Pipeline To run/use the pipeline, simply create a data loader using the pipeline and iterate through it to get the next batch of images with labels. -<<<<<<< HEAD:docs/user_guide/ch4.md -To run the pipeline, see https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/rocAL_pybind/examples/rocAL_api_python_unittest.py#L168 -======= -To run the pipeline, see `https://github.com/ROCm/rocAL/blob/master/rocAL_pybind/examples/rocAL_api_python_unittest.py#L168` +To run the pipeline, see `https://github.com/ROCm/rocAL/blob/master/tests/python_api/unit_test.py#L168` .. code-block:: python :caption: Run the Pipeline ->>>>>>> 80a9d60 (Documentation - reorg for diataxis (#102)):docs/how-to/using-with-python.rst # Dataloader data_loader = ROCALClassificationIterator(pipe,device=device) @@ -239,11 +231,7 @@ Performing Augmentations rocAL not only reads images from the disk and batches them into tensors, it can also perform various augmentations on those images. -<<<<<<< HEAD:docs/user_guide/ch4.md -To read images, decode them, and rotate them in the pipeline, see https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/rocAL_pybind/examples/rocAL_api_python_unittest.py#L77 -======= -To read images, decode them, and rotate them in the pipeline, see `https://github.com/ROCm/rocAL/blob/master/rocAL_pybind/examples/rocAL_api_python_unittest.py#L77` ->>>>>>> 80a9d60 (Documentation - reorg for diataxis (#102)):docs/how-to/using-with-python.rst +To read images, decode them, and rotate them in the pipeline, see `https://github.com/ROCm/rocAL/blob/master/tests/python_api/unit_test.py#L77` .. code-block:: python :caption: Perform Augmentations @@ -269,26 +257,18 @@ To run the pipeline, see: images, labels = pipe_out show_images(images) -<<<<<<< HEAD:docs/user_guide/ch4.md -All the rocAL data types are defined under [amd.rocal.types](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/blob/master/rocAL/rocAL_pybind/amd/rocal/types.py). Import this library in the application to access the various data types such as rocAL status, processing mode, tensor output type, image size evaluation policy, image color, tensor layout, decode device, resize scaling mode, and resize interpolation type. -======= rocAL Data Types ========================= -All the rocAL data types are defined under `amd.rocal.types `_. Import this library in the application to access the various data types such as rocAL status, processing mode, tensor output type, image size evaluation policy, image color, tensor layout, decode device, resize scaling mode, and resize interpolation type. ->>>>>>> 80a9d60 (Documentation - reorg for diataxis (#102)):docs/how-to/using-with-python.rst +All the rocAL data types are defined under `amd.rocal.types `_. Import this library in the application to access the various data types such as rocAL status, processing mode, tensor output type, image size evaluation policy, image color, tensor layout, decode device, resize scaling mode, and resize interpolation type. Here are some of the commonly used rocAL data types: * Processing modes: Values (GPU/CPU). Use the rocal_cpu argument in the pipeline to set the processing mode. -<<<<<<< HEAD:docs/user_guide/ch4.md -To see the usage of the above-mentioned data types, see https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/rocAL_pybind/amd/rocal/pipeline.py#L97 -======= * rocal_cpu = True: This performs data loading on the CPU. If GPUs are heavily used for training, it is viable to create the data-loading pipeline using CPU. * rocal_cpu = False: This performs data loading on the available GPU as specified using the device_id argument in the pipeline. ->>>>>>> 80a9d60 (Documentation - reorg for diataxis (#102)):docs/how-to/using-with-python.rst * Tensor output types: Values (NCHW/NHWC). Example: diff --git a/docs/install/install.rst b/docs/install/install.rst index 7e6dca28d..b9fe216f4 100644 --- a/docs/install/install.rst +++ b/docs/install/install.rst @@ -24,7 +24,7 @@ Prerequisites * Install ROCm with `amdgpu-install `_ with ``--usecase=graphics,rocm --no-32`` * `RPP `_ * `AMD OpenVX™ `_ and AMD OpenVX™ Extensions: ``VX_RPP`` and ``AMD Media`` - MIVisionX Components -* `Turbo JPEG `_ - Version 2.0.6.2 from ``https://github.com/rrawther/libjpeg-turbo.git`` +* `Turbo JPEG `_ - Version 3.0.1 from ``https://github.com/libjpeg-turbo/libjpeg-turbo.git`` * `Half-precision floating-point `_ library - Version 1.12.0 or higher * `Google Protobuf `_ - Version 3.12.4 or higher * `LMBD Library `_ diff --git a/docs/user_guide/ch1.md b/docs/user_guide/ch1.md index 97f08d2c7..422620071 100644 --- a/docs/user_guide/ch1.md +++ b/docs/user_guide/ch1.md @@ -80,4 +80,4 @@ rocAL operators offer the flexibility to run on CPU or GPU for building hybrid p | Image_random_crop | Decodes and randomly crops JPEG images | | Image_slice | Decodes and slices JPEG images | -To see examples demonstrating the usage of decoders and readers, [click here](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/master/rocAL/rocAL_pybind/examples) +To see examples demonstrating the usage of decoders and readers, [click here](https://github.com/ROCm/rocAL/tree/master/rocAL/docs/examples) diff --git a/docs/user_guide/ch2.md b/docs/user_guide/ch2.md index e4aa6367e..87aa1e19f 100644 --- a/docs/user_guide/ch2.md +++ b/docs/user_guide/ch2.md @@ -18,4 +18,4 @@ Figure 2. rocAL Master-Graph Architecture RPP is a comprehensive high-performance computer vision library optimized for the AMD CPU and GPU with HIP and OpenCL backends. It is available under the AMD ROCm software platform. It provides low-level functionality for all rocAL operators for single, image, and tensor datatypes. RPP provides an extensive library for vision augmentations that includes vision functions, color augmentations, filter augmentations, geometric distortions, and a few more features. -For more information on RPP along with the list of supported kernels, see https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp. +For more information on RPP along with the list of supported kernels, see https://github.com/ROCm/rpp. diff --git a/docs/user_guide/ch3.md b/docs/user_guide/ch3.md index aa64b032a..ad408c859 100644 --- a/docs/user_guide/ch3.md +++ b/docs/user_guide/ch3.md @@ -5,9 +5,9 @@ This chapter provides information about the installation of rocAL and related pa ## 3.1 Prerequisites * Linux distribution -* [AMD RPP](https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp) -* [AMD OpenVX™](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/master/amd_openvx) and AMD OpenVX™ Extensions: `VX_RPP` and `AMD Media` -* [Turbo JPEG](https://libjpeg-turbo.org/) - Version `2.0` or higher +* [AMD RPP](https://github.com/ROCm/rpp) +* [AMD OpenVX™](https://github.com/ROCm/rocAL/tree/master/amd_openvx) and AMD OpenVX™ Extensions: `VX_RPP` and `AMD Media` +* [Turbo JPEG](https://libjpeg-turbo.org/) - Version `3.0` or higher * [Half-precision floating-point](https://half.sourceforge.net) library - Version `1.12.0` or higher * [Google Protobuf](https://developers.google.com/protocol-buffers) - Version `3.12.4` or higher * [LMBD Library](http://www.lmdb.tech/doc/) @@ -20,23 +20,23 @@ To see the list of supported platforms for rocAL, see the ROCm Installation Guid ## 3.3 Installing rocAL -rocAL is shipped along with MIVisionX. To build and install the rocAL C++ library, follow the instructions given [here](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX#build--install-mivisionx) +To build and install the rocAL library, follow the instructions given [here](https://github.com/ROCm/rocAL#build-instructions) ## 3.4 Installing rocAL Python Package The rocAL Python package (rocal_pybind) is a separate redistributable wheel. rocal_pybind, which is created using Pybind11, enables data transfer between rocAL C++ API and Python API. With the help of rocal_pybind.so wrapper library, the rocAL functionality, which is primarily in C/C++, can be effectively used in Python. The Python package supports PyTorch, TensorFlow, Caffe2, and data readers available for various formats such as FileReader, COCO Reader, TFRecord Reader, and CaffeReader. -To build and install the Python package, see [rocAL python](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/master/rocAL/rocAL_pybind). +To build and install the Python package, install the PyPackageInstall instruction [here](https://github.com/ROCm/rocAL#build-instructions) ## 3.5 Installing rocAL Using Framework Dockers To test the rocAL Python APIs using PyTorch or TensorFlow, we recommend building a docker with rocAL and ROCm using any of the links below: -- [rocAL PyTorch docker](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/master/docker/pytorch) -- [rocAL TensorFlow docker](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/master/docker/tensorflow) +- [rocAL PyTorch docker](https://github.com/ROCm/rocAL/tree/master/docker/rocal-with-pytorch.dockerfile) +- [rocAL TensorFlow docker](https://github.com/ROCm/rocAL/tree/master/docker/rocal-with-tensorflow.dockerfile) To use rocAL on Ubuntu, use the following dockers: -- [rocAL on ubuntu20](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/blob/master/docker/mivisionx-on-ubuntu20.dockerfile) -- [rocAL on Ubuntu22](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/blob/master/docker/mivisionx-on-ubuntu22.dockerfile) +- [rocAL on ubuntu20](https://github.com/ROCm/rocAL/blob/master/docker/rocAL-on-ubuntu20.dockerfile) +- [rocAL on Ubuntu22](https://github.com/ROCm/rocAL/blob/master/docker/rocAL-on-ubuntu22.dockerfile) diff --git a/docs/user_guide/ch5.md b/docs/user_guide/ch5.md index ebdf3845a..144132a03 100644 --- a/docs/user_guide/ch5.md +++ b/docs/user_guide/ch5.md @@ -14,7 +14,7 @@ Build a rocAL PyTorch docker by following the steps here. Follow these steps: -1. Import libraries for [rocAL](https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/docs/examples/pytorch/test_training.py#L28). +1. Import libraries for [rocAL](https://github.com/ROCm/rocAL/blob/master/docs/examples/pytorch/toynet_training/train.py#L28). ``` from amd.rocal.plugin.pytorch import ROCALClassificationIterator @@ -23,7 +23,7 @@ import amd.rocal.fn as fn import amd.rocal.types as types ``` -2. See a rocAL pipeline for PyTorch below. It reads data from the dataset using a fileReader and uses image_slice to decode the raw images. The other required augmentation operations are also defined in the [pipeline](https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/docs/examples/pytorch/test_training.py#L38). +2. See a rocAL pipeline for PyTorch below. It reads data from the dataset using a fileReader and uses image_slice to decode the raw images. The other required augmentation operations are also defined in the [pipeline](https://github.com/ROCm/rocAL/blob/master/docs/examples/pytorch/toynet_training/train.py#L38). ``` def trainPipeline(data_path, batch_size, num_classes, one_hot, local_rank, world_size, num_thread, crop, rocal_cpu, fp16): @@ -61,7 +61,7 @@ import torch.nn.functional as F import torch.optim as optim ``` -4. Call the training pipeline with rocAL classification data [loader](https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/docs/examples/pytorch/test_training.py#L78). +4. Call the training pipeline with rocAL classification data [loader](https://github.com/ROCm/rocAL/blob/master/docs/examples/pytorch/toynet_training/train.py#L78). ``` Def get_pytorch_train_loader(self): @@ -72,7 +72,7 @@ Def get_pytorch_train_loader(self): train_loader = ROCALClassificationIterator(pipe_train, device=”cpu” if self.rocal_cpu else “cuda”, device_id = self.local_rank) ``` -5. Run the [training](https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/docs/examples/pytorch/test_training.py#L179). +5. Run the [training](https://github.com/ROCm/rocAL/blob/master/docs/examples/pytorch/toynet_training/train.py#L179). ``` # Training loop @@ -88,9 +88,9 @@ Def get_pytorch_train_loader(self): inputs, labels = inputs.to(device), labels.to(device) ``` -6. Run the training as shown [here](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/develop/rocAL/docs/examples/pytorch). +6. Run the training as shown [here](https://github.com/ROCm/rocAL/tree/develop/rocAL/docs/examples/pytorch/toynet_training). -To see a sample training script, click [here](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/develop/rocAL/docs/examples/pytorch). +To see a sample training script, click [here](https://github.com/ROCm/rocAL/tree/develop/rocAL/docs/examples/pytorch/toynet_training). ## 5.2 TensorFlow Integration @@ -104,7 +104,7 @@ Build a rocAL TensorFlow docker by following the steps here. Follow these steps: -1. Import libraries for [rocAL](https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/rocAL_pybind/examples/tf_petsTrainingExample/train_withROCAL_withTFRecordReader.py#L22). +1. Import libraries for [rocAL](https://github.com/ROCm/rocAL/blob/master/docs/examples/tf/pets_training/train.py#L22). ``` from amd.rocal.plugin.tf import ROCALIterator @@ -113,7 +113,7 @@ import amd.rocal.fn as fn import amd.rocal.types as types ``` -2. See a rocAL pipeline for TensorFlow below. It reads data from the TFRecords using TFRecord Reader and uses fn.decoders.image to decode the raw [images](https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/rocAL_pybind/examples/tf_petsTrainingExample/train_withROCAL_withTFRecordReader.py#L128). +2. See a rocAL pipeline for TensorFlow below. It reads data from the TFRecords using TFRecord Reader and uses fn.decoders.image to decode the raw [images](https://github.com/ROCm/rocAL/blob/master/examples/tf/pets_training/train.py#L128). ``` trainPipe = Pipeline(batch_size=TRAIN_BATCH_SIZE, num_threads=1, rocal_cpu=RUN_ON_HOST, tensor_layout = types.NHWC) @@ -140,7 +140,7 @@ trainPipe = Pipeline(batch_size=TRAIN_BATCH_SIZE, num_threads=1, rocal_cpu=RUN_O trainPipe.build() ``` -3. Import libraries for [TensorFlow](https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/rocAL_pybind/examples/tf_petsTrainingExample/train_withROCAL_withTFRecordReader.py#L174). +3. Import libraries for [TensorFlow](https://github.com/ROCm/rocAL/blob/master/examples/tf/pets_training/train.py#L174). ``` import tensorflow.compat.v1 as tf @@ -159,6 +159,6 @@ Run the training Session train_label_one_hot_list = get_label_one_hot(train_label_ndArray) ``` -4. Run the training as shown [here](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/master/rocAL/rocAL_pybind/examples/tf_petsTrainingExample). +4. Run the training as shown [here](https://github.com/ROCm/rocAL/tree/master/rocAL/examples/tf/pets_training/). -To see a sample training script, click [here](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/master/rocAL/rocAL_pybind/examples/tf_petsTrainingExample). +To see a sample training script, click [here](https://github.com/ROCm/rocAL/tree/master/rocAL/examples/tf/pets_training/). diff --git a/docs/user_guide/ch6.md b/docs/user_guide/ch6.md index f3f481945..570aef275 100644 --- a/docs/user_guide/ch6.md +++ b/docs/user_guide/ch6.md @@ -14,21 +14,21 @@ Returns: The context for the pipeline Arguments: -- RocalProcessMode: Defines whether rocal data loading should be on the CPU or [GPU](https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/rocAL/include/api/rocal_api_types.h#L91) +- RocalProcessMode: Defines whether rocal data loading should be on the CPU or [GPU](https://github.com/ROCm/rocAL/blob/master/rocAL/include/api/rocal_api_types.h#L91) ``` RocalProcessMode:: ROCAL_PROCESS_GPU RocalProcessMode::ROCAL_PROCESS_CPU ``` -- RocalTensorOutputType: Defines whether the output of rocal tensor is FP32 or [FP16](https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/rocAL/include/api/rocal_api_types.h#L124) +- RocalTensorOutputType: Defines whether the output of rocal tensor is FP32 or [FP16](https://github.com/ROCm/rocAL/blob/master/rocAL/include/api/rocal_api_types.h#L124) ``` RocalTensorOutputType::ROCAL_FP32 RocalTensorOutputType::ROCAL_FP16 ``` -[Example](https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/rocAL/include/api/rocal_api.h#L41): +[Example](https://github.com/ROCm/rocAL/blob/master/rocAL/include/api/rocal_api.h#L41): ``` extern "C" RocalContext ROCAL_API_CALL rocalCreate(size_t batch_size, RocalProcessMode affinity, int gpu_id = 0, size_t cpu_thread_count = 1, size_t prefetch_queue_depth = 3, RocalTensorOutputType output_tensor_data_type = RocalTensorOutputType::ROCAL_FP32); @@ -40,7 +40,7 @@ Use: To verify the graph for all the inputs and outputs Returns: A status code indicating the success or failure -[Example](https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/rocAL/include/api/rocal_api.h#L47): +[Example](https://github.com/ROCm/rocAL/blob/master/rocAL/include/api/rocal_api.h#L47): ``` extern "C" RocalStatus ROCAL_API_CALL rocalVerify(RocalContext context); @@ -52,7 +52,7 @@ Use: To process and run the built and verified graph Returns: A status code indicating the success or failure -[Example](https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/rocAL/include/api/rocal_api.h#L52): +[Example](https://github.com/ROCm/rocAL/blob/master/rocAL/include/api/rocal_api.h#L52): ``` extern "C" RocalStatus ROCAL_API_CALL rocalRun(RocalContext context); @@ -64,7 +64,7 @@ Use: To free all the resources allocated during the graph creation process Returns: A status code indicating the success or failure -[Example](https://github.com/ROCmSoftwarePlatform/rocAL/blob/master/rocAL/include/api/rocal_api.h#L57): +[Example](https://github.com/ROCm/rocAL/blob/master/rocAL/include/api/rocal_api.h#L57): ``` extern "C" RocalStatus ROCAL_API_CALL rocalRelease(RocalContext rocal_context); @@ -72,7 +72,7 @@ extern "C" RocalStatus ROCAL_API_CALL rocalRelease(RocalContext rocal_context) ## 6.1.5 Image Augmentation Using C++ API -The example below shows how to create a pipeline, read JPEG images, perform certain augmentations on them, and show the output using OpenCV by utilizing C++ [APIs](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/blob/develop/apps/image_augmentation/image_augmentation.cpp#L103). +The example below shows how to create a pipeline, read JPEG images, perform certain augmentations on them, and show the output using OpenCV by utilizing C++ [APIs](https://github.com/ROCm/rocAL/blob/develop/tests/cpp_api/image_augmentation/image_augmentation.cpp#L103). ``` Auto handle = rocalCreate(inputBatchSize, processing_device?RocalProcessMode::ROCAL_PROCESS_GPU:RocalProcessMode::ROCAL_PROCESS_CPU, 0,1); @@ -109,4 +109,4 @@ while (!rocalIsEmpty(handle)) } ``` -To see a sample image augmentation application in C++, click [here](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/develop/apps/image_augmentation). +To see a sample image augmentation application in C++, click [here](https://github.com/ROCm/rocAL/tree/develop/tests/cpp_api/image_augmentation). diff --git a/rocAL-setup.py b/rocAL-setup.py index 30f0e29d2..c0d74e94f 100644 --- a/rocAL-setup.py +++ b/rocAL-setup.py @@ -420,6 +420,6 @@ # MIVisionX os.system('sudo -v') os.system('(cd '+deps_dir+'; git clone -b '+mivisionxVersion+' https://github.com/ROCm/MIVisionX.git; cd MIVisionX; mkdir build-'+backend+'; cd build-'+backend+'; ' + - linuxCMake+' -DBACKEND='+backend+' -DROCAL=OFF ../; make -j4; sudo make install)') + linuxCMake+' -DBACKEND='+backend+' ../; make -j4; sudo make install)') print("\nrocAL Dependencies Installed with rocAL-setup.py V-"+__version__+"\n") diff --git a/rocAL_pybind/README.md b/rocAL_pybind/README.md index 057bf1a1e..84058228c 100644 --- a/rocAL_pybind/README.md +++ b/rocAL_pybind/README.md @@ -11,10 +11,10 @@ written primarily in C/C++ language can be used effectively in Python. * CMake Version 3.10 or higher * Python 3 * PIP3 - `sudo apt install python3-pip` -* [CuPy for rocm](https://github.com/ROCmSoftwarePlatform/cupy) +* [CuPy for rocm](https://github.com/ROCm/cupy) ## Install -rocAL_pybind installs during [MIVisionX build](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX#build--install-mivisionx) +rocAL_pybind installs during [rocAL build](https://github.com/ROCm/rocAL#build-instructions) #### Prerequisites @@ -24,4 +24,5 @@ pip3 install numpy opencv-python torch pillow ```` #### Run Test Scripts -* Test scripts and instructions to run them can be found [here](examples/) +* Test scripts and instructions to run them can be found [here](../tests/python_api/) +* Examples using python APIs can be found [here](../docs/examples/) diff --git a/rocAL_pybind/examples/PYTHON_UNITTEST_TEST_FILE.sh b/rocAL_pybind/examples/PYTHON_UNITTEST_TEST_FILE.sh deleted file mode 100755 index bd97c768e..000000000 --- a/rocAL_pybind/examples/PYTHON_UNITTEST_TEST_FILE.sh +++ /dev/null @@ -1,165 +0,0 @@ -#!/bin/bash -cwd=$(pwd) - -if [[ $ROCAL_DATA_PATH == "" ]] -then - echo "Need to export ROCAL_DATA_PATH" - exit -fi - -# Path to inputs and outputs available in MIVisionX-data -one_hot_data_path=${ROCAL_DATA_PATH}/rocal_data/images_jpg/labels_folder/ -image_path=${ROCAL_DATA_PATH}/rocal_data/coco/coco_10_img/train_10images_2017/ -coco_detection_path=${ROCAL_DATA_PATH}/rocal_data/coco/coco_10_img/train_10images_2017/ -coco_json_path=${ROCAL_DATA_PATH}/rocal_data/coco/coco_10_img/annotations/instances_train2017.json -tf_classification_path=${ROCAL_DATA_PATH}/rocal_data/tf/classification/ -tf_detection_path=${ROCAL_DATA_PATH}/rocal_data/tf/detection/ -caffe_classification_path=${ROCAL_DATA_PATH}/rocal_data/caffe/classification/ -caffe_detection_path=${ROCAL_DATA_PATH}/rocal_data/caffe/detection/ -caffe2_classification_path=${ROCAL_DATA_PATH}/rocal_data/caffe2/classification/ -caffe2_detection_path=${ROCAL_DATA_PATH}/rocal_data/caffe2/detection/ -mxnet_path=${ROCAL_DATA_PATH}/rocal_data/mxnet/ -output_path=./rocal_python_unittest_output_folder_$(date +%Y-%m-%d_%H-%M-%S)/ -golden_output_path=${ROCAL_DATA_PATH}/rocal_data/GoldenOutputsTensor/ - -display=0 -batch_size=2 -device=0 -width=640 -height=480 -device_name="host" -rgb_name=("gray" "rgb") -rgb=1 -dev_start=0 -dev_end=1 -rgb_start=0 -rgb_end=1 - -if [ "$#" -gt 0 ]; then - if [ "$1" -eq 0 ]; then # For only HOST backend - dev_start=0 - dev_end=0 - elif [ "$1" -eq 1 ]; then # For only HIP backend - dev_start=1 - dev_end=1 - elif [ "$1" -eq 2 ]; then # For both HOST and HIP backend - dev_start=0 - dev_end=1 - fi -fi - -if [ "$#" -gt 1 ]; then - if [ "$2" -eq 0 ]; then # For only Greyscale inputs - rgb_start=0 - rgb_end=0 - elif [ "$2" -eq 1 ]; then # For only RGB inputs - rgb_start=1 - rgb_end=1 - elif [ "$2" -eq 2 ]; then # For both RGB and Greyscale inputs - rgb_start=0 - rgb_end=1 - fi -fi - -mkdir "$output_path" - -for ((device=dev_start;device<=dev_end;device++)) -do - if [ $device -eq 1 ] - then - device_name="hip" - backend_arg=rocal-gpu - echo "Running HIP Backend..." - else - backend_arg=no-rocal-gpu - echo "Running HOST Backend..." - fi - for ((rgb=rgb_start;rgb<=rgb_end;rgb++)) - do - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$image_path" --augmentation-name lens_correction --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}LensCorrection_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$image_path" --augmentation-name exposure --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Exposure_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$image_path" --augmentation-name flip --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Flip_${rgb_name[$rgb]}_${device_name}" - - # coco detection - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$coco_detection_path" --reader-type coco --json-path "$coco_json_path" --augmentation-name gamma_correction --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Gamma_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$coco_detection_path" --reader-type coco --json-path "$coco_json_path" --augmentation-name contrast --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Contrast_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$coco_detection_path" --reader-type coco --json-path "$coco_json_path" --augmentation-name vignette --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Vignette_${rgb_name[$rgb]}_${device_name}" - - # # tf classification - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$tf_classification_path" --reader-type "tf_classification" --augmentation-name blend --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Blend_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$tf_classification_path" --reader-type "tf_classification" --augmentation-name warp_affine --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}WarpAffine_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$tf_classification_path" --reader-type "tf_classification" --augmentation-name blur --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Blur_${rgb_name[$rgb]}_${device_name}" - - # tf detection - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$tf_detection_path" --reader-type "tf_detection" --augmentation-name snp_noise --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}SNPNoise_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$tf_detection_path" --reader-type "tf_detection" --augmentation-name color_temp --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}ColorTemp_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$tf_detection_path" --reader-type "tf_detection" --augmentation-name fog --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Fog_${rgb_name[$rgb]}_${device_name}" - - # caffe classification - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe_classification_path" --reader-type "caffe_classification" --augmentation-name rotate --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Rotate_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe_classification_path" --reader-type "caffe_classification" --augmentation-name brightness --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Brightness_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe_classification_path" --reader-type "caffe_classification" --augmentation-name hue --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Hue_${rgb_name[$rgb]}_${device_name}" - - # caffe detection - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe_detection_path" --reader-type "caffe_detection" --augmentation-name saturation --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Saturation_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe_detection_path" --reader-type "caffe_detection" --augmentation-name color_twist --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}ColorTwist_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe_detection_path" --reader-type "caffe_detection" --augmentation-name rain --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Rain_${rgb_name[$rgb]}_${device_name}" - - # caffe2 classification - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe2_classification_path" --reader-type "caffe2_classification" --augmentation-name center_crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropCenter_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe2_classification_path" --reader-type "caffe2_classification" --augmentation-name resize_crop_mirror --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}ResizeCropMirror_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe2_classification_path" --reader-type "caffe2_classification" --augmentation-name snow --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Snow_${rgb_name[$rgb]}_${device_name}" - - # caffe2 detection - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe2_detection_path" --reader-type "caffe2_detection" --augmentation-name fish_eye --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}FishEye_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe2_detection_path" --reader-type "caffe2_detection" --augmentation-name pixelate --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Pixelate_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe2_detection_path" --reader-type "caffe2_detection" --augmentation-name center_crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropCenter_${rgb_name[$rgb]}_${device_name}_cmn" - - # mxnet - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$mxnet_path" --reader-type "mxnet" --augmentation-name jitter --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Jitter_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$mxnet_path" --reader-type "mxnet" --augmentation-name resize_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}ResizeMirrorNormalize_${rgb_name[$rgb]}_${device_name}" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$mxnet_path" --reader-type "mxnet" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_mxnet" - - # CMN - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$image_path" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_FileReader" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$coco_detection_path" --reader-type coco --json-path "$coco_json_path" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_coco" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$tf_classification_path" --reader-type "tf_classification" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_tfClassification" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$tf_detection_path" --reader-type "tf_detection" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_tfDetection" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe_classification_path" --reader-type "caffe_classification" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffeClassification" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe_detection_path" --reader-type "caffe_detection" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffeDetection" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe2_classification_path" --reader-type "caffe2_classification" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffe2Classification" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe2_detection_path" --reader-type "caffe2_detection" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffe2Detection" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$mxnet_path" --reader-type "mxnet" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_mxnet" - - # crop - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$image_path" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_FileReader" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$coco_detection_path" --reader-type coco --json-path "$coco_json_path" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_coco" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$tf_classification_path" --reader-type "tf_classification" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_tfClassification" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$tf_detection_path" --reader-type "tf_detection" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_tfDetection" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe_classification_path" --reader-type "caffe_classification" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffeClassification" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe_detection_path" --reader-type "caffe_detection" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffeDetection" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe2_classification_path" --reader-type "caffe2_classification" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffe2Classification" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe2_detection_path" --reader-type "caffe2_detection" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffe2Detection" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$mxnet_path" --reader-type "mxnet" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_mxnet" - - # resize - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$image_path" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 1 --scaling-mode 0 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_default_FileReader" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$coco_detection_path" --reader-type coco --json-path "$coco_json_path" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 1 --scaling-mode 1 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_stretch_coco" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$tf_classification_path" --reader-type "tf_classification" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 1 --scaling-mode 2 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_notsmaller_tfClassification" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$tf_detection_path" --reader-type "tf_detection" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 1 --scaling-mode 3 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_notlarger_tfDetection" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe_classification_path" --reader-type "caffe_classification" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 2 --scaling-mode 0 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bicubic_default_caffeClassification" - # python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe_detection_path" --reader-type "caffe_detection" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 0 --scaling-mode 0 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_nearestneighbor_default_caffeDetection" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe2_classification_path" --reader-type "caffe2_classification" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 3 --scaling-mode 0 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_lanczos_default_caffe2Classification" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$caffe2_detection_path" --reader-type "caffe2_detection" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 5 --scaling-mode 0 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_triangular_default_caffe2Detection" - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$mxnet_path" --reader-type "mxnet" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 4 --scaling-mode 0 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_gaussian_default_mxnet" - - # Special Case - One Hot Encoded Labels - python"$ver" rocAL_api_python_unittest.py --image-dataset-path "$one_hot_data_path" --augmentation-name one_hot --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}OneHot_${rgb_name[$rgb]}_${device_name}" - - done -done - -pwd - -# Run python script to compare rocAL outputs with golden ouptuts -python3 "$cwd"/image_comparison.py "$golden_output_path" "$output_path" diff --git a/rocAL_pybind/examples/README.md b/rocAL_pybind/examples/README.md deleted file mode 100644 index a4a192cb5..000000000 --- a/rocAL_pybind/examples/README.md +++ /dev/null @@ -1,68 +0,0 @@ -### TESTING ALL THE READER PIPELINES ON PYTHON IN SINGLE SHOT - -./TEST_FILE.sh - -### TO TEST A SINGLE READER / MULTIPLE READER PIPELINE FROM TEST_FILE.sh - -# Make the respective " Pipeline " to test equal to "1" -rocAL_api_python_unittest=1 -rocAL_api_coco_pipeline=0 - -## INSTALL PYBIND TO RUN ANY TESTS -* [rocAL Pybind Installation](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/master/README.md) - -## EXPORT ROCAL_DATA_PATH -export ROCAL_DATA_PATH=/Absolute/Path/Of/MIVisionX-data/ - -### TESTING ALL THE AUGMENTATIONS IN A SINGLE SHOT - -./PYTHON_UNITTEST_TEST_FILE.sh (This runs all augmentations and readers and compare them with golden outputs for both backends) - -### TO TEST MULTIPLE READER PIPELINE FROM READERS_TEST_FILE.sh - -./READERS_TEST_FILE.sh (The default value of number of gpu's is "1" & display is "ON" by default) - -./READERS_TEST_FILE.sh -n -d -./READERS_TEST_FILE.sh -n "1" -d "false" -b "cpu" - -### TO TEST A SINGLE READER PIPELINE FROM READERS_TEST_FILE.sh - -example : To run COCO Pipeline - -rocAL_api_python_unittest=0 -rocAL_api_coco_pipeline=1 -rocAL_api_caffe_reader=0 -rocAL_api_caffe2_reader=0 -rocAL_api_tf_classification_reader=0 -rocAL_api_tf_detection_pipeline=0 - -### TO TEST A SINGLE READER PIPELINE FROM CMD LINE - -example: COCO Pipeline - - # Mention the number of gpus - gpus_per_node=4 - - # Mention Batch Size - batch_size=10 - - # python version - ver=$(python -V 2>&1 | sed 's/.* \([0-9]\).\([0-9]\).*/\1\.\2/') - - # Mention dataset_path - data_dir=$ROCAL_DATA_PATH/coco/coco_10_img/val_10images_2017/ - - # Mention json path - json_path=$ROCAL_DATA_PATH/coco/coco_10_img/annotations/instances_val2017.json - - # rocAL_api_coco_pipeline.py - # By default : cpu backend, NCHW format , fp32 - # Annotation must be a json file - - # For printing all the arguments that can be passed from user - python$ver rocAL_api_coco_pipeline.py -h - - python$ver rocAL_api_coco_pipeline.py --image-dataset-path $data_dir --json-path $json_path --batch-size $batch_size --display --rocal-gpu --NHWC \ - --local-rank 0 --world-size $gpus_per_node --num-threads 1 --num-epochs 1 2>&1 | tee -a run.log.rocAL_api_coco_pipeline.txt - -### [ NOTE: REFER parse_config.py for more INFO on other args] diff --git a/rocAL_pybind/examples/tf_petsTrainingExample/README.md b/rocAL_pybind/examples/tf_petsTrainingExample/README.md deleted file mode 100644 index b6a19b8da..000000000 --- a/rocAL_pybind/examples/tf_petsTrainingExample/README.md +++ /dev/null @@ -1,20 +0,0 @@ -## Running tf_petsTrainingExample - -### Building the required TF Rocm docker -* Use the instructions in the [docker section](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/master/docker) to build the required [Tensorflow docker](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/master/docker/tensorflow) -* Upgrade pip to the latest version. - -### Building the required Pytorch Rocm docker -* Use the instructions in the [docker section](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/master/docker) to build the required [Pytorch docker](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX/tree/master/docker/pytorch) -* Upgrade pip to the latest version. -* Run requirements.sh to install the required packages. - -### Running the training - -* For first run, to setup dataset, edit "train_withROCAL_withTFRecordReader.py" and set "DATASET_DOWNLOAD_AND_PREPROCESS = True" -* For subsequent runs, after the dataset has already been downloaded and preprocessed, set "DATASET_DOWNLOAD_AND_PREPROCESS = False" - -To run this example for the first run or subsequent runs, just execute: -``` -python3 train_withROCAL_withTFRecordReader.py -``` diff --git a/tests/README.md b/tests/README.md index 5c0d87066..a01c7302f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -3,5 +3,5 @@ rocAL test suite to verify installation and functionality ## C++ API Test - -rocAL C++ API functionality tests \ No newline at end of file +* [C++ API functionality tests](cpp_api) +* [Python API functionality tests](python_api) \ No newline at end of file diff --git a/tests/cpp_api_tests/CMakeLists.txt b/tests/cpp_api/CMakeLists.txt similarity index 52% rename from tests/cpp_api_tests/CMakeLists.txt rename to tests/cpp_api/CMakeLists.txt index b41c96906..6cbb6d74c 100644 --- a/tests/cpp_api_tests/CMakeLists.txt +++ b/tests/cpp_api/CMakeLists.txt @@ -25,139 +25,139 @@ ################################################################################ cmake_minimum_required (VERSION 3.5) -# rocal_basic_test +# basic_test add_test( NAME - rocAL_basic_test_cpu + basic_test_cpu COMMAND "${CMAKE_CTEST_COMMAND}" - --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/rocAL_basic_test" - "${CMAKE_CURRENT_BINARY_DIR}/rocAL_basic_test" + --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/basic_test" + "${CMAKE_CURRENT_BINARY_DIR}/basic_test" --build-generator "${CMAKE_GENERATOR}" - --test-command "rocal_basic_test" + --test-command "basic_test" ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet-val.txt 1 0 224 224 ) -add_test(NAME rocAL_basic_test_gpu - COMMAND rocal_basic_test +add_test(NAME basic_test_gpu + COMMAND basic_test ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet-val.txt 1 0 224 224 - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rocAL_basic_test) -add_test(NAME rocAL_basic_test_gray - COMMAND rocal_basic_test + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/basic_test) +add_test(NAME basic_test_gray + COMMAND basic_test ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet-val.txt 1 1 224 224 0 - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rocAL_basic_test) -add_test(NAME rocAL_basic_test_rgb - COMMAND rocal_basic_test + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/basic_test) +add_test(NAME basic_test_rgb + COMMAND basic_test ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet-val.txt 1 1 224 224 1 - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rocAL_basic_test) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/basic_test) -# TBD - rocAL_dataloader unit test options non-functional - NEEDS TO BE ADDED ONCE RESOLVED +# TBD - dataloader unit test options non-functional - NEEDS TO BE ADDED ONCE RESOLVED #add_test( # NAME -# rocAL_dataloader +# dataloader # COMMAND # "${CMAKE_CTEST_COMMAND}" -# --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/rocAL_dataloader" -# "${CMAKE_CURRENT_BINARY_DIR}/rocAL_dataloader" +# --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/dataloader" +# "${CMAKE_CURRENT_BINARY_DIR}/dataloader" # --build-generator "${CMAKE_GENERATOR}" -# --test-command "rocal_dataloader" +# --test-command "dataloader" # ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet #) -# rocal_dataloader_mt +# dataloader_multithread add_test( NAME rocAL_dataloader_mt_cpu COMMAND "${CMAKE_CTEST_COMMAND}" - --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/rocAL_dataloader_mt" - "${CMAKE_CURRENT_BINARY_DIR}/rocAL_dataloader_mt" + --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/dataloader_multithread" + "${CMAKE_CURRENT_BINARY_DIR}/dataloader_multithread" --build-generator "${CMAKE_GENERATOR}" - --test-command "rocal_dataloader_mt" + --test-command "dataloader_multithread" ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet 0 ) add_test(NAME rocAL_dataloader_mt_gpu - COMMAND rocal_dataloader_mt + COMMAND dataloader_multithread ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet 1 - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rocAL_dataloader_mt) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/dataloader_multithread) -# TBD - rocAL_dataloader_tf unit test non-functional +# TBD - dataloader_tf unit test non-functional #add_test( # NAME -# rocAL_dataloader_tf +# dataloader_tf # COMMAND # "${CMAKE_CTEST_COMMAND}" -# --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/rocAL_dataloader_tf" -# "${CMAKE_CURRENT_BINARY_DIR}/rocAL_dataloader_tf" +# --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/dataloader_tf" +# "${CMAKE_CURRENT_BINARY_DIR}/dataloader_tf" # --build-generator "${CMAKE_GENERATOR}" -# --test-command "rocal_dataloader_tf" +# --test-command "dataloader_tf" # ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet #) -# rocal_performance_tests +# performance_tests # TBD - peformance test needs to run with default options add_test( NAME rocAL_performance_tests_cpu COMMAND "${CMAKE_CTEST_COMMAND}" - --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/rocAL_performance_tests" - "${CMAKE_CURRENT_BINARY_DIR}/rocAL_performance_tests" + --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/performance_tests" + "${CMAKE_CURRENT_BINARY_DIR}/performance_tests" --build-generator "${CMAKE_GENERATOR}" - --test-command "rocal_performance_tests" + --test-command "performance_tests" ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet 224 224 1 16 0 ) add_test(NAME rocAL_performance_tests_gpu - COMMAND rocal_performance_tests + COMMAND performance_tests ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet 224 224 1 16 1 - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rocAL_performance_tests) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/performance_tests) -# rocal_performance_tests_with_depth +# performance_tests_with_depth add_test( NAME - rocAL_performance_tests_with_depth_cpu + performance_tests_with_depth_cpu COMMAND "${CMAKE_CTEST_COMMAND}" - --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/rocAL_performance_tests_with_depth" - "${CMAKE_CURRENT_BINARY_DIR}/rocAL_performance_tests_with_depth" + --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/performance_tests_with_depth" + "${CMAKE_CURRENT_BINARY_DIR}/performance_tests_with_depth" --build-generator "${CMAKE_GENERATOR}" - --test-command "rocal_performance_tests_with_depth" + --test-command "performance_tests_with_depth" ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet 224 224 1 1 1 0 ) -add_test(NAME rocAL_performance_tests_with_depth_gpu - COMMAND rocal_performance_tests_with_depth +add_test(NAME performance_tests_with_depth_gpu + COMMAND performance_tests_with_depth ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet 224 224 1 1 1 1 - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rocAL_performance_tests_with_depth) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/performance_tests_with_depth) -# rocal_unittests +# unit_tests add_test( NAME - rocAL_unittests_cpu + unit_tests_cpu COMMAND "${CMAKE_CTEST_COMMAND}" - --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/rocAL_unittests" - "${CMAKE_CURRENT_BINARY_DIR}/rocAL_unittests" + --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/unit_tests" + "${CMAKE_CURRENT_BINARY_DIR}/unit_tests" --build-generator "${CMAKE_GENERATOR}" - --test-command "rocal_unittests" + --test-command "unit_tests" 0 ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet test 224 224 1 0 1 ) -add_test(NAME rocAL_unittests_gpu - COMMAND rocal_unittests +add_test(NAME unit_tests_gpu + COMMAND unit_tests 0 ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet test 224 224 1 1 1 - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rocAL_unittests) -add_test(NAME rocAL_unittests_gray - COMMAND rocal_unittests + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/unit_tests) +add_test(NAME unit_tests_gray + COMMAND unit_tests 0 ${CMAKE_SOURCE_DIR}/data/images/AMD-tinyDataSet test 224 224 1 1 0 - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rocAL_unittests) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/unit_tests) -# rocal_video_unittests +# video_tests add_test( NAME - rocAL_video_unittests + video_tests COMMAND "${CMAKE_CTEST_COMMAND}" - --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/rocAL_video_unittests" - "${CMAKE_CURRENT_BINARY_DIR}/rocAL_video_unittests" + --build-and-test "${CMAKE_CURRENT_SOURCE_DIR}/video_tests" + "${CMAKE_CURRENT_BINARY_DIR}/video_tests" --build-generator "${CMAKE_GENERATOR}" - --test-command "rocal_video_unittests" + --test-command "video_tests" ${CMAKE_SOURCE_DIR}/data/videos/AMD_driving_virtual_20.mp4 ) diff --git a/tests/cpp_api_tests/rocAL_basic_test/CMakeLists.txt b/tests/cpp_api/basic_test/CMakeLists.txt similarity index 99% rename from tests/cpp_api_tests/rocAL_basic_test/CMakeLists.txt rename to tests/cpp_api/basic_test/CMakeLists.txt index bcb2b7b57..24067caf0 100644 --- a/tests/cpp_api_tests/rocAL_basic_test/CMakeLists.txt +++ b/tests/cpp_api/basic_test/CMakeLists.txt @@ -25,7 +25,7 @@ ################################################################################ cmake_minimum_required(VERSION 3.5) -project(rocal_basic_test) +project(basic_test) set(CMAKE_CXX_STANDARD 14) diff --git a/tests/cpp_api/basic_test/README.md b/tests/cpp_api/basic_test/README.md new file mode 100644 index 000000000..09676e2b0 --- /dev/null +++ b/tests/cpp_api/basic_test/README.md @@ -0,0 +1,24 @@ +# basic test +This application demonstrates a basic usage of rocAL's C API to load images from the disk and tests the functionality for getting image labels and displays the output images. +

+ +## Build Instructions + +### Pre-requisites +* Ubuntu Linux, [version `16.04` or later](https://www.microsoft.com/software-download/windows10) +* rocAL library +* [OpenCV 3.1](https://github.com/opencv/opencv/releases) or higher +* ROCm Performance Primitives (RPP) + +### build + ```` + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/rocm/lib + mkdir build + cd build + cmake ../ + make + ```` +### running the application + ``` + ./basic_test decode_width decode_height decode_shard_counts + ``` \ No newline at end of file diff --git a/tests/cpp_api_tests/rocAL_basic_test/rocal_basic_test.cpp b/tests/cpp_api/basic_test/basic_test.cpp similarity index 97% rename from tests/cpp_api_tests/rocAL_basic_test/rocal_basic_test.cpp rename to tests/cpp_api/basic_test/basic_test.cpp index a7a91e7ba..cadf09dad 100644 --- a/tests/cpp_api_tests/rocAL_basic_test/rocal_basic_test.cpp +++ b/tests/cpp_api/basic_test/basic_test.cpp @@ -48,7 +48,7 @@ int main(int argc, const char **argv) { // check command-line usage const int MIN_ARG_COUNT = 2; if (argc < MIN_ARG_COUNT) { - std::cout << "Usage: rocal_basic_test decode_width decode_height decode_shard_counts \n"; + std::cout << "Usage: basic_test decode_width decode_height decode_shard_counts \n"; return -1; } int argIdx = 0; diff --git a/tests/cpp_api_tests/rocAL_dataloader/CMakeLists.txt b/tests/cpp_api/dataloader/CMakeLists.txt similarity index 99% rename from tests/cpp_api_tests/rocAL_dataloader/CMakeLists.txt rename to tests/cpp_api/dataloader/CMakeLists.txt index 2e3e0b25e..f533ce803 100644 --- a/tests/cpp_api_tests/rocAL_dataloader/CMakeLists.txt +++ b/tests/cpp_api/dataloader/CMakeLists.txt @@ -25,7 +25,7 @@ ################################################################################ cmake_minimum_required (VERSION 3.5) -project(rocal_dataloader) +project(dataloader) set(CMAKE_CXX_STANDARD 14) diff --git a/tests/cpp_api_tests/rocAL_dataloader/README.md b/tests/cpp_api/dataloader/README.md similarity index 66% rename from tests/cpp_api_tests/rocAL_dataloader/README.md rename to tests/cpp_api/dataloader/README.md index 92fd56615..13412750d 100644 --- a/tests/cpp_api_tests/rocAL_dataloader/README.md +++ b/tests/cpp_api/dataloader/README.md @@ -1,12 +1,12 @@ -# rocal_dataloader application +# dataloader application This application demonstrates a basic usage of rocAL's C API to load RAW images from the disk and modify them in different possible ways and displays the output images. -

+

## Build Instructions ### Pre-requisites * Ubuntu Linux, [version `16.04` or later](https://www.microsoft.com/software-download/windows10) -* rocAL library (Part of the MIVisionX toolkit) +* rocAL library * [OpenCV 3.1](https://github.com/opencv/opencv/releases) or higher * ROCm Performance Primitives (RPP) @@ -20,5 +20,5 @@ This application demonstrates a basic usage of rocAL's C API to load RAW images ```` ### running the application ```` - rocal_dataloader + ./dataloader decode_width decode_height batch_size display_on_off ```` diff --git a/tests/cpp_api_tests/rocAL_dataloader/rocAL_dataloader.cpp b/tests/cpp_api/dataloader/dataloader.cpp similarity index 98% rename from tests/cpp_api_tests/rocAL_dataloader/rocAL_dataloader.cpp rename to tests/cpp_api/dataloader/dataloader.cpp index 540efd196..235a78dcb 100644 --- a/tests/cpp_api_tests/rocAL_dataloader/rocAL_dataloader.cpp +++ b/tests/cpp_api/dataloader/dataloader.cpp @@ -49,7 +49,7 @@ int main(int argc, const char **argv) { // check command-line usage const int MIN_ARG_COUNT = 2; if (argc < MIN_ARG_COUNT) { - printf("Usage: image_augmentation decode_width decode_height batch_size display_on_off \n"); + printf("Usage: image_augmentation decode_width decode_height batch_size display_on_off \n"); return -1; } int argIdx = 0; diff --git a/tests/cpp_api_tests/rocAL_dataloader_mt/CMakeLists.txt b/tests/cpp_api/dataloader_multithread/CMakeLists.txt similarity index 94% rename from tests/cpp_api_tests/rocAL_dataloader_mt/CMakeLists.txt rename to tests/cpp_api/dataloader_multithread/CMakeLists.txt index da0f6c6ef..b2277ba8c 100644 --- a/tests/cpp_api_tests/rocAL_dataloader_mt/CMakeLists.txt +++ b/tests/cpp_api/dataloader_multithread/CMakeLists.txt @@ -25,7 +25,7 @@ ################################################################################ cmake_minimum_required (VERSION 3.5) -project (rocal_dataloader_mt) +project (dataloader_multithread) set(CMAKE_CXX_STANDARD 14) @@ -65,7 +65,7 @@ elseif ("${BACKEND}" STREQUAL "HIP") list(APPEND CMAKE_PREFIX_PATH ${ROCM_PATH} ${ROCM_PATH}/hip) find_package(HIP QUIET) if(HIP_FOUND) - message("-- ${White}rocAL_dataloader_mt -- Using HIP -- Path:" ${HIP_PATH} "\tVersion:" ${HIP_VERSION} "\tCompiler:" ${HIP_COMPILER} ${ColourReset}) + message("-- ${White}dataloader_multithread -- Using HIP -- Path:" ${HIP_PATH} "\tVersion:" ${HIP_VERSION} "\tCompiler:" ${HIP_COMPILER} ${ColourReset}) set(ENABLE_HIP 1) set(ENABLE_OPENCL 0) include_directories(${HIP_PATH}/include) @@ -82,7 +82,7 @@ endif() include_directories (${ROCM_PATH}/${CMAKE_INSTALL_INCLUDEDIR}/rocal) link_directories (${ROCM_PATH}/lib) -add_executable(${PROJECT_NAME} ./rocAL_dataloader_mt.cpp) +add_executable(${PROJECT_NAME} ./dataloader_multithread.cpp) if(OpenCV_FOUND) if(${OpenCV_VERSION_MAJOR} GREATER_EQUAL 3 ) message("-- OpenCV Found -- Version-${OpenCV_VERSION_MAJOR}.${OpenCV_VERSION_MINOR}.X Supported") diff --git a/tests/cpp_api_tests/rocAL_dataloader_mt/README.md b/tests/cpp_api/dataloader_multithread/README.md similarity index 56% rename from tests/cpp_api_tests/rocAL_dataloader_mt/README.md rename to tests/cpp_api/dataloader_multithread/README.md index c60f54596..2bc50c3c4 100644 --- a/tests/cpp_api_tests/rocAL_dataloader_mt/README.md +++ b/tests/cpp_api/dataloader_multithread/README.md @@ -1,12 +1,12 @@ -# rocAL_dataloader_mt application +# dataloader multithreaded application This application demonstrates a basic usage of rocAL's C API to use sharded data_loader in a multithreaded application. -

+

## Build Instructions ### Pre-requisites * Ubuntu Linux, [version `16.04` or later](https://www.microsoft.com/software-download/windows10) -* rocAL library (Part of the MIVisionX toolkit) +* rocAL library * [OpenCV 3.1](https://github.com/opencv/opencv/releases) or higher * ROCm Performance Primitives (RPP) @@ -20,5 +20,5 @@ This application demonstrates a basic usage of rocAL's C API to use sharded data ```` ### running the application ```` - rocAL_dataloader_mt =1)/(cpu:0)> + ./dataloader_multithread =1)/(cpu:0)> ```` diff --git a/tests/cpp_api_tests/rocAL_dataloader_mt/rocAL_dataloader_mt.cpp b/tests/cpp_api/dataloader_multithread/dataloader_multithread.cpp similarity index 97% rename from tests/cpp_api_tests/rocAL_dataloader_mt/rocAL_dataloader_mt.cpp rename to tests/cpp_api/dataloader_multithread/dataloader_multithread.cpp index 5a00fd03f..3334ce5c5 100644 --- a/tests/cpp_api_tests/rocAL_dataloader_mt/rocAL_dataloader_mt.cpp +++ b/tests/cpp_api/dataloader_multithread/dataloader_multithread.cpp @@ -195,9 +195,8 @@ int main(int argc, const char **argv) { // check command-line usage const int MIN_ARG_COUNT = 2; if (argc < MIN_ARG_COUNT) { - printf( - "Usage: rocal_dataloader_mt 1(gpu)/cpu=0> num_shards, \ - decode_width decode_height batch_size shuffle display_on_off dec_mode<0(tjpeg)/1(opencv)/2(hwdec)>\n"); + std::cout << "Usage: dataloader_multithread " << + "num_shards decode_width decode_height batch_size shuffle display_on_off dec_mode<0(tjpeg)/1(opencv)/2(hwdec)>" << std::endl; return -1; } int argIdx = 0; diff --git a/tests/cpp_api_tests/rocAL_unittests/CMakeLists.txt b/tests/cpp_api/dataloader_tf/CMakeLists.txt similarity index 99% rename from tests/cpp_api_tests/rocAL_unittests/CMakeLists.txt rename to tests/cpp_api/dataloader_tf/CMakeLists.txt index 59de69611..2e632a782 100644 --- a/tests/cpp_api_tests/rocAL_unittests/CMakeLists.txt +++ b/tests/cpp_api/dataloader_tf/CMakeLists.txt @@ -25,7 +25,8 @@ ################################################################################ cmake_minimum_required(VERSION 3.5) -project (rocal_unittests) +project(dataloader_tf) + set(CMAKE_CXX_STANDARD 14) # ROCm Path @@ -45,6 +46,7 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/../../cmake) find_package(OpenCV QUIET) find_package(AMDRPP QUIET) + include_directories(${ROCM_PATH}/${CMAKE_INSTALL_INCLUDEDIR}/rocal) link_directories(${ROCM_PATH}/lib) file(GLOB My_Source_Files ./*.cpp) diff --git a/tests/cpp_api_tests/rocAL_dataloader_tf/README.md b/tests/cpp_api/dataloader_tf/README.md similarity index 72% rename from tests/cpp_api_tests/rocAL_dataloader_tf/README.md rename to tests/cpp_api/dataloader_tf/README.md index 268875bc0..0b0bbec3a 100644 --- a/tests/cpp_api_tests/rocAL_dataloader_tf/README.md +++ b/tests/cpp_api/dataloader_tf/README.md @@ -1,11 +1,10 @@ -# rocal_dataloader_tf application +# dataloader tensorflow application This application demonstrates a basic usage of rocAL's C API to load TfRecords from the disk and modify them in different possible ways and displays the output images. ## Build Instructions ### Pre-requisites * Ubuntu 16.04/18.04 Linux -* rocAL library (Part of the MIVisionX toolkit) * [OpenCV 3.1](https://github.com/opencv/opencv/releases) or higher * Google protobuf 3.11.1 or higher * ROCm Performance Primitives (RPP) @@ -20,5 +19,5 @@ This application demonstrates a basic usage of rocAL's C API to load TfRecords f ```` ### running the application ```` - rocal_dataloader + ./dataloader_tf ```` diff --git a/tests/cpp_api_tests/rocAL_dataloader_tf/rocAL_dataloader_tf.cpp b/tests/cpp_api/dataloader_tf/dataloader_tf.cpp similarity index 98% rename from tests/cpp_api_tests/rocAL_dataloader_tf/rocAL_dataloader_tf.cpp rename to tests/cpp_api/dataloader_tf/dataloader_tf.cpp index 6fc4d6b6e..71eb05d3b 100644 --- a/tests/cpp_api_tests/rocAL_dataloader_tf/rocAL_dataloader_tf.cpp +++ b/tests/cpp_api/dataloader_tf/dataloader_tf.cpp @@ -49,7 +49,7 @@ int main(int argc, const char **argv) { // check command-line usage const int MIN_ARG_COUNT = 2; if (argc < MIN_ARG_COUNT) { - printf("Usage: rocal_dataloader_tf display_on_off \n"); + printf("Usage: dataloader_tf display_on_off \n"); return -1; } int argIdx = 0; diff --git a/tests/cpp_api_tests/rocAL_external_source/CMakeLists.txt b/tests/cpp_api/external_source/CMakeLists.txt similarity index 98% rename from tests/cpp_api_tests/rocAL_external_source/CMakeLists.txt rename to tests/cpp_api/external_source/CMakeLists.txt index 935d7de34..6b0b6589d 100644 --- a/tests/cpp_api_tests/rocAL_external_source/CMakeLists.txt +++ b/tests/cpp_api/external_source/CMakeLists.txt @@ -25,7 +25,7 @@ ################################################################################ cmake_minimum_required(VERSION 3.5) -project (rocal_external_source) +project (external_source) set(CMAKE_CXX_STANDARD 14) # ROCm Path diff --git a/tests/cpp_api_tests/rocAL_external_source/README.md b/tests/cpp_api/external_source/README.md similarity index 71% rename from tests/cpp_api_tests/rocAL_external_source/README.md rename to tests/cpp_api/external_source/README.md index acaf18f52..3a06650ec 100644 --- a/tests/cpp_api_tests/rocAL_external_source/README.md +++ b/tests/cpp_api/external_source/README.md @@ -1,4 +1,4 @@ -# rocal_external_source application +# external source application This application demonstrates a basic usage of rocAL's C++ API to load images from external source and add augmentations and displays the output images. @@ -23,6 +23,6 @@ This application demonstrates a basic usage of rocAL's C++ API to load images fr ### running the application - ````bash - rocal_external_source decode_width decode_height batch_size gray_scale/rgb/rgbplanar display_on_off external_source_mode - ```` + ``` + ./external_source decode_width decode_height batch_size gray_scale/rgb/rgbplanar display_on_off external_source_mode + ``` diff --git a/tests/cpp_api_tests/rocAL_external_source/rocal_external_source.cpp b/tests/cpp_api/external_source/external_source.cpp similarity index 99% rename from tests/cpp_api_tests/rocAL_external_source/rocal_external_source.cpp rename to tests/cpp_api/external_source/external_source.cpp index 3c3f24cc3..6327cc9bf 100644 --- a/tests/cpp_api_tests/rocAL_external_source/rocal_external_source.cpp +++ b/tests/cpp_api/external_source/external_source.cpp @@ -63,7 +63,7 @@ int main(int argc, const char **argv) { const int MIN_ARG_COUNT = 2; if (argc < MIN_ARG_COUNT) { printf( - "Usage: rocal_external_source " + "Usage: external_source " " decode_width decode_height batch_size " "gray_scale/rgb/rgbplanar display_on_off external_source_mode\n"); return -1; diff --git a/apps/image_augmentation/CMakeLists.txt b/tests/cpp_api/image_augmentation/CMakeLists.txt similarity index 98% rename from apps/image_augmentation/CMakeLists.txt rename to tests/cpp_api/image_augmentation/CMakeLists.txt index e25a3494e..ec3fd8703 100644 --- a/apps/image_augmentation/CMakeLists.txt +++ b/tests/cpp_api/image_augmentation/CMakeLists.txt @@ -47,7 +47,7 @@ set(CMAKE_SKIP_INSTALL_RPATH TRUE) set(CMAKE_INSTALL_LIBDIR "lib" CACHE STRING "Library install directory") include(GNUInstallDirs) -list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/../../cmake) +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/../../../cmake) find_package(AMDRPP REQUIRED) find_package(MIVisionX REQUIRED) diff --git a/apps/image_augmentation/README.md b/tests/cpp_api/image_augmentation/README.md similarity index 93% rename from apps/image_augmentation/README.md rename to tests/cpp_api/image_augmentation/README.md index 02270a4b6..8d0b58e69 100644 --- a/apps/image_augmentation/README.md +++ b/tests/cpp_api/image_augmentation/README.md @@ -26,5 +26,5 @@ This application demonstrates the basic usage of rocAL's C API to load JPEG imag ### Running the application ``` - image_augmentation +./image_augmentation ``` diff --git a/apps/image_augmentation/image_augmentation.cpp b/tests/cpp_api/image_augmentation/image_augmentation.cpp similarity index 98% rename from apps/image_augmentation/image_augmentation.cpp rename to tests/cpp_api/image_augmentation/image_augmentation.cpp index a21b89393..f67792e04 100644 --- a/apps/image_augmentation/image_augmentation.cpp +++ b/tests/cpp_api/image_augmentation/image_augmentation.cpp @@ -209,11 +209,11 @@ int main(int argc, const char** argv) { /*>>>>>>>>>>>>>>>>>>> Diplay using OpenCV <<<<<<<<<<<<<<<<<*/ // initializations for logos and heading cv::Mat AMD_Epyc_Black_resize, AMD_ROCm_Black_resize; - AMD_Epyc_Black_resize = cv::imread("../../../samples/images/amd-epyc-black-resize.png"); - AMD_ROCm_Black_resize = cv::imread("../../../samples/images/rocm-black-resize.png"); + AMD_Epyc_Black_resize = cv::imread("../../../../docs/data/amd-epyc-black-resize.png"); + AMD_ROCm_Black_resize = cv::imread("../../../../docs/data/rocm-black-resize.png"); int fontFace = CV_FONT_HERSHEY_DUPLEX; int thickness = 1.3; - std::string bufferName = "MIVisionX Image Augmentation"; + std::string bufferName = "rocAL Image Augmentation"; int h = rocalGetAugmentationBranchCount(handle) * rocalGetOutputHeight(handle) * inputBatchSize; int w = rocalGetOutputWidth(handle); diff --git a/tests/cpp_api_tests/rocAL_performance_tests/CMakeLists.txt b/tests/cpp_api/performance_tests/CMakeLists.txt similarity index 98% rename from tests/cpp_api_tests/rocAL_performance_tests/CMakeLists.txt rename to tests/cpp_api/performance_tests/CMakeLists.txt index 0d235c805..4fb5f91a8 100644 --- a/tests/cpp_api_tests/rocAL_performance_tests/CMakeLists.txt +++ b/tests/cpp_api/performance_tests/CMakeLists.txt @@ -25,7 +25,7 @@ ################################################################################ cmake_minimum_required(VERSION 3.5) -project (rocal_performance_tests) +project (performance_tests) set(CMAKE_CXX_STANDARD 14) diff --git a/tests/cpp_api_tests/rocAL_performance_tests/README.md b/tests/cpp_api/performance_tests/README.md similarity index 70% rename from tests/cpp_api_tests/rocAL_performance_tests/README.md rename to tests/cpp_api/performance_tests/README.md index e808e7bc4..e15d71533 100644 --- a/tests/cpp_api_tests/rocAL_performance_tests/README.md +++ b/tests/cpp_api/performance_tests/README.md @@ -6,7 +6,7 @@ This application is used to run performance tests on the rocAL API for graphs of ### Pre-requisites * Ubuntu Linux, [version `16.04` or later](https://www.microsoft.com/software-download/windows10) -* rocAL library (Part of the MIVisionX toolkit) +* rocAL library * [OpenCV 3.1](https://github.com/opencv/opencv/releases) or higher * ROCm Performance Primitives (RPP) @@ -19,5 +19,5 @@ This application is used to run performance tests on the rocAL API for graphs of ```` ### running the application ```` -rocAL_performance_tests [test image folder] [image width] [image height] [test case] [batch size] [0 for CPU, 1 for GPU] [0 for grayscale, 1 for RGB] +./performance_tests [test image folder] [image width] [image height] [test case] [batch size] [0 for CPU, 1 for GPU] [0 for grayscale, 1 for RGB] ```` diff --git a/tests/cpp_api_tests/rocAL_performance_tests/rocAL_performance_tests.cpp b/tests/cpp_api/performance_tests/performance_tests.cpp similarity index 98% rename from tests/cpp_api_tests/rocAL_performance_tests/rocAL_performance_tests.cpp rename to tests/cpp_api/performance_tests/performance_tests.cpp index 0ebfe69d6..37b1f1a99 100644 --- a/tests/cpp_api_tests/rocAL_performance_tests/rocAL_performance_tests.cpp +++ b/tests/cpp_api/performance_tests/performance_tests.cpp @@ -51,7 +51,7 @@ int test(int test_case, const char* path, int rgb, int processing_device, int wi int main(int argc, const char** argv) { // check command-line usage const int MIN_ARG_COUNT = 2; - printf("Usage: rocal_performance_tests \n"); + printf("Usage: performance_tests \n"); if (argc < MIN_ARG_COUNT) return -1; diff --git a/tests/cpp_api_tests/rocAL_performance_tests_with_depth/CMakeLists.txt b/tests/cpp_api/performance_tests_with_depth/CMakeLists.txt similarity index 98% rename from tests/cpp_api_tests/rocAL_performance_tests_with_depth/CMakeLists.txt rename to tests/cpp_api/performance_tests_with_depth/CMakeLists.txt index 7276c54b8..6c6483468 100644 --- a/tests/cpp_api_tests/rocAL_performance_tests_with_depth/CMakeLists.txt +++ b/tests/cpp_api/performance_tests_with_depth/CMakeLists.txt @@ -26,7 +26,7 @@ cmake_minimum_required (VERSION 3.5) -project (rocal_performance_tests_with_depth) +project (performance_tests_with_depth) set(CMAKE_CXX_STANDARD 14) diff --git a/tests/cpp_api_tests/rocAL_performance_tests_with_depth/README.md b/tests/cpp_api/performance_tests_with_depth/README.md similarity index 72% rename from tests/cpp_api_tests/rocAL_performance_tests_with_depth/README.md rename to tests/cpp_api/performance_tests_with_depth/README.md index b1c190e26..05352eb6b 100644 --- a/tests/cpp_api_tests/rocAL_performance_tests_with_depth/README.md +++ b/tests/cpp_api/performance_tests_with_depth/README.md @@ -6,7 +6,7 @@ This is very similar to the rocAL Performance Tests app except it takes an extra ### Pre-requisites * Ubuntu Linux, [version `16.04` or later](https://www.microsoft.com/software-download/windows10) -* rocAL library (Part of the MIVisionX toolkit) +* rocAL library * [OpenCV 3.1](https://github.com/opencv/opencv/releases) or higher * ROCm Performance Primitives (RPP) @@ -19,5 +19,5 @@ This is very similar to the rocAL Performance Tests app except it takes an extra ```` ### running the application ```` -rocAL_performance_tests_with_depth [test image folder] [image width] [image height] [test case] [batch size] [graph depth] [0 for CPU, 1 for GPU] [0 for grayscale, 1 for RGB] +performance_tests_with_depth [test image folder] [image width] [image height] [test case] [batch size] [graph depth] [0 for CPU, 1 for GPU] [0 for grayscale, 1 for RGB] ```` diff --git a/tests/cpp_api_tests/rocAL_performance_tests_with_depth/rocAL_performance_tests_with_depth.cpp b/tests/cpp_api/performance_tests_with_depth/performance_tests_with_depth.cpp similarity index 100% rename from tests/cpp_api_tests/rocAL_performance_tests_with_depth/rocAL_performance_tests_with_depth.cpp rename to tests/cpp_api/performance_tests_with_depth/performance_tests_with_depth.cpp diff --git a/tests/cpp_api_tests/rocAL_dataloader_tf/CMakeLists.txt b/tests/cpp_api/unit_tests/CMakeLists.txt similarity index 99% rename from tests/cpp_api_tests/rocAL_dataloader_tf/CMakeLists.txt rename to tests/cpp_api/unit_tests/CMakeLists.txt index 2b5b286fa..a76fde8e5 100644 --- a/tests/cpp_api_tests/rocAL_dataloader_tf/CMakeLists.txt +++ b/tests/cpp_api/unit_tests/CMakeLists.txt @@ -25,8 +25,7 @@ ################################################################################ cmake_minimum_required(VERSION 3.5) -project(rocal_dataloader_tf) - +project (unit_tests) set(CMAKE_CXX_STANDARD 14) # ROCm Path @@ -46,7 +45,6 @@ list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/../../cmake) find_package(OpenCV QUIET) find_package(AMDRPP QUIET) - include_directories(${ROCM_PATH}/${CMAKE_INSTALL_INCLUDEDIR}/rocal) link_directories(${ROCM_PATH}/lib) file(GLOB My_Source_Files ./*.cpp) diff --git a/tests/cpp_api_tests/rocAL_unittests/README.md b/tests/cpp_api/unit_tests/README.md similarity index 74% rename from tests/cpp_api_tests/rocAL_unittests/README.md rename to tests/cpp_api/unit_tests/README.md index dd4247fde..64353fe8e 100644 --- a/tests/cpp_api_tests/rocAL_unittests/README.md +++ b/tests/cpp_api/unit_tests/README.md @@ -5,7 +5,7 @@ This application can be used to verify the functionality of the API offered by r ### Pre-requisites * Ubuntu Linux, [version `16.04` or later](https://www.microsoft.com/software-download/windows10) -* rocAL library (Part of the MIVisionX toolkit) +* rocAL library * [OpenCV 3.4+](https://github.com/opencv/opencv/releases/tag/3.4.0) * ROCm Performance Primitives (RPP) * Python3 @@ -23,16 +23,16 @@ make ## Running the application ``` -./rocAL_unittests +./unit_tests -Usage: ./rocAL_unittests reader-type pipeline-type=1(classification)2(detection)3(keypoints) output_image_name test_case gpu=1/cpu=0 rgb=1/grayscale=0 one_hot_labels=num_of_classes/0 display_all=0(display_last_only)1(display_all) +Usage: ./unit_tests reader-type pipeline-type=1(classification)2(detection)3(keypoints) output_image_name test_case gpu=1/cpu=0 rgb=1/grayscale=0 one_hot_labels=num_of_classes/0 display_all=0(display_last_only)1(display_all) ``` ### Output verification The bash script `testAllScript.sh` can be used to run and dump the outputs for all test cases in rocAL and run the python script to verify the correctness of the generated outputs with the golden outputs. -Input data is available in the following link : [MIVisionX-data](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX-data) +Input data is available in the following link : [MIVisionX-data](https://github.com/ROCm/MIVisionX-data) `export ROCAL_DATA_PATH=` diff --git a/tests/cpp_api_tests/rocAL_unittests/pixel_comparison/image_comparison.py b/tests/cpp_api/unit_tests/pixel_comparison/image_comparison.py similarity index 100% rename from tests/cpp_api_tests/rocAL_unittests/pixel_comparison/image_comparison.py rename to tests/cpp_api/unit_tests/pixel_comparison/image_comparison.py diff --git a/tests/cpp_api/unit_tests/testAllScripts.sh b/tests/cpp_api/unit_tests/testAllScripts.sh new file mode 100755 index 000000000..dc3e4a82f --- /dev/null +++ b/tests/cpp_api/unit_tests/testAllScripts.sh @@ -0,0 +1,177 @@ +#!/bin/bash +cwd=$(pwd) +if [ -d build ];then + sudo rm -rf ./build/* +else + mkdir build +fi +cd build || exit +cmake .. +make -j"$(nproc)" + +if [[ $ROCAL_DATA_PATH == "" ]] +then + echo "Need to export ROCAL_DATA_PATH" + exit +fi + +# Path to inputs and outputs available in MIVisionX-data +image_path=${ROCAL_DATA_PATH}/rocal_data/coco/coco_10_img/train_10images_2017/ +coco_detection_path=${ROCAL_DATA_PATH}/rocal_data/coco/coco_10_img/train_10images_2017/ +tf_classification_path=${ROCAL_DATA_PATH}/rocal_data/tf/classification/ +tf_detection_path=${ROCAL_DATA_PATH}/rocal_data/tf/detection/ +caffe_classification_path=${ROCAL_DATA_PATH}/rocal_data/caffe/classification/ +caffe_detection_path=${ROCAL_DATA_PATH}/rocal_data/caffe/detection/ +caffe2_classification_path=${ROCAL_DATA_PATH}/rocal_data/caffe2/classification/ +caffe2_detection_path=${ROCAL_DATA_PATH}/rocal_data/caffe2/detection/ +mxnet_path=${ROCAL_DATA_PATH}/rocal_data/mxnet/ +output_path=../rocal_unittest_output_folder_$(date +%Y-%m-%d_%H-%M-%S)/ +golden_output_path=${ROCAL_DATA_PATH}/rocal_data/GoldenOutputsTensor/ + +display=0 +device=0 +width=640 +height=480 +device_name="host" +rgb_name=("gray" "rgb") +rgb=1 +dev_start=0 +dev_end=1 +rgb_start=0 +rgb_end=1 + +if [ "$#" -gt 0 ]; then + if [ "$1" -eq 0 ]; then # For only HOST backend + dev_start=0 + dev_end=0 + elif [ "$1" -eq 1 ]; then # For only HIP backend + dev_start=1 + dev_end=1 + elif [ "$1" -eq 2 ]; then # For both HOST and HIP backend + dev_start=0 + dev_end=1 + fi +fi + +if [ "$#" -gt 1 ]; then + if [ "$2" -eq 0 ]; then # For only Greyscale inputs + rgb_start=0 + rgb_end=0 + elif [ "$2" -eq 1 ]; then # For only RGB inputs + rgb_start=1 + rgb_end=1 + elif [ "$2" -eq 2 ]; then # For both RGB and Greyscale inputs + rgb_start=0 + rgb_end=1 + fi +fi + +mkdir "$output_path" + +for ((device=dev_start;device<=dev_end;device++)) +do + if [ $device -eq 1 ] + then + device_name="hip" + echo "Running HIP Backend..." + else + echo "Running HOST Backend..." + fi + for ((rgb=rgb_start;rgb<=rgb_end;rgb++)) + do + # FileSource Reader + ./unit_tests 0 "$image_path" "${output_path}LensCorrection_${rgb_name[$rgb]}_${device_name}" $width $height 45 $device $rgb 0 $display + ./unit_tests 0 "$image_path" "${output_path}Exposure_${rgb_name[$rgb]}_${device_name}" $width $height 46 $device $rgb 0 $display + ./unit_tests 0 "$image_path" "${output_path}Flip_${rgb_name[$rgb]}_${device_name}" $width $height 47 $device $rgb 0 $display + + # FileSource Reader + partial decoder + ./unit_tests 1 "$image_path" "${output_path}Snow_${rgb_name[$rgb]}_${device_name}_FileReader_partial" $width $height 41 $device $rgb 0 $display + ./unit_tests 1 "$image_path" "${output_path}Rain_${rgb_name[$rgb]}_${device_name}_FileReader_partial" $width $height 42 $device $rgb 0 $display + ./unit_tests 1 "$image_path" "${output_path}SNPNoise_${rgb_name[$rgb]}_${device_name}_FileReader_partial" $width $height 40 $device $rgb 0 $display + + # coco detection + ./unit_tests 2 "$coco_detection_path" "${output_path}Gamma_${rgb_name[$rgb]}_${device_name}" $width $height 33 $device $rgb 0 $display + ./unit_tests 2 "$coco_detection_path" "${output_path}Contrast_${rgb_name[$rgb]}_${device_name}" $width $height 34 $device $rgb 0 $display + ./unit_tests 2 "$coco_detection_path" "${output_path}Vignette_${rgb_name[$rgb]}_${device_name}" $width $height 38 $device $rgb 0 $display + + # coco detection + partial decoder + ./unit_tests 3 "$coco_detection_path" "${output_path}Snow_${rgb_name[$rgb]}_${device_name}_coco_partial" $width $height 41 $device $rgb 0 $display + ./unit_tests 3 "$coco_detection_path" "${output_path}Rain_${rgb_name[$rgb]}_${device_name}_coco_partial" $width $height 42 $device $rgb 0 $display + ./unit_tests 3 "$coco_detection_path" "${output_path}SNPNoise_${rgb_name[$rgb]}_${device_name}_coco_partial" $width $height 40 $device $rgb 0 $display + + # tf classification + ./unit_tests 4 "$tf_classification_path" "${output_path}Blend_${rgb_name[$rgb]}_${device_name}" $width $height 36 $device $rgb 0 $display + ./unit_tests 4 "$tf_classification_path" "${output_path}WarpAffine_${rgb_name[$rgb]}_${device_name}" $width $height 37 $device $rgb 0 $display + ./unit_tests 4 "$tf_classification_path" "${output_path}Blur_${rgb_name[$rgb]}_${device_name}" $width $height 35 $device $rgb 0 $display + + # tf detection + ./unit_tests 5 "$tf_detection_path" "${output_path}SNPNoise_${rgb_name[$rgb]}_${device_name}" $width $height 40 $device $rgb 0 $display + ./unit_tests 5 "$tf_detection_path" "${output_path}ColorTemp_${rgb_name[$rgb]}_${device_name}" $width $height 43 $device $rgb 0 $display + ./unit_tests 5 "$tf_detection_path" "${output_path}Fog_${rgb_name[$rgb]}_${device_name}" $width $height 44 $device $rgb 0 $display + + # caffe classification + ./unit_tests 6 "$caffe_classification_path" "${output_path}Rotate_${rgb_name[$rgb]}_${device_name}" $width $height 31 $device $rgb 0 $display + ./unit_tests 6 "$caffe_classification_path" "${output_path}Brightness_${rgb_name[$rgb]}_${device_name}" $width $height 32 $device $rgb 0 $display + ./unit_tests 6 "$caffe_classification_path" "${output_path}Hue_${rgb_name[$rgb]}_${device_name}" $width $height 48 $device $rgb 0 $display + + # caffe detection + ./unit_tests 7 "$caffe_detection_path" "${output_path}Saturation_${rgb_name[$rgb]}_${device_name}" $width $height 49 $device $rgb 0 $display + ./unit_tests 7 "$caffe_detection_path" "${output_path}ColorTwist_${rgb_name[$rgb]}_${device_name}" $width $height 50 $device $rgb 0 $display + ./unit_tests 7 "$caffe_detection_path" "${output_path}Rain_${rgb_name[$rgb]}_${device_name}" $width $height 42 $device $rgb 0 $display + + # caffe2 classification + ./unit_tests 8 "$caffe2_classification_path" "${output_path}CropCenter_${rgb_name[$rgb]}_${device_name}" $width $height 52 $device $rgb 0 $display + ./unit_tests 8 "$caffe2_classification_path" "${output_path}ResizeCropMirror_${rgb_name[$rgb]}_${device_name}" $width $height 53 $device $rgb 0 $display + ./unit_tests 8 "$caffe2_classification_path" "${output_path}Snow_${rgb_name[$rgb]}_${device_name}" $width $height 41 $device $rgb 0 $display + + # caffe2 detection + ./unit_tests 9 "$caffe2_detection_path" "${output_path}FishEye_${rgb_name[$rgb]}_${device_name}" $width $height 10 $device $rgb 0 $display + ./unit_tests 9 "$caffe2_detection_path" "${output_path}Pixelate_${rgb_name[$rgb]}_${device_name}" $width $height 19 $device $rgb 0 $display + ./unit_tests 9 "$caffe2_detection_path" "${output_path}CropCenterCMN_${rgb_name[$rgb]}_${device_name}" $width $height 55 $device $rgb 0 $display + + # mxnet + ./unit_tests 11 "$mxnet_path" "${output_path}Jitter_${rgb_name[$rgb]}_${device_name}" $width $height 39 $device $rgb 0 $display + ./unit_tests 11 "$mxnet_path" "${output_path}Pixelate_${rgb_name[$rgb]}_${device_name}" $width $height 19 $device $rgb 0 $display + ./unit_tests 11 "$mxnet_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_mxnet" $width $height 25 $device $rgb 0 $display + + # CMN + ./unit_tests 0 "$image_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_FileReader" $width $height 25 $device $rgb 0 $display + ./unit_tests 2 "$coco_detection_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_coco" $width $height 25 $device $rgb 0 $display + ./unit_tests 4 "$tf_classification_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_tfClassification" $width $height 25 $device $rgb 0 $display + ./unit_tests 5 "$tf_detection_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_tfDetection" $width $height 25 $device $rgb 0 $display + ./unit_tests 6 "$caffe_classification_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffeClassification" $width $height 25 $device $rgb 0 $display + ./unit_tests 7 "$caffe_detection_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffeDetection" $width $height 25 $device $rgb 0 $display + ./unit_tests 8 "$caffe2_classification_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffe2Classification" $width $height 25 $device $rgb 0 $display + ./unit_tests 9 "$caffe2_detection_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffe2Detection" $width $height 25 $device $rgb 0 $display + ./unit_tests 11 "$mxnet_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_mxnet" $width $height 25 $device $rgb 0 $display + + # crop + ./unit_tests 0 "$image_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_FileReader" $width $height 51 $device $rgb 0 $display + ./unit_tests 2 "$coco_detection_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_coco" $width $height 51 $device $rgb 0 $display + ./unit_tests 4 "$tf_classification_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_tfClassification" $width $height 51 $device $rgb 0 $display + ./unit_tests 5 "$tf_detection_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_tfDetection" $width $height 51 $device $rgb 0 $display + ./unit_tests 6 "$caffe_classification_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffeClassification" $width $height 51 $device $rgb 0 $display + ./unit_tests 7 "$caffe_detection_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffeDetection" $width $height 51 $device $rgb 0 $display + ./unit_tests 8 "$caffe2_classification_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffe2Classification" $width $height 51 $device $rgb 0 $display + ./unit_tests 9 "$caffe2_detection_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffe2Detection" $width $height 51 $device $rgb 0 $display + ./unit_tests 11 "$mxnet_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_mxnet" $width $height 51 $device $rgb 0 $display + + # resize + # Last two parameters are interpolation type and scaling mode + ./unit_tests 0 "$image_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_default_FileReader" $width $height 0 $device $rgb 0 $display 1 0 + ./unit_tests 2 "$coco_detection_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_stretch_coco" $width $height 0 $device $rgb 0 $display 1 1 + ./unit_tests 4 "$tf_classification_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_notsmaller_tfClassification" $width $height 0 $device $rgb 0 $display 1 2 + ./unit_tests 5 "$tf_detection_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_notlarger_tfDetection" $width $height 0 $device $rgb 0 $display 1 3 + ./unit_tests 6 "$caffe_classification_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bicubic_default_caffeClassification" $width $height 0 $device $rgb 0 $display 2 0 + # ./unit_tests 7 "$caffe_detection_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_nearestneighbor_default_caffeDetection" $width $height 0 $device $rgb 0 $display 0 0 + ./unit_tests 8 "$caffe2_classification_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_lanczos_default_caffe2Classification" $width $height 0 $device $rgb 0 $display 3 0 + ./unit_tests 9 "$caffe2_detection_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_triangular_default_caffe2Detection" $width $height 0 $device $rgb 0 $display 5 0 + ./unit_tests 11 "$mxnet_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_gaussian_default_mxnet" $width $height 0 $device $rgb 0 $display 4 0 + + done +done + +pwd + +# Run python script to compare rocAL outputs with golden ouptuts +python3 "$cwd"/pixel_comparison/image_comparison.py "$golden_output_path" "$output_path" diff --git a/tests/cpp_api_tests/rocAL_unittests/rocAL_unittests.cpp b/tests/cpp_api/unit_tests/unit_tests.cpp similarity index 99% rename from tests/cpp_api_tests/rocAL_unittests/rocAL_unittests.cpp rename to tests/cpp_api/unit_tests/unit_tests.cpp index d02e91d3a..212b8e765 100644 --- a/tests/cpp_api_tests/rocAL_unittests/rocAL_unittests.cpp +++ b/tests/cpp_api/unit_tests/unit_tests.cpp @@ -106,7 +106,7 @@ int main(int argc, const char **argv) { // check command-line usage const int MIN_ARG_COUNT = 2; if (argc < MIN_ARG_COUNT) { - printf("Usage: rocal_unittests reader-type output_image_name test_case gpu=1/cpu=0 rgb=1/grayscale=0 one_hot_labels=num_of_classes/0 display_all=0(display_last_only)1(display_all)\n"); + printf("Usage: unit_tests reader-type output_image_name test_case gpu=1/cpu=0 rgb=1/grayscale=0 one_hot_labels=num_of_classes/0 display_all=0(display_last_only)1(display_all)\n"); return -1; } diff --git a/tests/cpp_api_tests/rocAL_video_unittests/CMakeLists.txt b/tests/cpp_api/video_tests/CMakeLists.txt similarity index 98% rename from tests/cpp_api_tests/rocAL_video_unittests/CMakeLists.txt rename to tests/cpp_api/video_tests/CMakeLists.txt index 81bec5c67..65cb4c109 100644 --- a/tests/cpp_api_tests/rocAL_video_unittests/CMakeLists.txt +++ b/tests/cpp_api/video_tests/CMakeLists.txt @@ -25,7 +25,7 @@ ################################################################################ cmake_minimum_required(VERSION 3.5) -project (rocal_video_unittests) +project (video_tests) set(CMAKE_CXX_STANDARD 14) diff --git a/tests/cpp_api_tests/rocAL_video_unittests/README.md b/tests/cpp_api/video_tests/README.md similarity index 83% rename from tests/cpp_api_tests/rocAL_video_unittests/README.md rename to tests/cpp_api/video_tests/README.md index af791c77d..c3fefbec9 100644 --- a/tests/cpp_api_tests/rocAL_video_unittests/README.md +++ b/tests/cpp_api/video_tests/README.md @@ -5,7 +5,7 @@ This application can be used to verify the functionality of the video API offere ### Pre-requisites * Ubuntu Linux, version - `18.04` / `20.04` -* rocAL library (Part of the MIVisionX toolkit) +* rocAL library * [OpenCV 4.6.0](https://github.com/opencv/opencv/releases/tag/4.6.0) * [FFmpeg n4.4.2](https://github.com/FFmpeg/FFmpeg/releases/tag/n4.4.2) * ROCm Performance Primitives (RPP) @@ -21,12 +21,12 @@ The arguments passed to the Video Pipeline can be modified in the bash script [t The outputs will be dumped inside the build/output_frames folder. -The sample video files and folder are available in the In the following path : [video and sequence samples](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX-data/tree/main/rocal_data/video_and_sequence_samples). +The sample video files and folder are available in the In the following path : [video and sequence samples](https://github.com/ROCm/MIVisionX-data/tree/main/rocal_data/video_and_sequence_samples). The data samples can be downloaded from the MIVisionX-data repository. ``` -git clone https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX-data.git +git clone https://github.com/ROCm/MIVisionX-data.git ``` ## Description @@ -45,7 +45,7 @@ INPUT_PATH : Input passed by the user. It can be a video file path, folder path NOTE: * Inputs for cases 1 and 2 - Video file / folder containing videos - * Input for case 3 - Folder containing sequence of images [sample folder](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX-data/tree/main/rocal_data/video_and_sequence_samples/sequence) + * Input for case 3 - Folder containing sequence of images [sample folder](https://github.com/ROCm/MIVisionX-data/tree/main/rocal_data/video_and_sequence_samples/sequence) READER_CASE : Value passed can be 1/2/3 depending upon the selected reader (default value : 1). @@ -83,7 +83,7 @@ ENABLE_SEQUENCE_REARRANGE : If set to true, the frames in each sequence will be **Example 1: Video Reader** -> ./testScript.sh <[path/to/test_frame_num.mp4](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX-data/blob/main/rocal_data/video_and_sequence_samples/test_frame/test_frame_num.mp4)> 1 +> ./testScript.sh <[path/to/test_frame_num.mp4](https://github.com/ROCm/MIVisionX-data/blob/main/rocal_data/video_and_sequence_samples/test_frame/test_frame_num.mp4)> 1 Arguments to be modified in testScript.sh to get the following output: @@ -103,7 +103,7 @@ Also RESIZE_WIDTH and RESIZE_HEIGHT can be changed in testScript.sh **Example 2: Sequence Reader** -> ./testScript.sh <[path/to/sequence_folder](https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX-data/tree/main/rocal_data/video_and_sequence_samples/sequence)> 3 +> ./testScript.sh <[path/to/sequence_folder](https://github.com/ROCm/MIVisionX-data/tree/main/rocal_data/video_and_sequence_samples/sequence)> 3 ![sequence_reader.png](./samples/sequence_reader.png) @@ -125,7 +125,7 @@ ENABLE_SEQUENCE_REARRANGE=1 ![sequence_rearrange.png](./samples/sequence_rearrange.png) -New Sequence order : (2, 1, 1, 0), The order can be changed directly in rocAL_video_unittests.cpp file. The values specified in the order can only be in the range [0,sequence_length) +New Sequence order : (2, 1, 1, 0), The order can be changed directly in video_tests.cpp file. The values specified in the order can only be in the range [0,sequence_length) **NOTE**: diff --git a/tests/cpp_api_tests/rocAL_video_unittests/samples/sequence_reader.png b/tests/cpp_api/video_tests/samples/sequence_reader.png similarity index 100% rename from tests/cpp_api_tests/rocAL_video_unittests/samples/sequence_reader.png rename to tests/cpp_api/video_tests/samples/sequence_reader.png diff --git a/tests/cpp_api_tests/rocAL_video_unittests/samples/sequence_rearrange.png b/tests/cpp_api/video_tests/samples/sequence_rearrange.png similarity index 100% rename from tests/cpp_api_tests/rocAL_video_unittests/samples/sequence_rearrange.png rename to tests/cpp_api/video_tests/samples/sequence_rearrange.png diff --git a/tests/cpp_api_tests/rocAL_video_unittests/samples/video_reader.png b/tests/cpp_api/video_tests/samples/video_reader.png similarity index 100% rename from tests/cpp_api_tests/rocAL_video_unittests/samples/video_reader.png rename to tests/cpp_api/video_tests/samples/video_reader.png diff --git a/tests/cpp_api_tests/rocAL_video_unittests/testScript.sh b/tests/cpp_api/video_tests/testScript.sh similarity index 86% rename from tests/cpp_api_tests/rocAL_video_unittests/testScript.sh rename to tests/cpp_api/video_tests/testScript.sh index 728c90c3e..38f632d11 100755 --- a/tests/cpp_api_tests/rocAL_video_unittests/testScript.sh +++ b/tests/cpp_api/video_tests/testScript.sh @@ -51,12 +51,12 @@ FILELIST_FRAMENUM=1 # enables file number or timestamps parsing for tex ENABLE_METADATA=0 # outputs labels and names of the associated frames ENABLE_FRAME_NUMBER=0 # outputs the starting frame numbers of the sequences in the batch ENABLE_TIMESTAMPS=0 # outputs timestamps of the frames in the batch -ENABLE_SEQUENCE_REARRANGE=0 # rearranges the frames in the sequence NOTE: The order needs to be set in the rocAL_video_unittests.cpp +ENABLE_SEQUENCE_REARRANGE=0 # rearranges the frames in the sequence NOTE: The order needs to be set in the video_tests.cpp -echo ./rocal_video_unittests "$INPUT_PATH" $READER_CASE $DEVICE $HARDWARE_DECODE_MODE $BATCH_SIZE $SEQUENCE_LENGTH $STEP $STRIDE \ +echo ./video_tests "$INPUT_PATH" $READER_CASE $DEVICE $HARDWARE_DECODE_MODE $BATCH_SIZE $SEQUENCE_LENGTH $STEP $STRIDE \ $RGB $SAVE_FRAMES $SHUFFLE $RESIZE_WIDTH $RESIZE_HEIGHT $FILELIST_FRAMENUM \ $ENABLE_METADATA $ENABLE_FRAME_NUMBER $ENABLE_TIMESTAMPS $ENABLE_SEQUENCE_REARRANGE -./rocal_video_unittests "$INPUT_PATH" $READER_CASE $DEVICE $HARDWARE_DECODE_MODE $BATCH_SIZE $SEQUENCE_LENGTH $STEP $STRIDE \ +./video_tests "$INPUT_PATH" $READER_CASE $DEVICE $HARDWARE_DECODE_MODE $BATCH_SIZE $SEQUENCE_LENGTH $STEP $STRIDE \ $RGB $SAVE_FRAMES $SHUFFLE $RESIZE_WIDTH $RESIZE_HEIGHT $FILELIST_FRAMENUM \ $ENABLE_METADATA $ENABLE_FRAME_NUMBER $ENABLE_TIMESTAMPS $ENABLE_SEQUENCE_REARRANGE diff --git a/tests/cpp_api_tests/rocAL_video_unittests/rocAL_video_unittests.cpp b/tests/cpp_api/video_tests/video_tests.cpp similarity index 97% rename from tests/cpp_api_tests/rocAL_video_unittests/rocAL_video_unittests.cpp rename to tests/cpp_api/video_tests/video_tests.cpp index dae9a3128..16996446f 100644 --- a/tests/cpp_api_tests/rocAL_video_unittests/rocAL_video_unittests.cpp +++ b/tests/cpp_api/video_tests/video_tests.cpp @@ -67,7 +67,7 @@ int main(int argc, const char **argv) { // check command-line usage const int MIN_ARG_COUNT = 2; if (argc < MIN_ARG_COUNT) { - printf("Usage: rocal_video_unittests \n"); + printf("Usage: video_tests \n"); return -1; } diff --git a/tests/cpp_api_tests/rocAL_unittests/testAllScripts.sh b/tests/cpp_api_tests/rocAL_unittests/testAllScripts.sh deleted file mode 100755 index b41bd95b3..000000000 --- a/tests/cpp_api_tests/rocAL_unittests/testAllScripts.sh +++ /dev/null @@ -1,177 +0,0 @@ -#!/bin/bash -cwd=$(pwd) -if [ -d build ];then - sudo rm -rf ./build/* -else - mkdir build -fi -cd build || exit -cmake .. -make -j"$(nproc)" - -if [[ $ROCAL_DATA_PATH == "" ]] -then - echo "Need to export ROCAL_DATA_PATH" - exit -fi - -# Path to inputs and outputs available in MIVisionX-data -image_path=${ROCAL_DATA_PATH}/rocal_data/coco/coco_10_img/train_10images_2017/ -coco_detection_path=${ROCAL_DATA_PATH}/rocal_data/coco/coco_10_img/train_10images_2017/ -tf_classification_path=${ROCAL_DATA_PATH}/rocal_data/tf/classification/ -tf_detection_path=${ROCAL_DATA_PATH}/rocal_data/tf/detection/ -caffe_classification_path=${ROCAL_DATA_PATH}/rocal_data/caffe/classification/ -caffe_detection_path=${ROCAL_DATA_PATH}/rocal_data/caffe/detection/ -caffe2_classification_path=${ROCAL_DATA_PATH}/rocal_data/caffe2/classification/ -caffe2_detection_path=${ROCAL_DATA_PATH}/rocal_data/caffe2/detection/ -mxnet_path=${ROCAL_DATA_PATH}/rocal_data/mxnet/ -output_path=../rocal_unittest_output_folder_$(date +%Y-%m-%d_%H-%M-%S)/ -golden_output_path=${ROCAL_DATA_PATH}/rocal_data/GoldenOutputsTensor/ - -display=0 -device=0 -width=640 -height=480 -device_name="host" -rgb_name=("gray" "rgb") -rgb=1 -dev_start=0 -dev_end=1 -rgb_start=0 -rgb_end=1 - -if [ "$#" -gt 0 ]; then - if [ "$1" -eq 0 ]; then # For only HOST backend - dev_start=0 - dev_end=0 - elif [ "$1" -eq 1 ]; then # For only HIP backend - dev_start=1 - dev_end=1 - elif [ "$1" -eq 2 ]; then # For both HOST and HIP backend - dev_start=0 - dev_end=1 - fi -fi - -if [ "$#" -gt 1 ]; then - if [ "$2" -eq 0 ]; then # For only Greyscale inputs - rgb_start=0 - rgb_end=0 - elif [ "$2" -eq 1 ]; then # For only RGB inputs - rgb_start=1 - rgb_end=1 - elif [ "$2" -eq 2 ]; then # For both RGB and Greyscale inputs - rgb_start=0 - rgb_end=1 - fi -fi - -mkdir "$output_path" - -for ((device=dev_start;device<=dev_end;device++)) -do - if [ $device -eq 1 ] - then - device_name="hip" - echo "Running HIP Backend..." - else - echo "Running HOST Backend..." - fi - for ((rgb=rgb_start;rgb<=rgb_end;rgb++)) - do - # FileSource Reader - ./rocal_unittests 0 "$image_path" "${output_path}LensCorrection_${rgb_name[$rgb]}_${device_name}" $width $height 45 $device $rgb 0 $display - ./rocal_unittests 0 "$image_path" "${output_path}Exposure_${rgb_name[$rgb]}_${device_name}" $width $height 46 $device $rgb 0 $display - ./rocal_unittests 0 "$image_path" "${output_path}Flip_${rgb_name[$rgb]}_${device_name}" $width $height 47 $device $rgb 0 $display - - # FileSource Reader + partial decoder - ./rocal_unittests 1 "$image_path" "${output_path}Snow_${rgb_name[$rgb]}_${device_name}_FileReader_partial" $width $height 41 $device $rgb 0 $display - ./rocal_unittests 1 "$image_path" "${output_path}Rain_${rgb_name[$rgb]}_${device_name}_FileReader_partial" $width $height 42 $device $rgb 0 $display - ./rocal_unittests 1 "$image_path" "${output_path}SNPNoise_${rgb_name[$rgb]}_${device_name}_FileReader_partial" $width $height 40 $device $rgb 0 $display - - # coco detection - ./rocal_unittests 2 "$coco_detection_path" "${output_path}Gamma_${rgb_name[$rgb]}_${device_name}" $width $height 33 $device $rgb 0 $display - ./rocal_unittests 2 "$coco_detection_path" "${output_path}Contrast_${rgb_name[$rgb]}_${device_name}" $width $height 34 $device $rgb 0 $display - ./rocal_unittests 2 "$coco_detection_path" "${output_path}Vignette_${rgb_name[$rgb]}_${device_name}" $width $height 38 $device $rgb 0 $display - - # coco detection + partial decoder - ./rocal_unittests 3 "$coco_detection_path" "${output_path}Snow_${rgb_name[$rgb]}_${device_name}_coco_partial" $width $height 41 $device $rgb 0 $display - ./rocal_unittests 3 "$coco_detection_path" "${output_path}Rain_${rgb_name[$rgb]}_${device_name}_coco_partial" $width $height 42 $device $rgb 0 $display - ./rocal_unittests 3 "$coco_detection_path" "${output_path}SNPNoise_${rgb_name[$rgb]}_${device_name}_coco_partial" $width $height 40 $device $rgb 0 $display - - # tf classification - ./rocal_unittests 4 "$tf_classification_path" "${output_path}Blend_${rgb_name[$rgb]}_${device_name}" $width $height 36 $device $rgb 0 $display - ./rocal_unittests 4 "$tf_classification_path" "${output_path}WarpAffine_${rgb_name[$rgb]}_${device_name}" $width $height 37 $device $rgb 0 $display - ./rocal_unittests 4 "$tf_classification_path" "${output_path}Blur_${rgb_name[$rgb]}_${device_name}" $width $height 35 $device $rgb 0 $display - - # tf detection - ./rocal_unittests 5 "$tf_detection_path" "${output_path}SNPNoise_${rgb_name[$rgb]}_${device_name}" $width $height 40 $device $rgb 0 $display - ./rocal_unittests 5 "$tf_detection_path" "${output_path}ColorTemp_${rgb_name[$rgb]}_${device_name}" $width $height 43 $device $rgb 0 $display - ./rocal_unittests 5 "$tf_detection_path" "${output_path}Fog_${rgb_name[$rgb]}_${device_name}" $width $height 44 $device $rgb 0 $display - - # caffe classification - ./rocal_unittests 6 "$caffe_classification_path" "${output_path}Rotate_${rgb_name[$rgb]}_${device_name}" $width $height 31 $device $rgb 0 $display - ./rocal_unittests 6 "$caffe_classification_path" "${output_path}Brightness_${rgb_name[$rgb]}_${device_name}" $width $height 32 $device $rgb 0 $display - ./rocal_unittests 6 "$caffe_classification_path" "${output_path}Hue_${rgb_name[$rgb]}_${device_name}" $width $height 48 $device $rgb 0 $display - - # caffe detection - ./rocal_unittests 7 "$caffe_detection_path" "${output_path}Saturation_${rgb_name[$rgb]}_${device_name}" $width $height 49 $device $rgb 0 $display - ./rocal_unittests 7 "$caffe_detection_path" "${output_path}ColorTwist_${rgb_name[$rgb]}_${device_name}" $width $height 50 $device $rgb 0 $display - ./rocal_unittests 7 "$caffe_detection_path" "${output_path}Rain_${rgb_name[$rgb]}_${device_name}" $width $height 42 $device $rgb 0 $display - - # caffe2 classification - ./rocal_unittests 8 "$caffe2_classification_path" "${output_path}CropCenter_${rgb_name[$rgb]}_${device_name}" $width $height 52 $device $rgb 0 $display - ./rocal_unittests 8 "$caffe2_classification_path" "${output_path}ResizeCropMirror_${rgb_name[$rgb]}_${device_name}" $width $height 53 $device $rgb 0 $display - ./rocal_unittests 8 "$caffe2_classification_path" "${output_path}Snow_${rgb_name[$rgb]}_${device_name}" $width $height 41 $device $rgb 0 $display - - # caffe2 detection - ./rocal_unittests 9 "$caffe2_detection_path" "${output_path}FishEye_${rgb_name[$rgb]}_${device_name}" $width $height 10 $device $rgb 0 $display - ./rocal_unittests 9 "$caffe2_detection_path" "${output_path}Pixelate_${rgb_name[$rgb]}_${device_name}" $width $height 19 $device $rgb 0 $display - ./rocal_unittests 9 "$caffe2_detection_path" "${output_path}CropCenterCMN_${rgb_name[$rgb]}_${device_name}" $width $height 55 $device $rgb 0 $display - - # mxnet - ./rocal_unittests 11 "$mxnet_path" "${output_path}Jitter_${rgb_name[$rgb]}_${device_name}" $width $height 39 $device $rgb 0 $display - ./rocal_unittests 11 "$mxnet_path" "${output_path}Pixelate_${rgb_name[$rgb]}_${device_name}" $width $height 19 $device $rgb 0 $display - ./rocal_unittests 11 "$mxnet_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_mxnet" $width $height 25 $device $rgb 0 $display - - # CMN - ./rocal_unittests 0 "$image_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_FileReader" $width $height 25 $device $rgb 0 $display - ./rocal_unittests 2 "$coco_detection_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_coco" $width $height 25 $device $rgb 0 $display - ./rocal_unittests 4 "$tf_classification_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_tfClassification" $width $height 25 $device $rgb 0 $display - ./rocal_unittests 5 "$tf_detection_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_tfDetection" $width $height 25 $device $rgb 0 $display - ./rocal_unittests 6 "$caffe_classification_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffeClassification" $width $height 25 $device $rgb 0 $display - ./rocal_unittests 7 "$caffe_detection_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffeDetection" $width $height 25 $device $rgb 0 $display - ./rocal_unittests 8 "$caffe2_classification_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffe2Classification" $width $height 25 $device $rgb 0 $display - ./rocal_unittests 9 "$caffe2_detection_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffe2Detection" $width $height 25 $device $rgb 0 $display - ./rocal_unittests 11 "$mxnet_path" "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_mxnet" $width $height 25 $device $rgb 0 $display - - # crop - ./rocal_unittests 0 "$image_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_FileReader" $width $height 51 $device $rgb 0 $display - ./rocal_unittests 2 "$coco_detection_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_coco" $width $height 51 $device $rgb 0 $display - ./rocal_unittests 4 "$tf_classification_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_tfClassification" $width $height 51 $device $rgb 0 $display - ./rocal_unittests 5 "$tf_detection_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_tfDetection" $width $height 51 $device $rgb 0 $display - ./rocal_unittests 6 "$caffe_classification_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffeClassification" $width $height 51 $device $rgb 0 $display - ./rocal_unittests 7 "$caffe_detection_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffeDetection" $width $height 51 $device $rgb 0 $display - ./rocal_unittests 8 "$caffe2_classification_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffe2Classification" $width $height 51 $device $rgb 0 $display - ./rocal_unittests 9 "$caffe2_detection_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffe2Detection" $width $height 51 $device $rgb 0 $display - ./rocal_unittests 11 "$mxnet_path" "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_mxnet" $width $height 51 $device $rgb 0 $display - - # resize - # Last two parameters are interpolation type and scaling mode - ./rocal_unittests 0 "$image_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_default_FileReader" $width $height 0 $device $rgb 0 $display 1 0 - ./rocal_unittests 2 "$coco_detection_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_stretch_coco" $width $height 0 $device $rgb 0 $display 1 1 - ./rocal_unittests 4 "$tf_classification_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_notsmaller_tfClassification" $width $height 0 $device $rgb 0 $display 1 2 - ./rocal_unittests 5 "$tf_detection_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_notlarger_tfDetection" $width $height 0 $device $rgb 0 $display 1 3 - ./rocal_unittests 6 "$caffe_classification_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bicubic_default_caffeClassification" $width $height 0 $device $rgb 0 $display 2 0 - # ./rocal_unittests 7 "$caffe_detection_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_nearestneighbor_default_caffeDetection" $width $height 0 $device $rgb 0 $display 0 0 - ./rocal_unittests 8 "$caffe2_classification_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_lanczos_default_caffe2Classification" $width $height 0 $device $rgb 0 $display 3 0 - ./rocal_unittests 9 "$caffe2_detection_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_triangular_default_caffe2Detection" $width $height 0 $device $rgb 0 $display 5 0 - ./rocal_unittests 11 "$mxnet_path" "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_gaussian_default_mxnet" $width $height 0 $device $rgb 0 $display 4 0 - - done -done - -pwd - -# Run python script to compare rocAL outputs with golden ouptuts -python3 "$cwd"/pixel_comparison/image_comparison.py "$golden_output_path" "$output_path" diff --git a/tests/python_api/README.md b/tests/python_api/README.md new file mode 100644 index 000000000..cdffb1744 --- /dev/null +++ b/tests/python_api/README.md @@ -0,0 +1,105 @@ +## Set environmental variables +`export ROCAL_DATA_PATH=/Absolute/Path/Of/MIVisionX-data/`` +### NOTE: Refer parse_config.py for more info on other args. This script is used with the `readers_test_file.sh` and `unit_tests.sh` + +## Reader Pipeline Tests: +* To test all the reader pipelines with a single script. The `readers_test_file.sh` tests the following cases: + * unit test + * coco reader + * caffe reader + * caffe2 reader + * tf classification reader + * tf detection reader +* The default value of number of gpu's is "1" & display is "ON" by default +`./readers_test_file.sh` + +### Options: +* To test a single reader / multiple reader pipelines: Use the same script `readers_test_file.sh` as above and make the respective " Pipeline " to test equal to "1" + +``` +unit_test = 1 +coco_reader = 0 +``` + +* Example : To run COCO Pipeline +``` +unit_test=0 +coco_reader=1 +caffe_reader=0 +caffe2_reader=0 +tf_classification_reader=0 +tf_detection_reader=0 +``` + +* To set options: + * Number of GPUs `-n` + * display `-d` + * backend `-b` +``` +./readers_test_file.sh -n -d +./readers_test_file.sh -n "1" -d "false" -b "cpu" +``` + +## Augmentation + Reader Tests: +* This runs all augmentations and readers and compare them with golden outputs for both backends (HIP/HOST) + +`./unit_tests.sh` + +* This test also runs the `image_comaprison.py` script at the end to compare the results with the golden outputs from [MIVisionX-data](https://www.github.com/ROCm/MIVisionX-data). + +### Command line for individual files: +* Test a single reader pipeline +* Example: COCO Pipeline +``` + # Mention the number of gpus + gpus_per_node=4 + + # Mention Batch Size + batch_size=10 + + # python version + ver=$(python -V 2>&1 | sed 's/.* \([0-9]\).\([0-9]\).*/\1\.\2/') + + # Mention dataset_path + data_dir=$ROCAL_DATA_PATH/coco/coco_10_img/val_10images_2017/ + + # Mention json path + json_path=$ROCAL_DATA_PATH/coco/coco_10_img/annotations/instances_val2017.json + + # coco_reader.py + # By default : cpu backend, NCHW format , fp32 + # Annotation must be a json file + + # For printing all the arguments that can be passed from user + python$ver coco_reader.py -h + + python$ver coco_reader.py --image-dataset-path $data_dir --json-path $json_path --batch-size $batch_size --display --rocal-gpu --NHWC \ + --local-rank 0 --world-size $gpus_per_node --num-threads 1 --num-epochs 1 2>&1 | tee -a run.log.coco_reader.txt +``` +## Decoder Test: + +This test runs a simple decoder pipeline making use of a image file reader and resizing the given input images to 300x300. + +It uses the [AMD-TinyDataSet](../../data/images/AMD-tinyDataSet/) by default, unless otherwise specified by user. + +### Options: +```backend: +Input image folder: +``` + +* Usage: +```python3 decoder.py +python3 decoder.py gpu +``` + +## External source reader test: + +This test runs a pipeline making use of the external source reader in 3 different modes. It uses coco2017 images by default. +* Mode 0 - Filename +* Mode 1 - Raw compressed images +* Mode 2 - JPEG reader with OpenCV + +* Usage: +``` +python3 external_source_reader.py +``` \ No newline at end of file diff --git a/rocAL_pybind/examples/rocAL_api_caffe2_reader.py b/tests/python_api/caffe2_reader.py similarity index 95% rename from rocAL_pybind/examples/rocAL_api_caffe2_reader.py rename to tests/python_api/caffe2_reader.py index 4caf647d9..e8b51cadf 100644 --- a/rocAL_pybind/examples/rocAL_api_caffe2_reader.py +++ b/tests/python_api/caffe2_reader.py @@ -38,7 +38,7 @@ def draw_patches(img, idx, bboxes=None, args=None): image = image.transpose([1, 2, 0]) image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) if args.classification: - cv2.imwrite("OUTPUT_FOLDER/CAFFE2_READER/CLASSIFICATION/" + + cv2.imwrite("output_folder/caffe2_reader/classification/" + str(idx)+"_"+"train"+".png", image) else: if bboxes is not None: @@ -49,7 +49,7 @@ def draw_patches(img, idx, bboxes=None, args=None): thickness = 2 image = cv2.rectangle(image, (int(loc_[0]), int(loc_[1])), (int( (loc_[2])), int((loc_[3]))), color, thickness) - cv2.imwrite("OUTPUT_FOLDER/CAFFE2_READER/DETECTION/" + + cv2.imwrite("output_folder/caffe2_reader/detection/" + str(idx)+"_"+"train"+".png", image) @@ -70,9 +70,9 @@ def main(): print("num_classes:: ", num_classes) try: if args.classification: - path = "OUTPUT_FOLDER/CAFFE2_READER/CLASSIFICATION/" + path = "output_folder/caffe2_reader/classification/" else: - path = "OUTPUT_FOLDER/CAFFE2_READER/DETECTION/" + path = "output_folder/caffe2_reader/detection/" isExist = os.path.exists(path) if not isExist: os.makedirs(path) diff --git a/rocAL_pybind/examples/rocAL_api_caffe_reader.py b/tests/python_api/caffe_reader.py similarity index 95% rename from rocAL_pybind/examples/rocAL_api_caffe_reader.py rename to tests/python_api/caffe_reader.py index 9cec39c82..16f820501 100644 --- a/rocAL_pybind/examples/rocAL_api_caffe_reader.py +++ b/tests/python_api/caffe_reader.py @@ -38,7 +38,7 @@ def draw_patches(img, idx, bboxes=None, args=None): image = image.transpose([1, 2, 0]) image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) if args.classification: - cv2.imwrite("OUTPUT_FOLDER/CAFFE_READER/CLASSIFICATION/" + + cv2.imwrite("output_folder/caffe_reader/classification/" + str(idx)+"_"+"train"+".png", image) else: if bboxes is not None: @@ -49,7 +49,7 @@ def draw_patches(img, idx, bboxes=None, args=None): thickness = 2 image = cv2.rectangle(image, (int(loc_[0]), int(loc_[1])), (int( (loc_[2])), int((loc_[3]))), color, thickness) - cv2.imwrite("OUTPUT_FOLDER/CAFFE_READER/DETECTION/" + + cv2.imwrite("output_folder/caffe_reader/detection/" + str(idx)+"_"+"train"+".png", image) def main(): @@ -68,9 +68,9 @@ def main(): num_classes = len(next(os.walk(image_path))[1]) try: if args.classification: - path = "OUTPUT_FOLDER/CAFFE_READER/CLASSIFICATION/" + path = "output_folder/caffe_reader/classification/" else: - path = "OUTPUT_FOLDER/CAFFE_READER/DETECTION/" + path = "output_folder/caffe_reader/detection/" isExist = os.path.exists(path) if not isExist: os.makedirs(path) diff --git a/rocAL_pybind/examples/rocAL_api_coco_pipeline.py b/tests/python_api/coco_reader.py similarity index 99% rename from rocAL_pybind/examples/rocAL_api_coco_pipeline.py rename to tests/python_api/coco_reader.py index 79baaadd4..ce373abc0 100755 --- a/rocAL_pybind/examples/rocAL_api_coco_pipeline.py +++ b/tests/python_api/coco_reader.py @@ -138,7 +138,7 @@ def draw_patches(img, idx, bboxes, device, dtype, layout): image = cv2.UMat(image).get() image = cv2.rectangle(image, (int(loc_[0]), int(loc_[1])), (int( (loc_[2])), int((loc_[3]))), color, thickness) - cv2.imwrite("OUTPUT_FOLDER/COCO_READER/" + + cv2.imwrite("output_folder/coco_reader/" + str(idx)+"_"+"train"+".png", image) @@ -157,7 +157,7 @@ def main(): tensor_format = types.NHWC if args.NHWC else types.NCHW tensor_dtype = types.FLOAT16 if args.fp16 else types.FLOAT try: - path = "OUTPUT_FOLDER/COCO_READER/" + path = "output_folder/coco_reader/" isExist = os.path.exists(path) if not isExist: os.makedirs(path) diff --git a/docs/examples/image_processing/decoder.py b/tests/python_api/decoder.py similarity index 97% rename from docs/examples/image_processing/decoder.py rename to tests/python_api/decoder.py index 073fa383c..f2d51124f 100644 --- a/docs/examples/image_processing/decoder.py +++ b/tests/python_api/decoder.py @@ -9,7 +9,7 @@ import cupy as cp seed = 1549361629 -image_dir = "../../../data/images/AMD-tinyDataSet/" +image_dir = "../../data/images/AMD-tinyDataSet/" batch_size = 4 gpu_id = 0 diff --git a/rocAL_pybind/examples/rocAL_api_external_source_reader_example.py b/tests/python_api/external_source_reader.py similarity index 96% rename from rocAL_pybind/examples/rocAL_api_external_source_reader_example.py rename to tests/python_api/external_source_reader.py index a871d78e5..772209254 100644 --- a/rocAL_pybind/examples/rocAL_api_external_source_reader_example.py +++ b/tests/python_api/external_source_reader.py @@ -13,24 +13,24 @@ def main(): batch_size = 3 data_dir = os.environ["ROCAL_DATA_PATH"] + \ - "/coco/coco_10_img/train_10images_2017/" + "rocal_data/coco/coco_10_img/train_10images_2017/" device = "cpu" try: - path_mode0 = "OUTPUT_IMAGES_PYTHON/EXTERNAL_SOURCE_READER/MODE0/" + path_mode0 = "output_folder/external_source_reader/mode0/" isExist = os.path.exists(path_mode0) if not isExist: os.makedirs(path_mode0) except OSError as error: print(error) try: - path_mode1 = "OUTPUT_IMAGES_PYTHON/EXTERNAL_SOURCE_READER/MODE1/" + path_mode1 = "output_folder/external_source_reader/mode1/" isExist = os.path.exists(path_mode1) if not isExist: os.makedirs(path_mode1) except OSError as error: print(error) try: - path_mode2 = "OUTPUT_IMAGES_PYTHON/EXTERNAL_SOURCE_READER/MODE2/" + path_mode2 = "output_folder/external_source_reader/mode2/" isExist = os.path.exists(path_mode2) if not isExist: os.makedirs(path_mode2) @@ -44,7 +44,7 @@ def image_dump(img, idx, device="cpu", mode=0): img = (img).astype('uint8') if mode!=2: img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) - cv2.imwrite("OUTPUT_IMAGES_PYTHON/EXTERNAL_SOURCE_READER/MODE" + str(mode) + "/"+ + cv2.imwrite("output_folder/external_source_reader/mode" + str(mode) + "/"+ str(idx)+"_"+"train"+".png", img) ##################### MODE 0 ######################### diff --git a/rocAL_pybind/examples/image_comparison.py b/tests/python_api/image_comparison.py similarity index 100% rename from rocAL_pybind/examples/image_comparison.py rename to tests/python_api/image_comparison.py diff --git a/rocAL_pybind/examples/parse_config.py b/tests/python_api/parse_config.py similarity index 95% rename from rocAL_pybind/examples/parse_config.py rename to tests/python_api/parse_config.py index 0e6f2e9cc..ea919369e 100644 --- a/rocAL_pybind/examples/parse_config.py +++ b/tests/python_api/parse_config.py @@ -92,7 +92,7 @@ def parse_args(): common_group.add_argument('--seed', '-s', type=int, default=random.SystemRandom().randint(0, 2**32 - 1), help='manually set random seed') - # rocAL_api_python_unittest.py related options + # unit_test.py related options python_unit_test = parser.add_argument_group( 'python-unittest', 'python-unittest-related options') python_unit_test.add_argument('--reader-type', '-r', type=str, default="file", @@ -105,17 +105,17 @@ def parse_args(): help='interpolation type used for resize and crop') python_unit_test.add_argument('--scaling-mode', '-sm', type=int, default=0, help='scaling mode type used for resize') - # rocAL_api_coco_pipeline.py related options - coco_pipeline = parser.add_argument_group( + # coco_reader.py related options + coco_reader = parser.add_argument_group( 'coco-pipeline', 'coco-pipeline-related options') - coco_pipeline.add_argument('--json-path', '-json-path', type=str, + coco_reader.add_argument('--json-path', '-json-path', type=str, help='coco dataset json path') - # rocAL_api_caffe_reader.py related options + # caffe_reader.py related options caffe_pipeline = parser.add_argument_group( 'caffe-pipeline', 'caffe-pipeline-related options') caffe_pipeline.add_argument('--detection', '-detection', type=str, help='detection') - # rocAL_api_video_pipeline.py related options + # video_pipeline.py related options video_pipeline = parser.add_argument_group( 'video-pipeline', 'video-pipeline-related options') video_pipeline.add_argument('--video-path', '-video-path', type=str, diff --git a/rocAL_pybind/examples/rocAL_api_pipeline.py b/tests/python_api/pipeline.py similarity index 98% rename from rocAL_pybind/examples/rocAL_api_pipeline.py rename to tests/python_api/pipeline.py index 603dee8a4..a925f2877 100644 --- a/rocAL_pybind/examples/rocAL_api_pipeline.py +++ b/tests/python_api/pipeline.py @@ -39,7 +39,7 @@ def draw_patches(img, idx, device=True, layout="NCHW"): if layout == "NCHW": image = image.transpose([1, 2, 0]) image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) - cv2.imwrite("OUTPUT_FOLDER/FILE_READER/" + + cv2.imwrite("output_folder/file_reader/" + str(idx)+"_"+"train"+".png", image * 255) @@ -48,7 +48,7 @@ def main(): print('Please pass image_folder cpu/gpu batch_size') exit(0) try: - path = "OUTPUT_FOLDER/FILE_READER/" + path = "output_folder/file_reader/" isExist = os.path.exists(path) if not isExist: os.makedirs(path) diff --git a/rocAL_pybind/examples/prefetch_queue_depth/README.md b/tests/python_api/prefetch_queue_depth/README.md similarity index 100% rename from rocAL_pybind/examples/prefetch_queue_depth/README.md rename to tests/python_api/prefetch_queue_depth/README.md diff --git a/rocAL_pybind/examples/prefetch_queue_depth/prefetch_queue_depth.py b/tests/python_api/prefetch_queue_depth/prefetch_queue_depth.py similarity index 100% rename from rocAL_pybind/examples/prefetch_queue_depth/prefetch_queue_depth.py rename to tests/python_api/prefetch_queue_depth/prefetch_queue_depth.py diff --git a/rocAL_pybind/examples/rocAL_api_pytorch_classification_reader.py b/tests/python_api/pytorch_classification_reader.py similarity index 97% rename from rocAL_pybind/examples/rocAL_api_pytorch_classification_reader.py rename to tests/python_api/pytorch_classification_reader.py index 0ac21c72d..ceda16fb0 100644 --- a/rocAL_pybind/examples/rocAL_api_pytorch_classification_reader.py +++ b/tests/python_api/pytorch_classification_reader.py @@ -40,7 +40,7 @@ def draw_patches(img, idx, layout="nchw", dtype="fp32", device="cpu"): if dtype == "fp16": image = image.astype("uint8") image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) - cv2.imwrite("OUTPUT_FOLDER/FILE_READER/" + str(idx) + + cv2.imwrite("output_folder/file_reader/" + str(idx) + "_" + "train" + ".png", image * 255) @@ -49,7 +49,7 @@ def main(): print("Please pass image_folder cpu/gpu batch_size") exit(0) try: - path = "OUTPUT_FOLDER/FILE_READER/" + path = "output_folder/file_reader/" isExist = os.path.exists(path) if not isExist: os.makedirs(path) diff --git a/rocAL_pybind/examples/READERS_TEST_FILE.sh b/tests/python_api/readers_test_file.sh similarity index 88% rename from rocAL_pybind/examples/READERS_TEST_FILE.sh rename to tests/python_api/readers_test_file.sh index 8ef726294..3790ea14d 100755 --- a/rocAL_pybind/examples/READERS_TEST_FILE.sh +++ b/tests/python_api/readers_test_file.sh @@ -94,30 +94,30 @@ ver=$(python3 -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2 #################################################################################################################################### # USER TO MAKE CHANGES HERE FOR TEST # Make the respective " Pipeline " to test equal to 1 -rocAL_api_python_unittest=1 -rocAL_api_coco_pipeline=1 -rocAL_api_caffe_reader=1 -rocAL_api_caffe2_reader=1 -rocAL_api_tf_classification_reader=1 -rocAL_api_tf_detection_pipeline=1 -rocAL_api_video_pipeline=1 +unit_test=1 +coco_reader=1 +caffe_reader=1 +caffe2_reader=1 +tf_classification_reader=1 +tf_detection_reader=1 +video_pipeline=1 #################################################################################################################################### #################################################################################################################################### -if [[ rocAL_api_python_unittest -eq 1 ]]; then +if [[ unit_test -eq 1 ]]; then # Mention dataset_path data_dir=$ROCAL_DATA_PATH/rocal_data/images_jpg/labels_folder/ - # rocAL_api_python_unittest.py + # unit_test.py # By default : cpu backend, NCHW format , fp32 # Please pass image_folder augmentation_nanme in addition to other common args # Refer rocAL_api_python_unitest.py for all augmentation names - python"$ver" rocAL_api_python_unittest.py \ + python"$ver" unit_test.py \ --image-dataset-path $data_dir \ --augmentation-name snow \ --batch-size $batch_size \ @@ -134,7 +134,7 @@ fi #################################################################################################################################### -if [[ rocAL_api_coco_pipeline -eq 1 ]]; then +if [[ coco_reader -eq 1 ]]; then # Mention dataset_path data_dir=$ROCAL_DATA_PATH/rocal_data/coco/coco_10_img/val_10images_2017/ @@ -143,11 +143,11 @@ if [[ rocAL_api_coco_pipeline -eq 1 ]]; then # Mention json path json_path=$ROCAL_DATA_PATH/rocal_data/coco/coco_10_img/annotations/instances_val2017.json - # rocAL_api_coco_pipeline.py + # coco_reader.py # By default : cpu backend, NCHW format , fp32 # Please pass annotation path in addition to other common args # Annotation must be a json file - python"$ver" rocAL_api_coco_pipeline.py \ + python"$ver" coco_reader.py \ --image-dataset-path $data_dir \ --json-path $json_path \ --batch-size $batch_size \ @@ -164,17 +164,17 @@ fi #################################################################################################################################### -if [[ rocAL_api_caffe_reader -eq 1 ]]; then +if [[ caffe_reader -eq 1 ]]; then # Mention dataset_path # Classification data_dir=$ROCAL_DATA_PATH/rocal_data/caffe/classification/ - # rocAL_api_caffe_reader.py + # caffe_reader.py # By default : cpu backend, NCHW format , fp32 # use --classification for Classification / --no-classification for Detection - python"$ver" rocAL_api_caffe_reader.py \ + python"$ver" caffe_reader.py \ --image-dataset-path $data_dir \ --classification \ --batch-size $batch_size \ @@ -192,17 +192,17 @@ fi #################################################################################################################################### -if [[ rocAL_api_caffe_reader -eq 1 ]]; then +if [[ caffe_reader -eq 1 ]]; then # Mention dataset_path # Detection data_dir=$ROCAL_DATA_PATH/rocal_data/caffe/detection/ - # rocAL_api_caffe_reader.py + # caffe_reader.py # By default : cpu backend, NCHW format , fp32 # use --classification for Classification / --no-classification for Detection - python"$ver" rocAL_api_caffe_reader.py \ + python"$ver" caffe_reader.py \ --image-dataset-path $data_dir \ --no-classification \ --batch-size $batch_size \ @@ -220,17 +220,17 @@ fi #################################################################################################################################### -if [[ rocAL_api_caffe2_reader -eq 1 ]]; then +if [[ caffe2_reader -eq 1 ]]; then # Mention dataset_path # Classification data_dir=$ROCAL_DATA_PATH/rocal_data/caffe2/classification/ - # rocAL_api_caffe2_reader.py + # caffe2_reader.py # By default : cpu backend, NCHW format , fp32 # use --classification for Classification / --no-classification for Detection - python"$ver" rocAL_api_caffe2_reader.py \ + python"$ver" caffe2_reader.py \ --image-dataset-path $data_dir \ --classification \ --batch-size $batch_size \ @@ -247,17 +247,17 @@ fi #################################################################################################################################### -if [[ rocAL_api_caffe2_reader -eq 1 ]]; then +if [[ caffe2_reader -eq 1 ]]; then # Mention dataset_path # Detection data_dir=$ROCAL_DATA_PATH/rocal_data/caffe2/detection/ - # rocAL_api_caffe2_reader.py + # caffe2_reader.py # By default : cpu backend, NCHW format , fp32 # use --classification for Classification / --no-classification for Detection - python"$ver" rocAL_api_caffe2_reader.py \ + python"$ver" caffe2_reader.py \ --image-dataset-path $data_dir \ --no-classification \ --batch-size $batch_size \ @@ -274,16 +274,16 @@ fi #################################################################################################################################### -if [[ rocAL_api_tf_classification_reader -eq 1 ]]; then +if [[ tf_classification_reader -eq 1 ]]; then # Mention dataset_path # Classification data_dir=$ROCAL_DATA_PATH/rocal_data/tf/classification/ - # rocAL_api_tf_classification_reader.py + # tf_classification_reader.py # By default : cpu backend, NCHW format , fp32 # use --classification for Classification / --no-classification for Detection - python"$ver" rocAL_api_tf_classification_reader.py \ + python"$ver" tf_classification_reader.py \ --image-dataset-path $data_dir \ --classification \ --batch-size $batch_size \ @@ -300,16 +300,16 @@ fi #################################################################################################################################### -if [[ rocAL_api_tf_detection_pipeline -eq 1 ]]; then +if [[ tf_detection_reader -eq 1 ]]; then # Mention dataset_path # Detection data_dir=$ROCAL_DATA_PATH/rocal_data/tf/detection/ - # rocAL_api_tf_detection_pipeline.py + # tf_detection_reader.py # By default : cpu backend, NCHW format , fp32 # use --classification for Classification / --no-classification for Detection - python"$ver" rocAL_api_tf_detection_pipeline.py \ + python"$ver" tf_detection_reader.py \ --image-dataset-path $data_dir \ --no-classification \ --batch-size 100 \ @@ -326,15 +326,15 @@ fi #################################################################################################################################### -if [[ rocAL_api_video_pipeline -eq 1 ]]; then +if [[ video_pipeline -eq 1 ]]; then # Mention dataset_path # Detection data_dir=$ROCAL_DATA_PATH/rocal_data/video_and_sequence_samples/labelled_videos/ - # rocAL_api_video_pipeline.py + # video_pipeline.py # By default : cpu backend, NCHW format , fp32 - python"$ver" rocAL_api_video_pipeline.py \ + python"$ver" video_pipeline.py \ --video-path $data_dir \ --$backend_arg \ --batch-size 10 \ diff --git a/rocAL_pybind/examples/rocAL_api_tf_classification_reader.py b/tests/python_api/tf_classification_reader.py similarity index 97% rename from rocAL_pybind/examples/rocAL_api_tf_classification_reader.py rename to tests/python_api/tf_classification_reader.py index 44fcd87a1..adae068fa 100644 --- a/rocAL_pybind/examples/rocAL_api_tf_classification_reader.py +++ b/tests/python_api/tf_classification_reader.py @@ -37,7 +37,7 @@ def draw_patches(img, idx, device_type, args=None): if not args.NHWC: img = img.transpose([0, 1, 2]) image = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) - cv2.imwrite("OUTPUT_FOLDER/TF_READER/CLASSIFICATION/" + + cv2.imwrite("output_folder/tf_reader/classification/" + str(idx) + "_" + "train" + ".png", image) @@ -58,7 +58,7 @@ def main(): 'image/filename': 'image/filename' } try: - path = "OUTPUT_FOLDER/TF_READER/CLASSIFICATION/" + path = "output_folder/tf_reader/classification/" is_exist = os.path.exists(path) if not is_exist: os.makedirs(path) diff --git a/rocAL_pybind/examples/rocAL_api_tf_detection_pipeline.py b/tests/python_api/tf_detection_reader.py similarity index 98% rename from rocAL_pybind/examples/rocAL_api_tf_detection_pipeline.py rename to tests/python_api/tf_detection_reader.py index 8b9aa3920..beb477e68 100644 --- a/rocAL_pybind/examples/rocAL_api_tf_detection_pipeline.py +++ b/tests/python_api/tf_detection_reader.py @@ -66,7 +66,7 @@ def draw_patches(img, idx, bboxes, device_type, args=None): image = cv2.UMat(image).get() image = cv2.rectangle(image, (int(loc_[0]), int(loc_[1])), (int( (loc_[2])), int((loc_[3]))), color, thickness) - cv2.imwrite("OUTPUT_FOLDER/TF_READER/DETECTION/" + + cv2.imwrite("output_folder/tf_reader/detection/" + str(idx) + "_" + "train" + ".png", image) @@ -92,7 +92,7 @@ def main(): 'image/filename': 'image/filename' } try: - path = "OUTPUT_FOLDER/TF_READER/DETECTION" + path = "output_folder/tf_reader/detection" is_exist = os.path.exists(path) if not is_exist: os.makedirs(path) diff --git a/rocAL_pybind/examples/rocAL_api_python_unittest.py b/tests/python_api/unit_test.py similarity index 99% rename from rocAL_pybind/examples/rocAL_api_python_unittest.py rename to tests/python_api/unit_test.py index bfd81590c..803a4b1a9 100644 --- a/rocAL_pybind/examples/rocAL_api_python_unittest.py +++ b/tests/python_api/unit_test.py @@ -101,7 +101,7 @@ def main(): sys.exit(0) try: - path = "OUTPUT_FOLDER/FILE_READER/" + args.augmentation_name + path = "output_folder/file_reader/" + args.augmentation_name isExist = os.path.exists(path) if not isExist: os.makedirs(path) diff --git a/tests/python_api/unit_tests.sh b/tests/python_api/unit_tests.sh new file mode 100755 index 000000000..431cb3a57 --- /dev/null +++ b/tests/python_api/unit_tests.sh @@ -0,0 +1,168 @@ +#!/bin/bash +cwd=$(pwd) + +if [[ $ROCAL_DATA_PATH == "" ]] +then + echo "Need to export ROCAL_DATA_PATH" + exit +fi + +# Path to inputs and outputs available in MIVisionX-data +one_hot_data_path=${ROCAL_DATA_PATH}/rocal_data/images_jpg/labels_folder/ +image_path=${ROCAL_DATA_PATH}/rocal_data/coco/coco_10_img/train_10images_2017/ +coco_detection_path=${ROCAL_DATA_PATH}/rocal_data/coco/coco_10_img/train_10images_2017/ +coco_json_path=${ROCAL_DATA_PATH}/rocal_data/coco/coco_10_img/annotations/instances_train2017.json +tf_classification_path=${ROCAL_DATA_PATH}/rocal_data/tf/classification/ +tf_detection_path=${ROCAL_DATA_PATH}/rocal_data/tf/detection/ +caffe_classification_path=${ROCAL_DATA_PATH}/rocal_data/caffe/classification/ +caffe_detection_path=${ROCAL_DATA_PATH}/rocal_data/caffe/detection/ +caffe2_classification_path=${ROCAL_DATA_PATH}/rocal_data/caffe2/classification/ +caffe2_detection_path=${ROCAL_DATA_PATH}/rocal_data/caffe2/detection/ +mxnet_path=${ROCAL_DATA_PATH}/rocal_data/mxnet/ +output_path=./rocal_python_unittest_output_folder_$(date +%Y-%m-%d_%H-%M-%S)/ +golden_output_path=${ROCAL_DATA_PATH}/rocal_data/GoldenOutputsTensor/ + +display=0 +batch_size=2 +device=0 +width=640 +height=480 +device_name="host" +rgb_name=("gray" "rgb") +rgb=1 +dev_start=0 +dev_end=1 +rgb_start=0 +rgb_end=1 + +# python version +ver=$(python3 -c "import sys;t='{v[0]}.{v[1]}'.format(v=list(sys.version_info[:2]));sys.stdout.write(t)";) + +if [ "$#" -gt 0 ]; then + if [ "$1" -eq 0 ]; then # For only HOST backend + dev_start=0 + dev_end=0 + elif [ "$1" -eq 1 ]; then # For only HIP backend + dev_start=1 + dev_end=1 + elif [ "$1" -eq 2 ]; then # For both HOST and HIP backend + dev_start=0 + dev_end=1 + fi +fi + +if [ "$#" -gt 1 ]; then + if [ "$2" -eq 0 ]; then # For only Greyscale inputs + rgb_start=0 + rgb_end=0 + elif [ "$2" -eq 1 ]; then # For only RGB inputs + rgb_start=1 + rgb_end=1 + elif [ "$2" -eq 2 ]; then # For both RGB and Greyscale inputs + rgb_start=0 + rgb_end=1 + fi +fi + +mkdir "$output_path" + +for ((device=dev_start;device<=dev_end;device++)) +do + if [ $device -eq 1 ] + then + device_name="hip" + backend_arg=rocal-gpu + echo "Running HIP Backend..." + else + backend_arg=no-rocal-gpu + echo "Running HOST Backend..." + fi + for ((rgb=rgb_start;rgb<=rgb_end;rgb++)) + do + python"$ver" unit_test.py --image-dataset-path "$image_path" --augmentation-name lens_correction --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}LensCorrection_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$image_path" --augmentation-name exposure --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Exposure_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$image_path" --augmentation-name flip --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Flip_${rgb_name[$rgb]}_${device_name}" + + # coco detection + python"$ver" unit_test.py --image-dataset-path "$coco_detection_path" --reader-type coco --json-path "$coco_json_path" --augmentation-name gamma_correction --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Gamma_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$coco_detection_path" --reader-type coco --json-path "$coco_json_path" --augmentation-name contrast --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Contrast_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$coco_detection_path" --reader-type coco --json-path "$coco_json_path" --augmentation-name vignette --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Vignette_${rgb_name[$rgb]}_${device_name}" + + # # tf classification + python"$ver" unit_test.py --image-dataset-path "$tf_classification_path" --reader-type "tf_classification" --augmentation-name blend --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Blend_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$tf_classification_path" --reader-type "tf_classification" --augmentation-name warp_affine --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}WarpAffine_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$tf_classification_path" --reader-type "tf_classification" --augmentation-name blur --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Blur_${rgb_name[$rgb]}_${device_name}" + + # tf detection + python"$ver" unit_test.py --image-dataset-path "$tf_detection_path" --reader-type "tf_detection" --augmentation-name snp_noise --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}SNPNoise_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$tf_detection_path" --reader-type "tf_detection" --augmentation-name color_temp --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}ColorTemp_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$tf_detection_path" --reader-type "tf_detection" --augmentation-name fog --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Fog_${rgb_name[$rgb]}_${device_name}" + + # caffe classification + python"$ver" unit_test.py --image-dataset-path "$caffe_classification_path" --reader-type "caffe_classification" --augmentation-name rotate --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Rotate_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$caffe_classification_path" --reader-type "caffe_classification" --augmentation-name brightness --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Brightness_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$caffe_classification_path" --reader-type "caffe_classification" --augmentation-name hue --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Hue_${rgb_name[$rgb]}_${device_name}" + + # caffe detection + python"$ver" unit_test.py --image-dataset-path "$caffe_detection_path" --reader-type "caffe_detection" --augmentation-name saturation --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Saturation_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$caffe_detection_path" --reader-type "caffe_detection" --augmentation-name color_twist --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}ColorTwist_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$caffe_detection_path" --reader-type "caffe_detection" --augmentation-name rain --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Rain_${rgb_name[$rgb]}_${device_name}" + + # caffe2 classification + python"$ver" unit_test.py --image-dataset-path "$caffe2_classification_path" --reader-type "caffe2_classification" --augmentation-name center_crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropCenter_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$caffe2_classification_path" --reader-type "caffe2_classification" --augmentation-name resize_crop_mirror --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}ResizeCropMirror_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$caffe2_classification_path" --reader-type "caffe2_classification" --augmentation-name snow --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Snow_${rgb_name[$rgb]}_${device_name}" + + # caffe2 detection + python"$ver" unit_test.py --image-dataset-path "$caffe2_detection_path" --reader-type "caffe2_detection" --augmentation-name fish_eye --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}FishEye_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$caffe2_detection_path" --reader-type "caffe2_detection" --augmentation-name pixelate --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Pixelate_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$caffe2_detection_path" --reader-type "caffe2_detection" --augmentation-name center_crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropCenter_${rgb_name[$rgb]}_${device_name}_cmn" + + # mxnet + python"$ver" unit_test.py --image-dataset-path "$mxnet_path" --reader-type "mxnet" --augmentation-name jitter --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Jitter_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$mxnet_path" --reader-type "mxnet" --augmentation-name resize_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}ResizeMirrorNormalize_${rgb_name[$rgb]}_${device_name}" + python"$ver" unit_test.py --image-dataset-path "$mxnet_path" --reader-type "mxnet" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_mxnet" + + # CMN + python"$ver" unit_test.py --image-dataset-path "$image_path" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_FileReader" + python"$ver" unit_test.py --image-dataset-path "$coco_detection_path" --reader-type coco --json-path "$coco_json_path" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_coco" + python"$ver" unit_test.py --image-dataset-path "$tf_classification_path" --reader-type "tf_classification" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_tfClassification" + python"$ver" unit_test.py --image-dataset-path "$tf_detection_path" --reader-type "tf_detection" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_tfDetection" + python"$ver" unit_test.py --image-dataset-path "$caffe_classification_path" --reader-type "caffe_classification" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffeClassification" + python"$ver" unit_test.py --image-dataset-path "$caffe_detection_path" --reader-type "caffe_detection" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffeDetection" + python"$ver" unit_test.py --image-dataset-path "$caffe2_classification_path" --reader-type "caffe2_classification" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffe2Classification" + python"$ver" unit_test.py --image-dataset-path "$caffe2_detection_path" --reader-type "caffe2_detection" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_caffe2Detection" + python"$ver" unit_test.py --image-dataset-path "$mxnet_path" --reader-type "mxnet" --augmentation-name crop_mirror_normalize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}CropMirrorNormalize_${rgb_name[$rgb]}_${device_name}_mxnet" + + # crop + python"$ver" unit_test.py --image-dataset-path "$image_path" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_FileReader" + python"$ver" unit_test.py --image-dataset-path "$coco_detection_path" --reader-type coco --json-path "$coco_json_path" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_coco" + python"$ver" unit_test.py --image-dataset-path "$tf_classification_path" --reader-type "tf_classification" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_tfClassification" + python"$ver" unit_test.py --image-dataset-path "$tf_detection_path" --reader-type "tf_detection" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_tfDetection" + python"$ver" unit_test.py --image-dataset-path "$caffe_classification_path" --reader-type "caffe_classification" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffeClassification" + python"$ver" unit_test.py --image-dataset-path "$caffe_detection_path" --reader-type "caffe_detection" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffeDetection" + python"$ver" unit_test.py --image-dataset-path "$caffe2_classification_path" --reader-type "caffe2_classification" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffe2Classification" + python"$ver" unit_test.py --image-dataset-path "$caffe2_detection_path" --reader-type "caffe2_detection" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_caffe2Detection" + python"$ver" unit_test.py --image-dataset-path "$mxnet_path" --reader-type "mxnet" --augmentation-name crop --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}Crop_${rgb_name[$rgb]}_${device_name}_mxnet" + + # resize + python"$ver" unit_test.py --image-dataset-path "$image_path" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 1 --scaling-mode 0 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_default_FileReader" + python"$ver" unit_test.py --image-dataset-path "$coco_detection_path" --reader-type coco --json-path "$coco_json_path" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 1 --scaling-mode 1 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_stretch_coco" + python"$ver" unit_test.py --image-dataset-path "$tf_classification_path" --reader-type "tf_classification" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 1 --scaling-mode 2 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_notsmaller_tfClassification" + python"$ver" unit_test.py --image-dataset-path "$tf_detection_path" --reader-type "tf_detection" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 1 --scaling-mode 3 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bilinear_notlarger_tfDetection" + python"$ver" unit_test.py --image-dataset-path "$caffe_classification_path" --reader-type "caffe_classification" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 2 --scaling-mode 0 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_bicubic_default_caffeClassification" + # python"$ver" unit_test.py --image-dataset-path "$caffe_detection_path" --reader-type "caffe_detection" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 0 --scaling-mode 0 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_nearestneighbor_default_caffeDetection" + python"$ver" unit_test.py --image-dataset-path "$caffe2_classification_path" --reader-type "caffe2_classification" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 3 --scaling-mode 0 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_lanczos_default_caffe2Classification" + python"$ver" unit_test.py --image-dataset-path "$caffe2_detection_path" --reader-type "caffe2_detection" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 5 --scaling-mode 0 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_triangular_default_caffe2Detection" + python"$ver" unit_test.py --image-dataset-path "$mxnet_path" --reader-type "mxnet" --augmentation-name resize --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --interpolation-type 4 --scaling-mode 0 --$backend_arg -f "${output_path}Resize_${rgb_name[$rgb]}_${device_name}_gaussian_default_mxnet" + + # Special Case - One Hot Encoded Labels + python"$ver" unit_test.py --image-dataset-path "$one_hot_data_path" --augmentation-name one_hot --batch-size $batch_size --max-width $width --max-height $height --color-format $rgb --$backend_arg -f "${output_path}OneHot_${rgb_name[$rgb]}_${device_name}" + + done +done + +pwd + +# Run python script to compare rocAL outputs with golden ouptuts +python3 "$cwd"/image_comparison.py "$golden_output_path" "$output_path" diff --git a/rocAL_pybind/examples/rocAL_api_video_pipeline.py b/tests/python_api/video_pipeline.py similarity index 97% rename from rocAL_pybind/examples/rocAL_api_video_pipeline.py rename to tests/python_api/video_pipeline.py index 1a1556a9a..5f106ff21 100644 --- a/rocAL_pybind/examples/rocAL_api_video_pipeline.py +++ b/tests/python_api/video_pipeline.py @@ -103,10 +103,10 @@ def draw_frames(img, batch_idx, iter_idx, layout): image = image.transpose([1, 2, 0]) image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR) import os - if not os.path.exists("OUTPUT_FOLDER/VIDEO_READER"): - os.makedirs("OUTPUT_FOLDER/VIDEO_READER") + if not os.path.exists("output_folder/video_reader"): + os.makedirs("output_folder/video_reader") image = cv2.UMat(image).get() - cv2.imwrite("OUTPUT_FOLDER/VIDEO_READER/" + + cv2.imwrite("output_folder/video_reader/" + "iter_"+str(iter_idx)+"_batch_"+str(batch_idx)+".png", image)