From 958320894db50e62cb35d1e4562f6edb63eb8070 Mon Sep 17 00:00:00 2001 From: LakshmiKumar23 Date: Mon, 19 Feb 2024 15:39:38 -0800 Subject: [PATCH 01/12] setup and docker updates --- docker/rocal-on-ubuntu-20.dockerfile | 9 ++++----- docker/rocal-on-ubuntu-22.dockerfile | 12 ++++++------ docker/rocal-with-pytorch.dockerfile | 11 ++++++----- rocAL-setup.py | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docker/rocal-on-ubuntu-20.dockerfile b/docker/rocal-on-ubuntu-20.dockerfile index c72add09a..fcbf13f0a 100644 --- a/docker/rocal-on-ubuntu-20.dockerfile +++ b/docker/rocal-on-ubuntu-20.dockerfile @@ -41,10 +41,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install rocblas rocblas-dev miopen # install rocAL dependency RUN DEBIAN_FRONTEND=noninteractive apt-get -y install wget libbz2-dev libssl-dev python-dev python3-dev libgflags-dev libgoogle-glog-dev liblmdb-dev nasm yasm libjsoncpp-dev clang && \ - git clone -b 2.0.6.2 https://github.com/rrawther/libjpeg-turbo.git && cd libjpeg-turbo && mkdir build && cd build && \ - cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_STATIC=FALSE -DCMAKE_INSTALL_DOCDIR=/usr/share/doc/libjpeg-turbo-2.0.3 \ - -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib ../ && make -j4 && sudo make install && cd ../../ && \ - git clone -b 1.1.0 https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp.git && cd rpp && mkdir build && cd build && \ + git clone -b 3.0.1 https://github.com/libjpeg-turbo/libjpeg-turbo.git && cd libjpeg-turbo && mkdir build && cd build && \ + cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_STATIC=FALSE -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib -DWITH_JPEG8=TRUE ../ && \ + git clone https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp.git && cd rpp && mkdir build && cd build && \ cmake -DBACKEND=HIP ../ && make -j4 && sudo make install && cd ../../ && \ git clone -b v3.12.4 https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && \ ./autogen.sh && ./configure && make -j8 && make check -j8 && sudo make install && sudo ldconfig && cd @@ -53,7 +52,7 @@ ENV ROCM_HOME=/opt/rocm RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g++ hipblas hipsparse rocrand hipfft rocfft rocthrust-dev hipcub-dev python3-dev && \ git clone https://github.com/Tencent/rapidjson.git && cd rapidjson && mkdir build && cd build && \ cmake ../ && make -j4 && sudo make install && cd ../../ && \ - pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ + pip install pytest==3.1 && git clone -b rocm6.1_internal_testing https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ git clone https://github.com/ROCmSoftwarePlatform/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv && pip install numpy==1.21 diff --git a/docker/rocal-on-ubuntu-22.dockerfile b/docker/rocal-on-ubuntu-22.dockerfile index b813a21db..9691e519b 100644 --- a/docker/rocal-on-ubuntu-22.dockerfile +++ b/docker/rocal-on-ubuntu-22.dockerfile @@ -42,22 +42,22 @@ RUN apt-get -y install curl make g++ unzip libomp-dev libpthread-stubs0-dev wget RUN mkdir rocAL_deps && cd rocAL_deps && wget https://sourceforge.net/projects/half/files/half/1.12.0/half-1.12.0.zip && \ unzip half-1.12.0.zip -d half-files && sudo mkdir -p /usr/local/include/half && sudo cp half-files/include/half.hpp /usr/local/include/half && cd RUN apt-get update -y && apt-get -y install autoconf automake libbz2-dev libssl-dev python3-dev libgflags-dev libgoogle-glog-dev liblmdb-dev nasm yasm libjsoncpp-dev && \ - git clone -b 2.0.6.2 https://github.com/rrawther/libjpeg-turbo.git && cd libjpeg-turbo && mkdir build && cd build && \ - cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_STATIC=FALSE -DCMAKE_INSTALL_DOCDIR=/usr/share/doc/libjpeg-turbo-2.0.3 \ - -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib ../ && make -j4 && sudo make install && cd + git clone -b 3.0.1 https://github.com/libjpeg-turbo/libjpeg-turbo.git && cd libjpeg-turbo && mkdir build && cd build && \ + cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_STATIC=FALSE -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib -DWITH_JPEG8=TRUE ../ && \ + make -j4 && sudo make install && cd RUN apt-get -y install sqlite3 libsqlite3-dev libtool build-essential RUN git clone -b v3.21.9 https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && \ ./autogen.sh && ./configure && make -j8 && make check -j8 && sudo make install && sudo ldconfig && cd -RUN git clone -b 1.1.0 https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp.git && cd rpp && mkdir build && cd build && \ +RUN git clone -b https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp.git && cd rpp && mkdir build && cd build && \ cmake -DBACKEND=HIP ../ && make -j4 && sudo make install && cd ENV CUPY_INSTALL_USE_HIP=1 ENV ROCM_HOME=/opt/rocm RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g++ hipblas hipsparse rocrand hipfft rocfft rocthrust-dev hipcub-dev python3-dev && \ git clone https://github.com/Tencent/rapidjson.git && cd rapidjson && mkdir build && cd build && \ cmake ../ && make -j4 && sudo make install && cd ../../ && \ - pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ + pip install pytest==3.1 && git clone -b rocm6.1_internal_testing https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ - git clone https://github.com/ROCmSoftwarePlatform/cupy.git && cd cupy && git submodule update --init && \ + git clone -b rocm6.1_internal_testing https://github.com/ROCmSoftwarePlatform/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv && pip install numpy==1.21 ENV ROCAL_WORKSPACE=/workspace diff --git a/docker/rocal-with-pytorch.dockerfile b/docker/rocal-with-pytorch.dockerfile index a26a845c0..20e78c349 100644 --- a/docker/rocal-with-pytorch.dockerfile +++ b/docker/rocal-with-pytorch.dockerfile @@ -32,9 +32,10 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install rocblas rocblas-dev miopen # install rocAL dependency RUN DEBIAN_FRONTEND=noninteractive apt-get -y install wget libbz2-dev libssl-dev python-dev python3-dev libgflags-dev libgoogle-glog-dev liblmdb-dev nasm yasm libjsoncpp-dev clang && \ git clone -b 2.0.6.2 https://github.com/rrawther/libjpeg-turbo.git && cd libjpeg-turbo && mkdir build && cd build && \ - cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_STATIC=FALSE -DCMAKE_INSTALL_DOCDIR=/usr/share/doc/libjpeg-turbo-2.0.3 \ - -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib ../ && make -j4 && sudo make install && cd ../../ && \ - git clone -b 1.1.0 https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp.git && cd rpp && mkdir build && cd build && \ + git clone -b 3.0.1 https://github.com/libjpeg-turbo/libjpeg-turbo.git && cd libjpeg-turbo && mkdir build && cd build && \ + cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_STATIC=FALSE -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib -DWITH_JPEG8=TRUE ../ && \ + make -j4 && sudo make install && cd + git clone https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp.git && cd rpp && mkdir build && cd build && \ cmake -DBACKEND=HIP ../ && make -j4 && sudo make install && cd ../../ && \ git clone -b v3.12.4 https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && \ ./autogen.sh && ./configure && make -j8 && make check -j8 && sudo make install && sudo ldconfig && cd @@ -43,9 +44,9 @@ ENV ROCM_HOME=/opt/rocm RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g++ hipblas hipsparse rocrand hipfft rocfft rocthrust-dev hipcub-dev python3-dev && \ git clone https://github.com/Tencent/rapidjson.git && cd rapidjson && mkdir build && cd build && \ cmake ../ && make -j4 && sudo make install && cd ../../ && \ - pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ + pip install pytest==3.1 && git clone -b rocm6.1_internal_testing https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ - git clone https://github.com/ROCmSoftwarePlatform/cupy.git && cd cupy && git submodule update --init && \ + git clone rocm6.1_internal_testing https://github.com/ROCmSoftwarePlatform/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv && pip install numpy==1.21 ENV ROCAL_WORKSPACE=/workspace diff --git a/rocAL-setup.py b/rocAL-setup.py index 1032aef6c..919642da8 100644 --- a/rocAL-setup.py +++ b/rocAL-setup.py @@ -345,7 +345,7 @@ os.system('sudo '+linuxFlag+' '+linuxSystemInstall + ' '+linuxSystemInstall_check+' install -y git g++ hipblas hipsparse rocrand hipfft rocfft rocthrust-devel hipcub-devel python3-devel') os.system('sudo -v') - os.system('(cd '+deps_dir+'; git clone -b v12.2.0 https://github.com/ROCmSoftwarePlatform/cupy.git; export CUPY_INSTALL_USE_HIP=1; export ROCM_HOME=/opt/rocm; cd cupy; git submodule update --init; pip install -e . --no-cache-dir -vvvv)') + os.system('(cd '+deps_dir+'; git clone -b rocm6.1_internal_testing https://github.com/ROCmSoftwarePlatform/cupy.git; export CUPY_INSTALL_USE_HIP=1; export ROCM_HOME=/opt/rocm; cd cupy; git submodule update --init; pip install -e . --no-cache-dir -vvvv)') os.system('pip install numpy==1.21') # Install ffmpeg From 920fb0e9148ff38f82938acdf04c0f60757e8f75 Mon Sep 17 00:00:00 2001 From: LakshmiKumar23 Date: Mon, 19 Feb 2024 15:42:19 -0800 Subject: [PATCH 02/12] docker updates --- docker/rocal-on-ubuntu-20.dockerfile | 2 +- docker/rocal-on-ubuntu-22.dockerfile | 2 +- docker/rocal-with-pytorch.dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/rocal-on-ubuntu-20.dockerfile b/docker/rocal-on-ubuntu-20.dockerfile index fcbf13f0a..7bc53add5 100644 --- a/docker/rocal-on-ubuntu-20.dockerfile +++ b/docker/rocal-on-ubuntu-20.dockerfile @@ -52,7 +52,7 @@ ENV ROCM_HOME=/opt/rocm RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g++ hipblas hipsparse rocrand hipfft rocfft rocthrust-dev hipcub-dev python3-dev && \ git clone https://github.com/Tencent/rapidjson.git && cd rapidjson && mkdir build && cd build && \ cmake ../ && make -j4 && sudo make install && cd ../../ && \ - pip install pytest==3.1 && git clone -b rocm6.1_internal_testing https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ + pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ git clone https://github.com/ROCmSoftwarePlatform/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv && pip install numpy==1.21 diff --git a/docker/rocal-on-ubuntu-22.dockerfile b/docker/rocal-on-ubuntu-22.dockerfile index 9691e519b..5dbfd2f74 100644 --- a/docker/rocal-on-ubuntu-22.dockerfile +++ b/docker/rocal-on-ubuntu-22.dockerfile @@ -55,7 +55,7 @@ ENV ROCM_HOME=/opt/rocm RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g++ hipblas hipsparse rocrand hipfft rocfft rocthrust-dev hipcub-dev python3-dev && \ git clone https://github.com/Tencent/rapidjson.git && cd rapidjson && mkdir build && cd build && \ cmake ../ && make -j4 && sudo make install && cd ../../ && \ - pip install pytest==3.1 && git clone -b rocm6.1_internal_testing https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ + pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ git clone -b rocm6.1_internal_testing https://github.com/ROCmSoftwarePlatform/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv && pip install numpy==1.21 diff --git a/docker/rocal-with-pytorch.dockerfile b/docker/rocal-with-pytorch.dockerfile index 20e78c349..90ad02ebe 100644 --- a/docker/rocal-with-pytorch.dockerfile +++ b/docker/rocal-with-pytorch.dockerfile @@ -44,7 +44,7 @@ ENV ROCM_HOME=/opt/rocm RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g++ hipblas hipsparse rocrand hipfft rocfft rocthrust-dev hipcub-dev python3-dev && \ git clone https://github.com/Tencent/rapidjson.git && cd rapidjson && mkdir build && cd build && \ cmake ../ && make -j4 && sudo make install && cd ../../ && \ - pip install pytest==3.1 && git clone -b rocm6.1_internal_testing https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ + pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ git clone rocm6.1_internal_testing https://github.com/ROCmSoftwarePlatform/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv && pip install numpy==1.21 From 9de0341cf40c9b3a5ae9a8955aaadd05b480916f Mon Sep 17 00:00:00 2001 From: LakshmiKumar23 Date: Mon, 19 Feb 2024 15:48:57 -0800 Subject: [PATCH 03/12] rpp updates --- docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile | 2 +- docker/rocal-on-ubuntu-20.dockerfile | 2 +- docker/rocal-on-ubuntu-22.dockerfile | 2 +- rocAL-setup.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile index 0ca865469..22f6cdfe7 100644 --- a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile +++ b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile @@ -47,7 +47,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install wget libbz2-dev libssl-dev git clone -b 2.0.6.2 https://github.com/rrawther/libjpeg-turbo.git && cd libjpeg-turbo && mkdir build && cd build && \ cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_STATIC=FALSE -DCMAKE_INSTALL_DOCDIR=/usr/share/doc/libjpeg-turbo-2.0.3 \ -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib ../ && make -j4 && sudo make install && cd ../../ && \ - git clone -b 1.1.0 https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp.git && cd rpp && mkdir build && cd build && \ + git clone -b 1.1.0 https://github.com/ROCm/rpp && cd rpp && mkdir build && cd build && \ cmake -DBACKEND=HIP ../ && make -j4 && sudo make install && cd ../../ && \ git clone -b v3.12.4 https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && \ ./autogen.sh && ./configure && make -j8 && make check -j8 && sudo make install && sudo ldconfig && cd diff --git a/docker/rocal-on-ubuntu-20.dockerfile b/docker/rocal-on-ubuntu-20.dockerfile index 7bc53add5..04ba31be4 100644 --- a/docker/rocal-on-ubuntu-20.dockerfile +++ b/docker/rocal-on-ubuntu-20.dockerfile @@ -43,7 +43,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install rocblas rocblas-dev miopen RUN DEBIAN_FRONTEND=noninteractive apt-get -y install wget libbz2-dev libssl-dev python-dev python3-dev libgflags-dev libgoogle-glog-dev liblmdb-dev nasm yasm libjsoncpp-dev clang && \ git clone -b 3.0.1 https://github.com/libjpeg-turbo/libjpeg-turbo.git && cd libjpeg-turbo && mkdir build && cd build && \ cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_STATIC=FALSE -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib -DWITH_JPEG8=TRUE ../ && \ - git clone https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp.git && cd rpp && mkdir build && cd build && \ + git clone https://github.com/ROCm/rpp && cd rpp && mkdir build && cd build && \ cmake -DBACKEND=HIP ../ && make -j4 && sudo make install && cd ../../ && \ git clone -b v3.12.4 https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && \ ./autogen.sh && ./configure && make -j8 && make check -j8 && sudo make install && sudo ldconfig && cd diff --git a/docker/rocal-on-ubuntu-22.dockerfile b/docker/rocal-on-ubuntu-22.dockerfile index 5dbfd2f74..015e95f76 100644 --- a/docker/rocal-on-ubuntu-22.dockerfile +++ b/docker/rocal-on-ubuntu-22.dockerfile @@ -48,7 +48,7 @@ RUN apt-get update -y && apt-get -y install autoconf automake libbz2-dev libssl- RUN apt-get -y install sqlite3 libsqlite3-dev libtool build-essential RUN git clone -b v3.21.9 https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && \ ./autogen.sh && ./configure && make -j8 && make check -j8 && sudo make install && sudo ldconfig && cd -RUN git clone -b https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp.git && cd rpp && mkdir build && cd build && \ +RUN git clone -b https://github.com/ROCm/rpp && cd rpp && mkdir build && cd build && \ cmake -DBACKEND=HIP ../ && make -j4 && sudo make install && cd ENV CUPY_INSTALL_USE_HIP=1 ENV ROCM_HOME=/opt/rocm diff --git a/rocAL-setup.py b/rocAL-setup.py index 919642da8..857e92ad9 100644 --- a/rocAL-setup.py +++ b/rocAL-setup.py @@ -318,7 +318,7 @@ ' -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_STATIC=FALSE -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib -DWITH_JPEG8=TRUE ..; make -j 4; sudo make install )') # RPP os.system('sudo -v') - os.system('(cd '+deps_dir+'; git clone -b '+rppVersion+' https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp.git; cd rpp; mkdir build-'+backend+'; cd build-'+backend+'; ' + + os.system('(cd '+deps_dir+'; git clone -b '+rppVersion+' https://github.com/ROCm/rpp; cd rpp; mkdir build-'+backend+'; cd build-'+backend+'; ' + linuxCMake+' -DBACKEND='+backend+' -DCMAKE_INSTALL_PREFIX='+ROCM_PATH+' ../; make -j4; sudo make install)') # RapidJSON os.system('sudo -v') From a6f186d0d65f3084c205ab5664b663c0baac2232 Mon Sep 17 00:00:00 2001 From: LakshmiKumar23 Date: Mon, 19 Feb 2024 15:52:16 -0800 Subject: [PATCH 04/12] rpp updates --- docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile | 2 +- docker/rocal-on-ubuntu-22.dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile index 22f6cdfe7..1184ea48f 100644 --- a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile +++ b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile @@ -47,7 +47,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install wget libbz2-dev libssl-dev git clone -b 2.0.6.2 https://github.com/rrawther/libjpeg-turbo.git && cd libjpeg-turbo && mkdir build && cd build && \ cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_STATIC=FALSE -DCMAKE_INSTALL_DOCDIR=/usr/share/doc/libjpeg-turbo-2.0.3 \ -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib ../ && make -j4 && sudo make install && cd ../../ && \ - git clone -b 1.1.0 https://github.com/ROCm/rpp && cd rpp && mkdir build && cd build && \ + git clone https://github.com/ROCm/rpp && cd rpp && mkdir build && cd build && \ cmake -DBACKEND=HIP ../ && make -j4 && sudo make install && cd ../../ && \ git clone -b v3.12.4 https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && \ ./autogen.sh && ./configure && make -j8 && make check -j8 && sudo make install && sudo ldconfig && cd diff --git a/docker/rocal-on-ubuntu-22.dockerfile b/docker/rocal-on-ubuntu-22.dockerfile index 015e95f76..b73fe23c2 100644 --- a/docker/rocal-on-ubuntu-22.dockerfile +++ b/docker/rocal-on-ubuntu-22.dockerfile @@ -48,7 +48,7 @@ RUN apt-get update -y && apt-get -y install autoconf automake libbz2-dev libssl- RUN apt-get -y install sqlite3 libsqlite3-dev libtool build-essential RUN git clone -b v3.21.9 https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && \ ./autogen.sh && ./configure && make -j8 && make check -j8 && sudo make install && sudo ldconfig && cd -RUN git clone -b https://github.com/ROCm/rpp && cd rpp && mkdir build && cd build && \ +RUN git clone https://github.com/ROCm/rpp && cd rpp && mkdir build && cd build && \ cmake -DBACKEND=HIP ../ && make -j4 && sudo make install && cd ENV CUPY_INSTALL_USE_HIP=1 ENV ROCM_HOME=/opt/rocm From bfeb277cb316df118a148313aac3f8399fbf6aea Mon Sep 17 00:00:00 2001 From: LakshmiKumar23 Date: Tue, 20 Feb 2024 09:56:18 -0800 Subject: [PATCH 05/12] rocal repo updates --- ...-on-ubuntu-20-with-pytorch-with-mesa.dockerfile | 12 ++++++++---- docker/rocal-on-ubuntu-20.dockerfile | 12 ++++++++---- docker/rocal-on-ubuntu-22.dockerfile | 14 +++++++++----- docker/rocal-with-pytorch.dockerfile | 11 +++++++---- rocAL-setup.py | 2 +- 5 files changed, 33 insertions(+), 18 deletions(-) diff --git a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile index 1184ea48f..f5ae66281 100644 --- a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile +++ b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile @@ -1,8 +1,8 @@ ARG PYTORCH_VERSION=latest FROM rocm/pytorch:${PYTORCH_VERSION} -ARG ROCM_INSTALLER_REPO=https://repo.radeon.com/amdgpu-install/5.4/ubuntu/focal/amdgpu-install_5.4.50400-1_all.deb -ARG ROCM_INSTALLER_PACKAGE=amdgpu-install_5.4.50400-1_all.deb +ARG ROCM_INSTALLER_REPO=https://repo.radeon.com/amdgpu-install/6.0.2/ubuntu/focal/amdgpu-install_6.0.60002-1_all.deb +ARG ROCM_INSTALLER_PACKAGE=amdgpu-install_6.0.60002-1_all.deb ENV ROCAL_DEPS_ROOT=/rocAL-deps WORKDIR $ROCAL_DEPS_ROOT @@ -20,7 +20,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install initramfs-tools libnuma-de wget ${ROCM_INSTALLER_REPO} && \ sudo apt-get install -y ./${ROCM_INSTALLER_PACKAGE} && \ sudo apt-get update -y && \ - sudo amdgpu-install -y --usecase=graphics,rocm + sudo amdgpu-install -y --usecase=graphics,rocm --no-32 # install OpenCV RUN DEBIAN_FRONTEND=noninteractive apt-get -y install build-essential libgtk2.0-dev libavcodec-dev libavformat-dev libswscale-dev python-dev python-numpy \ @@ -65,5 +65,9 @@ ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE # install MIVisionX -RUN git clone https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX.git && \ +RUN git clone https://github.com/ROCm/MIVisionX.git && \ mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../MIVisionX && make -j8 && make install + +# Install rocAL +RUN git clone -b develop https://github.com/ROCm/rocAL && \ + mkdir build && cd build && cmake ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file diff --git a/docker/rocal-on-ubuntu-20.dockerfile b/docker/rocal-on-ubuntu-20.dockerfile index 04ba31be4..a2e0343ba 100644 --- a/docker/rocal-on-ubuntu-20.dockerfile +++ b/docker/rocal-on-ubuntu-20.dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:20.04 -ARG ROCM_INSTALLER_REPO=https://repo.radeon.com/amdgpu-install/5.4.1/ubuntu/focal/amdgpu-install_5.4.50401-1_all.deb -ARG ROCM_INSTALLER_PACKAGE=amdgpu-install_5.4.50401-1_all.deb +ARG ROCM_INSTALLER_REPO=https://repo.radeon.com/amdgpu-install/6.0.2/ubuntu/focal/amdgpu-install_6.0.60002-1_all.deb +ARG ROCM_INSTALLER_PACKAGE=amdgpu-install_6.0.60002-1_all.deb ENV ROCAL_DEPS_ROOT=/rocAL-deps WORKDIR $ROCAL_DEPS_ROOT @@ -17,7 +17,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install initramfs-tools libnuma-de wget ${ROCM_INSTALLER_REPO} && \ sudo apt-get install -y ./${ROCM_INSTALLER_PACKAGE} && \ sudo apt-get update -y && \ - sudo amdgpu-install -y --usecase=graphics,rocm + sudo amdgpu-install -y --usecase=graphics,rocm --no-32 # install OpenCV RUN DEBIAN_FRONTEND=noninteractive apt-get -y install build-essential libgtk2.0-dev libavcodec-dev libavformat-dev libswscale-dev python-dev python-numpy \ @@ -61,5 +61,9 @@ ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE # install MIVisionX -RUN git clone https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX.git && \ +RUN git clone https://github.com/ROCm/MIVisionX.git && \ mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../MIVisionX && make -j8 && make install + +# Install rocAL +RUN git clone -b develop https://github.com/ROCm/rocAL && \ + mkdir build && cd build && cmake ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file diff --git a/docker/rocal-on-ubuntu-22.dockerfile b/docker/rocal-on-ubuntu-22.dockerfile index b73fe23c2..b7ad3616c 100644 --- a/docker/rocal-on-ubuntu-22.dockerfile +++ b/docker/rocal-on-ubuntu-22.dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:22.04 -ARG ROCM_INSTALLER_REPO=https://repo.radeon.com/amdgpu-install/5.4.1/ubuntu/jammy/amdgpu-install_5.4.50401-1_all.deb -ARG ROCM_INSTALLER_PACKAGE=amdgpu-install_5.4.50401-1_all.deb +ARG ROCM_INSTALLER_REPO=https://repo.radeon.com/amdgpu-install/6.0.2/ubuntu/jammy/amdgpu-install_6.0.60002-1_all.deb +ARG ROCM_INSTALLER_PACKAGE=amdgpu-install_6.0.60002-1_all.deb ENV ROCAL_DEPS_ROOT=/rocAL-deps WORKDIR $ROCAL_DEPS_ROOT @@ -17,7 +17,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install initramfs-tools libnuma-de wget ${ROCM_INSTALLER_REPO} && \ sudo apt-get install -y ./${ROCM_INSTALLER_PACKAGE} && \ sudo apt-get update -y && \ - sudo amdgpu-install -y --usecase=graphics,rocm + sudo amdgpu-install -y --usecase=graphics,rocm --no-32 # install OpenCV RUN DEBIAN_FRONTEND=noninteractive apt-get -y install build-essential libgtk2.0-dev libavcodec-dev libavformat-dev libswscale-dev python3-dev python3-numpy \ @@ -64,5 +64,9 @@ ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE # Install MIVisionX -RUN git clone https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX.git && \ - mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../MIVisionX && make -j8 && make install \ No newline at end of file +RUN git clone https://github.com/ROCm/MIVisionX && \ + mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../MIVisionX && make -j8 && make install + +# Install rocAL +RUN git clone -b develop https://github.com/ROCm/rocAL && \ + mkdir build && cd build && cmake ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file diff --git a/docker/rocal-with-pytorch.dockerfile b/docker/rocal-with-pytorch.dockerfile index 90ad02ebe..69cb46adb 100644 --- a/docker/rocal-with-pytorch.dockerfile +++ b/docker/rocal-with-pytorch.dockerfile @@ -31,11 +31,10 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install rocblas rocblas-dev miopen # install rocAL dependency RUN DEBIAN_FRONTEND=noninteractive apt-get -y install wget libbz2-dev libssl-dev python-dev python3-dev libgflags-dev libgoogle-glog-dev liblmdb-dev nasm yasm libjsoncpp-dev clang && \ - git clone -b 2.0.6.2 https://github.com/rrawther/libjpeg-turbo.git && cd libjpeg-turbo && mkdir build && cd build && \ git clone -b 3.0.1 https://github.com/libjpeg-turbo/libjpeg-turbo.git && cd libjpeg-turbo && mkdir build && cd build && \ cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=RELEASE -DENABLE_STATIC=FALSE -DCMAKE_INSTALL_DEFAULT_LIBDIR=lib -DWITH_JPEG8=TRUE ../ && \ - make -j4 && sudo make install && cd - git clone https://github.com/GPUOpen-ProfessionalCompute-Libraries/rpp.git && cd rpp && mkdir build && cd build && \ + make -j4 && sudo make install && cd && \ + git clone https://github.com/ROCm/rpp.git && cd rpp && mkdir build && cd build && \ cmake -DBACKEND=HIP ../ && make -j4 && sudo make install && cd ../../ && \ git clone -b v3.12.4 https://github.com/protocolbuffers/protobuf.git && cd protobuf && git submodule update --init --recursive && \ ./autogen.sh && ./configure && make -j8 && make check -j8 && sudo make install && sudo ldconfig && cd @@ -53,5 +52,9 @@ ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE # install MIVisionX -RUN git clone https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX.git && \ +RUN git clone https://github.com/ROCm/MIVisionX.git && \ mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../MIVisionX && make -j8 && make install + +# Install rocAL +RUN git clone -b develop https://github.com/ROCm/rocAL && \ + mkdir build && cd build && cmake -DPYTHONVERSION=3.9 ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file diff --git a/rocAL-setup.py b/rocAL-setup.py index 857e92ad9..6244aea81 100644 --- a/rocAL-setup.py +++ b/rocAL-setup.py @@ -419,7 +419,7 @@ # MIVisionX os.system('sudo -v') - os.system('(cd '+deps_dir+'; git clone -b '+mivisionxVersion+' https://github.com/GPUOpen-ProfessionalCompute-Libraries/MIVisionX.git; cd MIVisionX; mkdir build-'+backend+'; cd build-'+backend+'; ' + + os.system('(cd '+deps_dir+'; git clone -b '+mivisionxVersion+' https://github.com/ROCm/MIVisionX.git; cd MIVisionX; mkdir build-'+backend+'; cd build-'+backend+'; ' + linuxCMake+' -DBACKEND='+backend+' -DROCAL=OFF ../; make -j4; sudo make install)') print("\nrocAL Dependencies Installed with rocAL-setup.py V-"+__version__+"\n") From 6f1b3e534b2292c1f7a3e87cf7e9b5a687340a43 Mon Sep 17 00:00:00 2001 From: LAkshmiKuamr23 Date: Tue, 20 Feb 2024 09:58:44 -0800 Subject: [PATCH 06/12] rocm repor link change --- docker/rocal-with-pytorch.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/rocal-with-pytorch.dockerfile b/docker/rocal-with-pytorch.dockerfile index 69cb46adb..1059ade0d 100644 --- a/docker/rocal-with-pytorch.dockerfile +++ b/docker/rocal-with-pytorch.dockerfile @@ -45,7 +45,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ cmake ../ && make -j4 && sudo make install && cd ../../ && \ pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ - git clone rocm6.1_internal_testing https://github.com/ROCmSoftwarePlatform/cupy.git && cd cupy && git submodule update --init && \ + git clone rocm6.1_internal_testing https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv && pip install numpy==1.21 ENV ROCAL_WORKSPACE=/workspace From 5ed1a981bf2e1183439de8075c2aad101871ffa5 Mon Sep 17 00:00:00 2001 From: LAkshmiKuamr23 Date: Tue, 20 Feb 2024 10:03:41 -0800 Subject: [PATCH 07/12] setup - link update for repo --- rocAL-setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocAL-setup.py b/rocAL-setup.py index 6244aea81..7321d9044 100644 --- a/rocAL-setup.py +++ b/rocAL-setup.py @@ -345,7 +345,7 @@ os.system('sudo '+linuxFlag+' '+linuxSystemInstall + ' '+linuxSystemInstall_check+' install -y git g++ hipblas hipsparse rocrand hipfft rocfft rocthrust-devel hipcub-devel python3-devel') os.system('sudo -v') - os.system('(cd '+deps_dir+'; git clone -b rocm6.1_internal_testing https://github.com/ROCmSoftwarePlatform/cupy.git; export CUPY_INSTALL_USE_HIP=1; export ROCM_HOME=/opt/rocm; cd cupy; git submodule update --init; pip install -e . --no-cache-dir -vvvv)') + os.system('(cd '+deps_dir+'; git clone -b rocm6.1_internal_testing https://github.com/ROCm/cupy.git; export CUPY_INSTALL_USE_HIP=1; export ROCM_HOME=/opt/rocm; cd cupy; git submodule update --init; pip install -e . --no-cache-dir -vvvv)') os.system('pip install numpy==1.21') # Install ffmpeg From 326917eb2321f1261963c2f43cb3cb9ff0a6fdc5 Mon Sep 17 00:00:00 2001 From: LAkshmiKuamr23 Date: Tue, 20 Feb 2024 11:32:55 -0800 Subject: [PATCH 08/12] cupy updates --- docker/rocal-on-ubuntu-20.dockerfile | 2 +- docker/rocal-on-ubuntu-22.dockerfile | 2 +- docker/rocal-with-pytorch.dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/rocal-on-ubuntu-20.dockerfile b/docker/rocal-on-ubuntu-20.dockerfile index a2e0343ba..73c74c370 100644 --- a/docker/rocal-on-ubuntu-20.dockerfile +++ b/docker/rocal-on-ubuntu-20.dockerfile @@ -54,7 +54,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ cmake ../ && make -j4 && sudo make install && cd ../../ && \ pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ - git clone https://github.com/ROCmSoftwarePlatform/cupy.git && cd cupy && git submodule update --init && \ + git clone -b rocm6.1_internal_testing https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv && pip install numpy==1.21 ENV ROCAL_WORKSPACE=/workspace diff --git a/docker/rocal-on-ubuntu-22.dockerfile b/docker/rocal-on-ubuntu-22.dockerfile index b7ad3616c..cb3ceea5b 100644 --- a/docker/rocal-on-ubuntu-22.dockerfile +++ b/docker/rocal-on-ubuntu-22.dockerfile @@ -57,7 +57,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ cmake ../ && make -j4 && sudo make install && cd ../../ && \ pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ - git clone -b rocm6.1_internal_testing https://github.com/ROCmSoftwarePlatform/cupy.git && cd cupy && git submodule update --init && \ + git clone -b rocm6.1_internal_testing https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv && pip install numpy==1.21 ENV ROCAL_WORKSPACE=/workspace diff --git a/docker/rocal-with-pytorch.dockerfile b/docker/rocal-with-pytorch.dockerfile index 1059ade0d..b9b84eedb 100644 --- a/docker/rocal-with-pytorch.dockerfile +++ b/docker/rocal-with-pytorch.dockerfile @@ -45,7 +45,7 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ cmake ../ && make -j4 && sudo make install && cd ../../ && \ pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ - git clone rocm6.1_internal_testing https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ + git clone -b rocm6.1_internal_testing https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv && pip install numpy==1.21 ENV ROCAL_WORKSPACE=/workspace From 959643ab2f89bc138393698b6c3ed02fc9af3eea Mon Sep 17 00:00:00 2001 From: LAkshmiKuamr23 Date: Tue, 20 Feb 2024 11:56:13 -0800 Subject: [PATCH 09/12] cupy updates --- docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile | 4 +++- docker/rocal-on-ubuntu-20.dockerfile | 4 +++- docker/rocal-on-ubuntu-22.dockerfile | 4 +++- docker/rocal-with-pytorch.dockerfile | 4 +++- rocAL-setup.py | 2 +- 5 files changed, 13 insertions(+), 5 deletions(-) diff --git a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile index f5ae66281..9bcb60c3a 100644 --- a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile +++ b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile @@ -58,8 +58,10 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ cmake ../ && make -j4 && sudo make install && cd ../../ && \ pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ + pip install numpy==1.24.2 scipy==1.9.3 cython==0.29.* git+https://github.com/ROCmSoftwarePlatform/hipify_torch.git && \ + env CC=$MPI_HOME/bin/mpicc python -m pip install mpi4py && \ git clone https://github.com/ROCmSoftwarePlatform/cupy.git && cd cupy && git submodule update --init && \ - pip install -e . --no-cache-dir -vvvv && pip install numpy==1.21 + pip install -e . --no-cache-dir -vvvv ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE diff --git a/docker/rocal-on-ubuntu-20.dockerfile b/docker/rocal-on-ubuntu-20.dockerfile index 73c74c370..94292ff55 100644 --- a/docker/rocal-on-ubuntu-20.dockerfile +++ b/docker/rocal-on-ubuntu-20.dockerfile @@ -54,8 +54,10 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ cmake ../ && make -j4 && sudo make install && cd ../../ && \ pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ + pip install numpy==1.24.2 scipy==1.9.3 cython==0.29.* git+https://github.com/ROCmSoftwarePlatform/hipify_torch.git && \ + env CC=$MPI_HOME/bin/mpicc python -m pip install mpi4py && \ git clone -b rocm6.1_internal_testing https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ - pip install -e . --no-cache-dir -vvvv && pip install numpy==1.21 + pip install -e . --no-cache-dir -vvvv ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE diff --git a/docker/rocal-on-ubuntu-22.dockerfile b/docker/rocal-on-ubuntu-22.dockerfile index cb3ceea5b..04514b8e1 100644 --- a/docker/rocal-on-ubuntu-22.dockerfile +++ b/docker/rocal-on-ubuntu-22.dockerfile @@ -57,8 +57,10 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ cmake ../ && make -j4 && sudo make install && cd ../../ && \ pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ + pip install numpy==1.24.2 scipy==1.9.3 cython==0.29.* git+https://github.com/ROCmSoftwarePlatform/hipify_torch.git && \ + env CC=$MPI_HOME/bin/mpicc python -m pip install mpi4py && \ git clone -b rocm6.1_internal_testing https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ - pip install -e . --no-cache-dir -vvvv && pip install numpy==1.21 + pip install -e . --no-cache-dir -vvvv ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE diff --git a/docker/rocal-with-pytorch.dockerfile b/docker/rocal-with-pytorch.dockerfile index b9b84eedb..a3ae16e3c 100644 --- a/docker/rocal-with-pytorch.dockerfile +++ b/docker/rocal-with-pytorch.dockerfile @@ -45,8 +45,10 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ cmake ../ && make -j4 && sudo make install && cd ../../ && \ pip install pytest==3.1 && git clone -b v2.10.4 https://github.com/pybind/pybind11 && cd pybind11 && mkdir build && cd build && \ cmake -DDOWNLOAD_CATCH=ON -DDOWNLOAD_EIGEN=ON ../ && make -j4 && sudo make install && cd ../../ && \ + pip install numpy==1.24.2 scipy==1.9.3 cython==0.29.* git+https://github.com/ROCmSoftwarePlatform/hipify_torch.git && \ + env CC=$MPI_HOME/bin/mpicc python -m pip install mpi4py && \ git clone -b rocm6.1_internal_testing https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ - pip install -e . --no-cache-dir -vvvv && pip install numpy==1.21 + pip install -e . --no-cache-dir -vvvv ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE diff --git a/rocAL-setup.py b/rocAL-setup.py index 7321d9044..30f0e29d2 100644 --- a/rocAL-setup.py +++ b/rocAL-setup.py @@ -345,8 +345,8 @@ os.system('sudo '+linuxFlag+' '+linuxSystemInstall + ' '+linuxSystemInstall_check+' install -y git g++ hipblas hipsparse rocrand hipfft rocfft rocthrust-devel hipcub-devel python3-devel') os.system('sudo -v') + os.system('pip install numpy==1.24.2 scipy==1.9.3 cython==0.29.*') os.system('(cd '+deps_dir+'; git clone -b rocm6.1_internal_testing https://github.com/ROCm/cupy.git; export CUPY_INSTALL_USE_HIP=1; export ROCM_HOME=/opt/rocm; cd cupy; git submodule update --init; pip install -e . --no-cache-dir -vvvv)') - os.system('pip install numpy==1.21') # Install ffmpeg if "Ubuntu" in platfromInfo: From c033f8a40a444556bd88f08ccd12244696e70d51 Mon Sep 17 00:00:00 2001 From: LAkshmiKuamr23 Date: Tue, 20 Feb 2024 12:59:06 -0800 Subject: [PATCH 10/12] build instructions update --- docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile | 6 +++--- docker/rocal-on-ubuntu-20.dockerfile | 6 +++--- docker/rocal-on-ubuntu-22.dockerfile | 6 +++--- docker/rocal-with-pytorch.dockerfile | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile index 9bcb60c3a..1f95e6163 100644 --- a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile +++ b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile @@ -67,9 +67,9 @@ ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE # install MIVisionX -RUN git clone https://github.com/ROCm/MIVisionX.git && \ - mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../MIVisionX && make -j8 && make install +RUN git clone https://github.com/ROCm/MIVisionX.git && cd MIVisionX && \ + mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../ && make -j8 && make install # Install rocAL -RUN git clone -b develop https://github.com/ROCm/rocAL && \ +RUN git clone -b develop https://github.com/ROCm/rocAL && cd rocAL && \ mkdir build && cd build && cmake ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file diff --git a/docker/rocal-on-ubuntu-20.dockerfile b/docker/rocal-on-ubuntu-20.dockerfile index 94292ff55..d62959567 100644 --- a/docker/rocal-on-ubuntu-20.dockerfile +++ b/docker/rocal-on-ubuntu-20.dockerfile @@ -63,9 +63,9 @@ ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE # install MIVisionX -RUN git clone https://github.com/ROCm/MIVisionX.git && \ - mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../MIVisionX && make -j8 && make install +RUN git clone https://github.com/ROCm/MIVisionX.git && cd MIVisionX && \ + mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../ && make -j8 && make install # Install rocAL -RUN git clone -b develop https://github.com/ROCm/rocAL && \ +RUN git clone -b develop https://github.com/ROCm/rocAL && cd rocAL && \ mkdir build && cd build && cmake ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file diff --git a/docker/rocal-on-ubuntu-22.dockerfile b/docker/rocal-on-ubuntu-22.dockerfile index 04514b8e1..6aecf18c5 100644 --- a/docker/rocal-on-ubuntu-22.dockerfile +++ b/docker/rocal-on-ubuntu-22.dockerfile @@ -66,9 +66,9 @@ ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE # Install MIVisionX -RUN git clone https://github.com/ROCm/MIVisionX && \ - mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../MIVisionX && make -j8 && make install +RUN git clone https://github.com/ROCm/MIVisionX && cd MIVisionX && \ + mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../ && make -j8 && make install # Install rocAL -RUN git clone -b develop https://github.com/ROCm/rocAL && \ +RUN git clone -b develop https://github.com/ROCm/rocAL && cd rocAL && \ mkdir build && cd build && cmake ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file diff --git a/docker/rocal-with-pytorch.dockerfile b/docker/rocal-with-pytorch.dockerfile index a3ae16e3c..7b1932bc5 100644 --- a/docker/rocal-with-pytorch.dockerfile +++ b/docker/rocal-with-pytorch.dockerfile @@ -54,9 +54,9 @@ ENV ROCAL_WORKSPACE=/workspace WORKDIR $ROCAL_WORKSPACE # install MIVisionX -RUN git clone https://github.com/ROCm/MIVisionX.git && \ +RUN git clone https://github.com/ROCm/MIVisionX.git && cd MIVisionX && \ mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../MIVisionX && make -j8 && make install # Install rocAL -RUN git clone -b develop https://github.com/ROCm/rocAL && \ +RUN git clone -b develop https://github.com/ROCm/rocAL && cd rocAL && \ mkdir build && cd build && cmake -DPYTHONVERSION=3.9 ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file From 3ff342a3fd6e778dc74706866fa6c835da8e9a7e Mon Sep 17 00:00:00 2001 From: LAkshmiKuamr23 Date: Tue, 20 Feb 2024 13:06:07 -0800 Subject: [PATCH 11/12] build updates --- ...ocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile | 8 ++++---- docker/rocal-on-ubuntu-20.dockerfile | 8 ++++---- docker/rocal-on-ubuntu-22.dockerfile | 8 ++++---- docker/rocal-with-pytorch.dockerfile | 10 +++++----- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile index 1f95e6163..6b2654f9b 100644 --- a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile +++ b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile @@ -63,13 +63,13 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ git clone https://github.com/ROCmSoftwarePlatform/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv -ENV ROCAL_WORKSPACE=/workspace -WORKDIR $ROCAL_WORKSPACE - # install MIVisionX RUN git clone https://github.com/ROCm/MIVisionX.git && cd MIVisionX && \ mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../ && make -j8 && make install +ENV ROCAL_WORKSPACE=/workspace +WORKDIR $ROCAL_WORKSPACE + # Install rocAL -RUN git clone -b develop https://github.com/ROCm/rocAL && cd rocAL && \ +RUN git clone -b develop https://github.com/ROCm/rocAL && \ mkdir build && cd build && cmake ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file diff --git a/docker/rocal-on-ubuntu-20.dockerfile b/docker/rocal-on-ubuntu-20.dockerfile index d62959567..48f83cf6a 100644 --- a/docker/rocal-on-ubuntu-20.dockerfile +++ b/docker/rocal-on-ubuntu-20.dockerfile @@ -59,13 +59,13 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ git clone -b rocm6.1_internal_testing https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv -ENV ROCAL_WORKSPACE=/workspace -WORKDIR $ROCAL_WORKSPACE - # install MIVisionX RUN git clone https://github.com/ROCm/MIVisionX.git && cd MIVisionX && \ mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../ && make -j8 && make install +ENV ROCAL_WORKSPACE=/workspace +WORKDIR $ROCAL_WORKSPACE + # Install rocAL -RUN git clone -b develop https://github.com/ROCm/rocAL && cd rocAL && \ +RUN git clone -b develop https://github.com/ROCm/rocAL && \ mkdir build && cd build && cmake ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file diff --git a/docker/rocal-on-ubuntu-22.dockerfile b/docker/rocal-on-ubuntu-22.dockerfile index 6aecf18c5..b622c3a4a 100644 --- a/docker/rocal-on-ubuntu-22.dockerfile +++ b/docker/rocal-on-ubuntu-22.dockerfile @@ -62,13 +62,13 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ git clone -b rocm6.1_internal_testing https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv -ENV ROCAL_WORKSPACE=/workspace -WORKDIR $ROCAL_WORKSPACE - # Install MIVisionX RUN git clone https://github.com/ROCm/MIVisionX && cd MIVisionX && \ mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../ && make -j8 && make install +ENV ROCAL_WORKSPACE=/workspace +WORKDIR $ROCAL_WORKSPACE + # Install rocAL -RUN git clone -b develop https://github.com/ROCm/rocAL && cd rocAL && \ +RUN git clone -b develop https://github.com/ROCm/rocAL && \ mkdir build && cd build && cmake ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file diff --git a/docker/rocal-with-pytorch.dockerfile b/docker/rocal-with-pytorch.dockerfile index 7b1932bc5..f3a432e8b 100644 --- a/docker/rocal-with-pytorch.dockerfile +++ b/docker/rocal-with-pytorch.dockerfile @@ -50,13 +50,13 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get -y install python3 python3-pip git g+ git clone -b rocm6.1_internal_testing https://github.com/ROCm/cupy.git && cd cupy && git submodule update --init && \ pip install -e . --no-cache-dir -vvvv -ENV ROCAL_WORKSPACE=/workspace -WORKDIR $ROCAL_WORKSPACE - # install MIVisionX RUN git clone https://github.com/ROCm/MIVisionX.git && cd MIVisionX && \ - mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../MIVisionX && make -j8 && make install + mkdir build && cd build && cmake -DBACKEND=HIP -DROCAL=OFF ../ && make -j8 && make install && cd + +ENV ROCAL_WORKSPACE=/workspace +WORKDIR $ROCAL_WORKSPACE # Install rocAL -RUN git clone -b develop https://github.com/ROCm/rocAL && cd rocAL && \ +RUN git clone -b develop https://github.com/ROCm/rocAL && \ mkdir build && cd build && cmake -DPYTHONVERSION=3.9 ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file From e195a025c614406928913b34db4792dca3bce108 Mon Sep 17 00:00:00 2001 From: LAkshmiKuamr23 Date: Tue, 20 Feb 2024 13:07:46 -0800 Subject: [PATCH 12/12] build updates --- docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile | 2 +- docker/rocal-on-ubuntu-20.dockerfile | 2 +- docker/rocal-on-ubuntu-22.dockerfile | 2 +- docker/rocal-with-pytorch.dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile index 6b2654f9b..a8d3ab5ef 100644 --- a/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile +++ b/docker/rocal-on-ubuntu-20-with-pytorch-with-mesa.dockerfile @@ -72,4 +72,4 @@ WORKDIR $ROCAL_WORKSPACE # Install rocAL RUN git clone -b develop https://github.com/ROCm/rocAL && \ - mkdir build && cd build && cmake ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file + mkdir build && cd build && cmake ../rocAL && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file diff --git a/docker/rocal-on-ubuntu-20.dockerfile b/docker/rocal-on-ubuntu-20.dockerfile index 48f83cf6a..24a25ed47 100644 --- a/docker/rocal-on-ubuntu-20.dockerfile +++ b/docker/rocal-on-ubuntu-20.dockerfile @@ -68,4 +68,4 @@ WORKDIR $ROCAL_WORKSPACE # Install rocAL RUN git clone -b develop https://github.com/ROCm/rocAL && \ - mkdir build && cd build && cmake ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file + mkdir build && cd build && cmake ../rocAL && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file diff --git a/docker/rocal-on-ubuntu-22.dockerfile b/docker/rocal-on-ubuntu-22.dockerfile index b622c3a4a..a28403457 100644 --- a/docker/rocal-on-ubuntu-22.dockerfile +++ b/docker/rocal-on-ubuntu-22.dockerfile @@ -71,4 +71,4 @@ WORKDIR $ROCAL_WORKSPACE # Install rocAL RUN git clone -b develop https://github.com/ROCm/rocAL && \ - mkdir build && cd build && cmake ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file + mkdir build && cd build && cmake ../rocAL && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file diff --git a/docker/rocal-with-pytorch.dockerfile b/docker/rocal-with-pytorch.dockerfile index f3a432e8b..f955e9146 100644 --- a/docker/rocal-with-pytorch.dockerfile +++ b/docker/rocal-with-pytorch.dockerfile @@ -59,4 +59,4 @@ WORKDIR $ROCAL_WORKSPACE # Install rocAL RUN git clone -b develop https://github.com/ROCm/rocAL && \ - mkdir build && cd build && cmake -DPYTHONVERSION=3.9 ../ && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file + mkdir build && cd build && cmake -DPYTHONVERSION=3.9 ../rocAL && make -j8 && cmake --build . --target PyPackageInstall && make install \ No newline at end of file