Merge branch 'main' of https://github.com/Apra-Labs/ApraPipes into H2…

…64ReversePlay
Apra-Labs · Sep 12, 2023 · aaab868 · aaab868
2 parents 30b250f + 2be555c
commit aaab868
Show file tree

Hide file tree

Showing 26 changed files with 661 additions and 363 deletions.
diff --git a/.github/workflows/build-test-lin.yml b/.github/workflows/build-test-lin.yml
@@ -30,7 +30,7 @@ on:
       prep-cmd:
         type: string
         description: 'commands required to be run on a builder to prep it for build'
-        default: 'sudo apt-get update -qq && sudo apt-get -y install ca-certificates curl zip unzip tar  autoconf   automake  autopoint build-essential  flex git-core   libass-dev   libfreetype6-dev   libgnutls28-dev   libmp3lame-dev   libsdl2-dev   libtool   libsoup-gnome2.4-dev   libva-dev   libvdpau-dev   libvorbis-dev   libxcb1-dev   libxcb-shm0-dev   libxcb-xfixes0-dev libncurses5-dev libncursesw5-dev  ninja-build   pkg-config   texinfo   wget   yasm   zlib1g-dev   nasm   gperf  bison python3 python3-pip dos2unix && pip3 install meson'
+        default: 'sudo apt-get update -qq && sudo apt-get -y install ca-certificates curl zip unzip tar  autoconf   automake  autopoint build-essential  flex git-core   libass-dev   libfreetype6-dev   libgnutls28-dev   libmp3lame-dev   libsdl2-dev   libtool   libsoup-gnome2.4-dev   libva-dev   libvdpau-dev   libvorbis-dev   libxdamage-dev   libxcb1-dev   libxcb-shm0-dev   libxcb-xfixes0-dev libncurses5-dev libncursesw5-dev  ninja-build   pkg-config   texinfo   wget   yasm   zlib1g-dev   nasm   gperf  bison python3 python3-pip dos2unix && pip3 install meson'
         required: false
       prep-check-cmd:
         type: string

diff --git a/README.md b/README.md
@@ -46,34 +46,16 @@ Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson
 
   ### Cudnn
   * Download [Cudnn](https://developer.nvidia.com/rdp/cudnn-archive#a-collapse765-102) and extract files where cuda is installed. Note: Please be aware that this process requires some effort. Here are the necessary steps:
-   * Download the correct tar/zip file matching your cuda version. _Do not download the exe/installer/deb package._
+   * Download the correct zip file matching your cuda version. _Do not download the exe/installer/deb package._
    * Windows: 
-     * Download [this file](https://developer.nvidia.com/compute/cudnn/secure/8.3.2/local_installers/10.2/cudnn-windows-x86_64-8.3.2.44_cuda10.2-archive.zip). 
-     * Extract the downloaded file and copy files to ```C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2``` using an __administrative command prompt__ as follows
-       ```
-       cd .\extracted_folder
-       cd include
-       copy *.h "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\include\"
-       cd ..\lib
-       copy *.lib "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\lib\x64\"
-       cd ..\bin
-       copy *.dll "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\bin\"
-       ```
+     * Download [this file](https://developer.nvidia.com/compute/cudnn/secure/8.3.2/local_installers/10.2/cudnn-windows-x86_64-8.3.2.44_cuda10.2-archive.zip).
+
   ###  Prerequisites 
   * Install Visual Studio 2019 Community 
     * Install Desktop development C++
     * .NET Desktop development
     * Universal Windows Development Platform
-  * Install choco:
-    Open Windows PowerShell as Administrator and run:
-    ```
-    Set-ExecutionPolicy AllSigned
-    Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
-    ``` 
-  * Install build dependencies using choco: 
-    ```
-    choco feature enable -n allowEmptyChecksums && choco install 7zip git python3 cmake pkgconfiglite -y && pip3 install ninja && pip3 install meson
-    ```
+
   * Clone with submodules and LFS. 
     ```
     git clone --recursive https://github.com/Apra-Labs/ApraPipes.git
@@ -85,6 +67,8 @@ Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson
   <summary>Build</summary>
 
   ### Build Without Cuda
+  Open PowerShell as an administrator and execute the following commands
+
   If your windows system does not have an NVIDIA GPU use this script
   ```
   build_windows_no_cuda.bat
@@ -137,39 +121,11 @@ Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson
     20.04 - [CUDA Toolkit 11.7](https://developer.nvidia.com/cuda-11-7-0-download-archive?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=20.04)
 
   ### Cudnn  
-  * Download [Cudnn](https://developer.nvidia.com/rdp/cudnn-archive#a-collapse765-102) and extract files where cuda is installed. Note: Please be aware that this process requires some effort. Here are the necessary steps:
+  * Download [Cudnn](https://developer.nvidia.com/rdp/cudnn-archive#a-collapse765-102)
   * Linux:
       * Download [this file](https://developer.nvidia.com/compute/cudnn/secure/8.3.2/local_installers/10.2/cudnn-linux-x86_64-8.3.2.44_cuda10.2-archive.tar.xz)
-      * extract the files
-        ``` 
-        xz -d cudnn-linux-x86_64-8.3.2.44_cuda10.2-archive.tar.xz
-        tar xvf cudnn-linux-x86_64-8.3.2.44_cuda10.2-archive.tar
-        ```
-      * copy files retaining the links
-        ```
-        cd ./cudnn-linux-x86_64-8.3.2.44_cuda10.2-archive
-        sudo cp -P include/* /usr/local/cuda/include/
-        sudo cp -P lib/* /usr/local/cuda/lib64/
-        ```
-  * To add Cuda to path append following lines to ~/.bashrc. 
-    ```
-    export PATH=/usr/local/cuda/bin${PATH:+:${PATH}}
-    export LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
-    ```
-  * Reload ~/.bashrc:
-    ```
-    source ~/.bashrc:
-    ```
-  * Verify by checking nvcc --version and nvidia-smi both must point to correct versions of cuda and nvidia driver in the system.
+
   ### Prerequisites
-  * Run the following to get latest build tools
-    ```
-    sudo apt-get update && sudo apt-get -y install   autoconf   automake  autopoint  build-essential  git-core  git-lfs libass-dev   libfreetype6-dev  libgnutls28-dev   libmp3lame-dev libsdl2-dev  libssl-dev libtool libsoup-gnome2.4-dev libncurses5-dev libva-dev   libvdpau-dev   libvorbis-dev   libxcb1-dev   libxcb-shm0-dev   libxcb-xfixes0-dev  ninja-build   pkg-config   texinfo   wget   yasm   zlib1g-dev   nasm   gperf bison curl zip unzip tar python3-pip flex && pip3 install meson
-    ```  
-  * Note: start a new terminal as pip3 settings do not get effective on the same shell
-  * CMake minimum version 3.24 - Follow [this article](https://anglehit.com/how-to-install-the-latest-version-of-cmake-via-command-line/) to update cmake
-  * Clone with submodules and LFS. 
-    ```
     git clone --recursive https://github.com/Apra-Labs/ApraPipes.git
     ```
 
@@ -185,11 +141,11 @@ Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson
   ### Build Without Cuda
   If your windows system does not have an NVIDIA GPU use this script
   ```
-  ./build_linux_no_cuda.sh
+  sudo ./build_linux_no_cuda.sh
   ```
   ### Build With Cuda
   ```
-  ./build_linux_cuda.sh
+  sudo ./build_linux_cuda.sh
   ```
 
   Build can take ~2 hours depending on the machine configuration.
@@ -229,26 +185,11 @@ Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson
   
   ###  Prerequisites
   * Setup the board with [Jetpack 4.4](https://docs.nvidia.com/sdk-manager/install-with-sdkm-jetson/index.html) or higher as supported.
-  * Run the following commands to build required libraries. 
-    ```
-    sudo apt-get update && sudo apt-get -y install git-lfs libncurses5-dev ninja-build nasm curl libudev-dev libssl-dev && sudo snap install cmake --classic
-    ```
-  * Append following lines to ~/.bashrc. 
-    ```
-    export VCPKG_FORCE_SYSTEM_BINARIES=1
-    export PATH=/usr/local/cuda/bin${PATH:+:${PATH}}
-    export LD_LIBRARY_PATH=/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
-    ```
-  * Reload ~/.bashrc:
-    ```
-    source ~/.bashrc:
-    ```
+  
   * Clone with submodules and LFS. 
     ```
     git clone --recursive https://github.com/Apra-Labs/ApraPipes.git
     ```
-  * Run `./bootstrap-vcpkg.sh` in vcpkg/ directory
-  * Run `./vcpkg integrate install`
 </details>
 
 <details>
@@ -261,7 +202,7 @@ Aprapipes is automatically built and tested on Ubuntu (18.04 and 20.04), Jetson
   ```
   * ApraPipes builds CUDA version on Jerson Boads.
   ```
-  ./build_jetson.sh
+  sudo ./build_jetson.sh
   ```
 
   Build can take ~12 hours on Jetson Nano. 

diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt
@@ -467,7 +467,6 @@ IF (ENABLE_ARM64)
 		test/jpegdecoderl4tm_tests.cpp
 #		test/l4tm_dec_enc_1_tests.cpp  #todo this test needs to be improved to add to jetson suite
 		test/opencvresize_tests.cpp
-		test/h264encoderv4l2helper_tests.cpp
 		test/h264encoderv4l2_tests.cpp
 		test/nvarguscamerahelper_tests.cpp
 		test/nvarguscamera_tests.cpp

diff --git a/base/fix-vcpkg-json.ps1 b/base/fix-vcpkg-json.ps1
@@ -3,11 +3,11 @@ param([String]$fileName='vcpkg.json', [switch]$removeOpenCV,  [switch]$removeCUD
 
 $v = Get-Content $fileName -raw | ConvertFrom-Json
 
-if($removeCUDA.IsPresent)
+if ($removeCUDA.IsPresent)
 {
-    $opencv = $v.dependencies | Where-Object { $_.name -eq 'opencv4'}
-    $opencv.features= $opencv.features | Where-Object { $_ -ne 'cuda' }
-    $opencv.features= $opencv.features | Where-Object { $_ -ne 'cudnn' }
+    $v.dependencies |
+        Where-Object { $_.name -eq 'opencv4' } |
+        ForEach-Object { $_.features = $_.features -ne 'cuda' -ne 'cudnn' }
 }
 
 if($removeOpenCV.IsPresent)
@@ -24,4 +24,4 @@ if($onlyOpenCV.IsPresent)
 }
 
 
-$v | ConvertTo-Json -depth 32| set-content $fileName
+$v | ConvertTo-Json -depth 32| set-content $fileName
diff --git a/base/fix-vcpkg-json.sh b/base/fix-vcpkg-json.sh
@@ -13,35 +13,47 @@ echo "$v"
 
 # Process command line arguments
 if $removeCUDA; then
-    echo "Removing CUDA..."   
-    opencv=$(echo "$v" | jq '.dependencies[] | select(type == "string" or .name == "opencv4")')
-    opencv=$(echo "$opencv" | jq '.features |= map(select(. != "cuda" and . != "cudnn"))')
-    v=$(echo "$v" | jq '.dependencies[] |= select(type == "string" or .name != "opencv4")')
-    v=$(echo "$v" | jq '.dependencies[] |= select(type != "null")')
-    v=$(echo "$v" | jq ".dependencies += [$opencv]")
+    echo "Removing CUDA..."
+    # Loop through each "opencv4" instance
+    for index in $(echo "$v" | jq -r '.dependencies | keys | .[]'); do
+        name=$(echo "$v" | jq -r ".dependencies[$index].name")
+        if [ "$name" == "opencv4" ]; then
+            # Remove "cuda" and "cudnn" features for this "opencv4" instance
+            v=$(echo "$v" | jq ".dependencies[$index].features |= map(select(. != \"cuda\" and . != \"cudnn\"))")
+        fi
+    done
 fi
 
 if $removeOpenCV; then
     echo "Removing OpenCV..."
-     v=$(echo "$v" | jq '.dependencies[] |= select(type == "string" or .name != "opencv4")')
-     v=$(echo "$v" | jq '.dependencies[] |= select(type != "null")')
+    count=$(echo "$v" | jq '.dependencies | length')
+    for ((index=count-1; index >= 0; index--)); do
+        name=$(echo "$v" | jq -r ".dependencies[$index].name")
+        if [ "$name" == "opencv4" ]; then
+            # Remove the entire "opencv4" instance
+            v=$(echo "$v" | jq "del(.dependencies[$index])")
+        fi
+    done
 fi
 
 if $onlyOpenCV; then
-     echo "Keeping only OpenCV..."
-     opencv=$(echo "$v" | jq '.dependencies[] | select(.name == "opencv4")')
-     opencv=$(echo "$opencv" | jq '.features |= map(select(. != "cuda" and . != "cudnn"))')
-     v=$(echo "$v" | jq '.dependencies = []')
-     v=$(echo "$v" | jq ".dependencies += [$opencv]")
-
+    echo "Keeping only OpenCV..."
+    # Loop through each dependency and remove other dependencies except "opencv4"
+    count=$(echo "$v" | jq '.dependencies | length')
+    for ((index = count - 1; index >= 0; index--)); do
+        name=$(echo "$v" | jq -r ".dependencies[$index].name")
+        if [ "$name" != "opencv4" ]; then
+            # Remove the entire dependency
+            v=$(echo "$v" | jq "del(.dependencies[$index])")
+        fi
+    done
 fi
 
- #Save the modified object back to the JSON file
- echo "Modified JSON file contents:"
- echo "$v" | jq .
- echo "$v" | jq . > "$fileName"
+# Save the modified object back to the JSON file
+echo "Modified JSON file contents:"
+echo "$v" | jq .
+echo "$v" | jq . > "$fileName"
 
 # Print all command line arguments
 echo "Command line arguments:"
-echo "$@"
-
+echo "$@"
diff --git a/base/include/ColorConversionXForm.h b/base/include/ColorConversionXForm.h
@@ -41,6 +41,7 @@ class ColorConversion : public Module
 protected:
 	bool process(frame_container& frames);
 	bool processSOS(frame_sp& frame);
+	bool shouldTriggerSOS();
 	bool validateInputPins();
 	bool validateOutputPins();
 	void setConversionStrategy(framemetadata_sp inputMetadata, framemetadata_sp outputMetadata);
@@ -53,5 +54,7 @@ class ColorConversion : public Module
 	std::string mOutputPinId;
 	framemetadata_sp mOutputMetadata;
 	framemetadata_sp mInputMetadata;
+	bool mShouldTriggerSos = true;
+	bool setMetadata(framemetadata_sp& metadata);
 };
 
diff --git a/base/include/H264EncoderV4L2.h b/base/include/H264EncoderV4L2.h
@@ -7,12 +7,15 @@ class H264EncoderV4L2Helper;
 class H264EncoderV4L2Props : public ModuleProps
 {
 public:
-	H264EncoderV4L2Props(): targetKbps(1024)
+	H264EncoderV4L2Props(bool _enableMotionVectors = false, int _motionVectorThreshold = 5): targetKbps(1024)
 	{
-
+		enableMotionVectors = _enableMotionVectors;
+		motionVectorThreshold = _motionVectorThreshold;
 	}
 
 	uint32_t targetKbps;
+	bool enableMotionVectors = false;
+	int motionVectorThreshold = 5;
 };
 
 class H264EncoderV4L2 : public Module
@@ -36,6 +39,6 @@ class H264EncoderV4L2 : public Module
 	std::shared_ptr<H264EncoderV4L2Helper> mHelper;
 
 	H264EncoderV4L2Props mProps;
-	framemetadata_sp mOutputMetadata;
-	std::string mOutputPinId;
+	std::string motionVectorFramePinId;
+	std::string h264FrameOutputPinId;
 };
diff --git a/base/include/H264EncoderV4L2Helper.h b/base/include/H264EncoderV4L2Helper.h
@@ -1,19 +1,18 @@
 #pragma once
+#include <boost/pool/object_pool.hpp>
 
+#include "ExtFrame.h"
 #include "AV4L2ElementPlane.h"
 #include "V4L2CUYUV420Converter.h"
-
-#include "ExtFrame.h"
-#include <boost/pool/object_pool.hpp>
+#include <v4l2_nv_extensions.h>
 
 class H264EncoderV4L2Helper
 {
 public:
-    typedef std::function<void(frame_sp &)> SendFrame;
+    typedef std::function<void(frame_container& errorFrame)> SendFrameContainer;    
+    static std::shared_ptr<H264EncoderV4L2Helper> create(enum v4l2_memory memType, uint32_t pixelFormat, uint32_t width, uint32_t height, uint32_t step, uint32_t bitrate, bool enableMotionVectors, int motionVectorThreshold, uint32_t fps, std::string h264FrameOutputPinId, std::string motionVectorFramePinId,  framemetadata_sp h264Metadata, std::function<frame_sp(size_t size, string& pinId)> makeFrame, SendFrameContainer sendFrameContainer);
 
-    static std::shared_ptr<H264EncoderV4L2Helper> create(enum v4l2_memory memType, uint32_t pixelFormat, uint32_t width, uint32_t height, uint32_t step, uint32_t bitrate, uint32_t fps, SendFrame sendFrame);
-
-    H264EncoderV4L2Helper(enum v4l2_memory memType, uint32_t pixelFormat, uint32_t width, uint32_t height, uint32_t step, uint32_t bitrate, uint32_t fps, SendFrame sendFrame);
+    H264EncoderV4L2Helper(enum v4l2_memory memType, uint32_t pixelFormat, uint32_t width, uint32_t height, uint32_t step, uint32_t bitrate, bool enableMotionVectors, int motionVectorThreshold, uint32_t fps,std::string h264FrameOutputPinId, std::string motionVectorFramePinId,  framemetadata_sp h264Metadata, std::function<frame_sp(size_t size, string& pinId)> makeFrame, SendFrameContainer sendFrameContainer);
     ~H264EncoderV4L2Helper();
 
     void stop();
@@ -32,22 +31,33 @@ class H264EncoderV4L2Helper
     void setLevel();
     void setFrameRate(uint32_t framerate_num, uint32_t framerate_den);
 
+    int setExtControlsMV(v4l2_ext_controls &ctl);
+    int enableMotionVectorReporting();
     void initEncoderParams(uint32_t bitrate, uint32_t fps);
     int setExtControls(v4l2_ext_control &control);
 
     void capturePlaneDQCallback(AV4L2Buffer *buffer);
     void reuseCatureBuffer(ExtFrame *pointer, uint32_t index, std::shared_ptr<H264EncoderV4L2Helper> self);
 
     bool processEOS();
-
+    int getExtControls(v4l2_ext_controls &ctl);
+    int getMotionVectors(uint32_t buffer_index,
+            v4l2_ctrl_videoenc_outputbuf_metadata_MV &enc_mv_metadata);
+    void serializeMotionVectors(v4l2_ctrl_videoenc_outputbuf_metadata_MV enc_mv_metadata, frame_container &frames);
 private:
     std::shared_ptr<H264EncoderV4L2Helper> mSelf;
     int mFD;
     std::unique_ptr<AV4L2ElementPlane> mOutputPlane;
     std::unique_ptr<AV4L2ElementPlane> mCapturePlane;
-
     boost::object_pool<ExtFrame> frame_opool;
-    SendFrame mSendFrame;
-
+    SendFrameContainer mSendFrameContainer;
+    int mWidth = 0;
+    int mHeight = 0;
+    bool enableMotionVectors;
+    int motionVectorThreshold;
+    std::string h264FrameOutputPinId;
+    std::string motionVectorFramePinId;
+    framemetadata_sp h264Metadata;
+    std::function<frame_sp(size_t size, string& pinId)> makeFrame;
     std::unique_ptr<V4L2CUYUV420Converter> mConverter;
 };