1. Added motion vector as 2nd output from the H264EncoderV4l2 module.

base/CMakeLists.txt

@@ -467,7 +467,6 @@ IF (ENABLE_ARM64)
 		test/jpegdecoderl4tm_tests.cpp
 #		test/l4tm_dec_enc_1_tests.cpp  #todo this test needs to be improved to add to jetson suite
 		test/opencvresize_tests.cpp
-		test/h264encoderv4l2helper_tests.cpp
 		test/h264encoderv4l2_tests.cpp
 		test/nvarguscamerahelper_tests.cpp
 		test/nvarguscamera_tests.cpp

base/include/H264EncoderV4L2.h

@@ -7,12 +7,15 @@ class H264EncoderV4L2Helper;
 class H264EncoderV4L2Props : public ModuleProps
 {
 public:
-	H264EncoderV4L2Props(): targetKbps(1024)
+	H264EncoderV4L2Props(bool _enableMotionVectors = false, int _motionVectorThreshold = 5): targetKbps(1024)
 	{
-
+		enableMotionVectors = _enableMotionVectors;
+		motionVectorThreshold = _motionVectorThreshold;
 	}
 
 	uint32_t targetKbps;
+	bool enableMotionVectors = false;
+	int motionVectorThreshold = 5;
 };
 
 class H264EncoderV4L2 : public Module
@@ -36,6 +39,6 @@ class H264EncoderV4L2 : public Module
 	std::shared_ptr<H264EncoderV4L2Helper> mHelper;
 
 	H264EncoderV4L2Props mProps;
-	framemetadata_sp mOutputMetadata;
-	std::string mOutputPinId;
+	std::string motionVectorFramePinId;
+	std::string h264FrameOutputPinId;
 };

base/include/H264EncoderV4L2Helper.h

@@ -1,19 +1,18 @@
 #pragma once
+#include <boost/pool/object_pool.hpp>
 
+#include "ExtFrame.h"
 #include "AV4L2ElementPlane.h"
 #include "V4L2CUYUV420Converter.h"
-
-#include "ExtFrame.h"
-#include <boost/pool/object_pool.hpp>
+#include <v4l2_nv_extensions.h>
 
 class H264EncoderV4L2Helper
 {
 public:
-    typedef std::function<void(frame_sp &)> SendFrame;
+    typedef std::function<void(frame_container& errorFrame)> SendFrameContainer;    
+    static std::shared_ptr<H264EncoderV4L2Helper> create(enum v4l2_memory memType, uint32_t pixelFormat, uint32_t width, uint32_t height, uint32_t step, uint32_t bitrate, bool enableMotionVectors, int motionVectorThreshold, uint32_t fps, std::string h264FrameOutputPinId, std::string motionVectorFramePinId,  framemetadata_sp h264Metadata, std::function<frame_sp(size_t size, string& pinId)> makeFrame, SendFrameContainer sendFrameContainer);
 
-    static std::shared_ptr<H264EncoderV4L2Helper> create(enum v4l2_memory memType, uint32_t pixelFormat, uint32_t width, uint32_t height, uint32_t step, uint32_t bitrate, uint32_t fps, SendFrame sendFrame);
-
-    H264EncoderV4L2Helper(enum v4l2_memory memType, uint32_t pixelFormat, uint32_t width, uint32_t height, uint32_t step, uint32_t bitrate, uint32_t fps, SendFrame sendFrame);
+    H264EncoderV4L2Helper(enum v4l2_memory memType, uint32_t pixelFormat, uint32_t width, uint32_t height, uint32_t step, uint32_t bitrate, bool enableMotionVectors, int motionVectorThreshold, uint32_t fps,std::string h264FrameOutputPinId, std::string motionVectorFramePinId,  framemetadata_sp h264Metadata, std::function<frame_sp(size_t size, string& pinId)> makeFrame, SendFrameContainer sendFrameContainer);
     ~H264EncoderV4L2Helper();
 
     void stop();
@@ -32,22 +31,33 @@ class H264EncoderV4L2Helper
     void setLevel();
     void setFrameRate(uint32_t framerate_num, uint32_t framerate_den);
 
+    int setExtControlsMV(v4l2_ext_controls &ctl);
+    int enableMotionVectorReporting();
     void initEncoderParams(uint32_t bitrate, uint32_t fps);
     int setExtControls(v4l2_ext_control &control);
 
     void capturePlaneDQCallback(AV4L2Buffer *buffer);
     void reuseCatureBuffer(ExtFrame *pointer, uint32_t index, std::shared_ptr<H264EncoderV4L2Helper> self);
 
     bool processEOS();
-
+    int getExtControls(v4l2_ext_controls &ctl);
+    int getMotionVectors(uint32_t buffer_index,
+            v4l2_ctrl_videoenc_outputbuf_metadata_MV &enc_mv_metadata);
+    void serializeMotionVectors(v4l2_ctrl_videoenc_outputbuf_metadata_MV enc_mv_metadata, frame_container &frames);
 private:
     std::shared_ptr<H264EncoderV4L2Helper> mSelf;
     int mFD;
     std::unique_ptr<AV4L2ElementPlane> mOutputPlane;
     std::unique_ptr<AV4L2ElementPlane> mCapturePlane;
-
     boost::object_pool<ExtFrame> frame_opool;
-    SendFrame mSendFrame;
-
+    SendFrameContainer mSendFrameContainer;
+    int mWidth = 0;
+    int mHeight = 0;
+    bool enableMotionVectors;
+    int motionVectorThreshold;
+    std::string h264FrameOutputPinId;
+    std::string motionVectorFramePinId;
+    framemetadata_sp h264Metadata;
+    std::function<frame_sp(size_t size, string& pinId)> makeFrame;
     std::unique_ptr<V4L2CUYUV420Converter> mConverter;
 };

base/src/H264EncoderV4L2.cpp

@@ -5,12 +5,15 @@
 #include "Frame.h"
 #include "Logger.h"
 #include "Utils.h"
+#include "H264Metadata.h"
 #include "AIPExceptions.h"
 
 H264EncoderV4L2::H264EncoderV4L2(H264EncoderV4L2Props props) : Module(TRANSFORM, "H264EncoderV4L2", props), mProps(props)
 {
-	mOutputMetadata = framemetadata_sp(new FrameMetadata(FrameMetadata::H264_DATA));
-	mOutputPinId = addOutputPin(mOutputMetadata);
+	auto mOutputMetadata = framemetadata_sp(new H264Metadata(0, 0));
+	auto motionVectorOutputMetadata = framemetadata_sp(new FrameMetadata(FrameMetadata::OVERLAY_INFO_IMAGE));
+	h264FrameOutputPinId = addOutputPin(mOutputMetadata);
+	motionVectorFramePinId = addOutputPin(motionVectorOutputMetadata);
 }
 
 H264EncoderV4L2::~H264EncoderV4L2() 
@@ -55,15 +58,10 @@ bool H264EncoderV4L2::validateInputPins()
 
 bool H264EncoderV4L2::validateOutputPins()
 {
-	if (getNumberOfOutputPins() != 1)
-	{
-		LOG_ERROR << "<" << getId() << ">::validateOutputPins size is expected to be 1. Actual<" << getNumberOfOutputPins() << ">";
-		return false;
-	}
 
 	framemetadata_sp metadata = getFirstOutputMetadata();
 	FrameMetadata::FrameType frameType = metadata->getFrameType();
-	if (frameType != FrameMetadata::H264_DATA)
+	if (frameType != FrameMetadata::H264_DATA && frameType != FrameMetadata::OVERLAY_INFO_IMAGE)
 	{
 		LOG_ERROR << "<" << getId() << ">::validateOutputPins input frameType is expected to be H264_DATA. Actual<" << frameType << ">";
 		return false;
@@ -144,12 +142,14 @@ bool H264EncoderV4L2::processSOS(frame_sp &frame)
 		v4l2MemType = V4L2_MEMORY_DMABUF;
 	}
 
-	mHelper = H264EncoderV4L2Helper::create(v4l2MemType, pixelFormat, width, height, step, 1024 * mProps.targetKbps, 30, [&](frame_sp &frame) -> void {
-		frame->setMetadata(mOutputMetadata);
-
-		frame_container frames;
-		frames.insert(make_pair(mOutputPinId, frame));
-		send(frames);
+	auto h264OutMetadata = framemetadata_sp(new H264Metadata(width, height));
+	Module::setMetadata(h264FrameOutputPinId, h264OutMetadata);
+
+	mHelper = H264EncoderV4L2Helper::create(v4l2MemType, pixelFormat, width, height, step, 1024 * mProps.targetKbps, mProps.enableMotionVectors, mProps.motionVectorThreshold, 30,h264FrameOutputPinId, motionVectorFramePinId, h264OutMetadata,
+	[&](size_t size, string& pinId)
+	{ return makeFrame(size, pinId); },
+	[&](frame_container &frameContainer) -> void {
+		send(frameContainer);
 	});
 
 	return true;

base/src/H264EncoderV4L2Helper.cpp

@@ -1,9 +1,10 @@
-#include "H264EncoderV4L2Helper.h"
-#include "v4l2_nv_extensions.h"
-
+#include <cmath>
+#include "Overlay.h"
 #include "Logger.h"
 #include "AIPExceptions.h"
 
+#include "H264EncoderV4L2Helper.h"
+
 inline bool checkv4l2(int ret, int iLine, const char *szFile, std::string message, bool raiseException)
 {
     if (ret < 0)
@@ -23,9 +24,9 @@ inline bool checkv4l2(int ret, int iLine, const char *szFile, std::string messag
 
 #define CHECKV4L2(call, message, raiseException) checkv4l2(call, __LINE__, __FILE__, message, raiseException)
 
-std::shared_ptr<H264EncoderV4L2Helper> H264EncoderV4L2Helper::create(enum v4l2_memory memType, uint32_t pixelFormat, uint32_t width, uint32_t height, uint32_t step, uint32_t bitrate, uint32_t fps, SendFrame sendFrame)
+std::shared_ptr<H264EncoderV4L2Helper> H264EncoderV4L2Helper::create(enum v4l2_memory memType, uint32_t pixelFormat, uint32_t width, uint32_t height, uint32_t step, uint32_t bitrate, bool enableMotionVectors, int motionVectorThreshold, uint32_t fps, std::string h264FrameOutputPinId, std::string motionVectorFramePinId,  framemetadata_sp h264Metadata, std::function<frame_sp(size_t size, string& pinId)> makeFrame, SendFrameContainer sendFrameContainer)
 {
-    auto instance = std::make_shared<H264EncoderV4L2Helper>(memType, pixelFormat, width, height, step, bitrate, fps, sendFrame);
+    auto instance = std::make_shared<H264EncoderV4L2Helper>(memType, pixelFormat, width, height, step, bitrate, enableMotionVectors, motionVectorThreshold, fps,h264FrameOutputPinId, motionVectorFramePinId, h264Metadata, makeFrame, sendFrameContainer);
     instance->setSelf(instance);
 
     return instance;
@@ -36,7 +37,7 @@ void H264EncoderV4L2Helper::setSelf(std::shared_ptr<H264EncoderV4L2Helper> &self
     mSelf = self;
 }
 
-H264EncoderV4L2Helper::H264EncoderV4L2Helper(enum v4l2_memory memType, uint32_t pixelFormat, uint32_t width, uint32_t height, uint32_t step, uint32_t bitrate, uint32_t fps, SendFrame sendFrame) : mSendFrame(sendFrame), mFD(-1)
+H264EncoderV4L2Helper::H264EncoderV4L2Helper(enum v4l2_memory memType, uint32_t pixelFormat, uint32_t width, uint32_t height, uint32_t step, uint32_t bitrate, bool _enableMotionVectors, int _motionVectorThreshold, uint32_t fps, std::string _h264FrameOutputPinId, std::string _motionVectorFramePinId,  framemetadata_sp _h264Metadata, std::function<frame_sp(size_t size, string& pinId)> _makeFrame, SendFrameContainer sendFrameContainer) : mSendFrameContainer(sendFrameContainer), mFD(-1), mWidth(width), mHeight(height), enableMotionVectors(_enableMotionVectors), motionVectorThreshold(_motionVectorThreshold), h264FrameOutputPinId(_h264FrameOutputPinId), motionVectorFramePinId(_motionVectorFramePinId), h264Metadata(_h264Metadata), makeFrame(_makeFrame)
 {
     initV4L2();
 
@@ -124,12 +125,47 @@ void H264EncoderV4L2Helper::initV4L2()
     }
 }
 
+
+int
+H264EncoderV4L2Helper::setExtControlsMV(v4l2_ext_controls &ctl)
+{
+    int ret;
+
+    ret = v4l2_ioctl(mFD, VIDIOC_S_EXT_CTRLS, &ctl);
+
+    return ret;
+}
+
+
+int
+H264EncoderV4L2Helper::enableMotionVectorReporting()
+{
+    struct v4l2_ext_control control;
+    struct v4l2_ext_controls ctrls;
+
+    memset(&control, 0, sizeof(control));
+    memset(&ctrls, 0, sizeof(ctrls));
+
+    ctrls.count = 1;
+    ctrls.controls = &control;
+    ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG;
+
+    control.id = V4L2_CID_MPEG_VIDEOENC_ENABLE_METADATA_MV;
+    control.value = 1;
+
+    setExtControlsMV(ctrls);
+}
+
 void H264EncoderV4L2Helper::initEncoderParams(uint32_t bitrate, uint32_t fps)
 {
     setBitrate(bitrate);
     setProfile();
     setLevel();
     setFrameRate(fps, 1);
+    if(enableMotionVectors)
+    {
+        enableMotionVectorReporting();
+    }
 }
 
 void H264EncoderV4L2Helper::setBitrate(uint32_t bitrate)
@@ -200,10 +236,94 @@ int H264EncoderV4L2Helper::setExtControls(v4l2_ext_control &control)
     return v4l2_ioctl(mFD, VIDIOC_S_EXT_CTRLS, &ctrls);
 }
 
+int
+H264EncoderV4L2Helper::getExtControls(v4l2_ext_controls &ctl)
+{
+    int ret;
+
+    ret = v4l2_ioctl(mFD, VIDIOC_G_EXT_CTRLS, &ctl);
+
+    return ret;
+}
+
+int
+H264EncoderV4L2Helper::getMotionVectors(uint32_t buffer_index,
+        v4l2_ctrl_videoenc_outputbuf_metadata_MV &enc_mv_metadata)
+{
+    v4l2_ctrl_video_metadata metadata;
+    struct v4l2_ext_control control;
+    struct v4l2_ext_controls ctrls;
+
+    ctrls.count = 1;
+    ctrls.controls = &control;
+    ctrls.ctrl_class = V4L2_CTRL_CLASS_MPEG;
+
+    metadata.buffer_index = buffer_index;
+    metadata.VideoEncMetadataMV = &enc_mv_metadata;
+
+    control.id = V4L2_CID_MPEG_VIDEOENC_METADATA_MV;
+    control.string = (char *)&metadata;
+
+    getExtControls(ctrls);
+}
+
+void H264EncoderV4L2Helper::serializeMotionVectors(v4l2_ctrl_videoenc_outputbuf_metadata_MV enc_mv_metadata, frame_container &frames)
+{
+    uint32_t numMVs = enc_mv_metadata.bufSize / sizeof(MVInfo);
+    MVInfo *pInfo = enc_mv_metadata.pMVInfo;
+
+    std::vector<CircleOverlay> circleOverlays;
+    CompositeOverlay compositeOverlay;
+
+    int totalMacroblockInRow = floor(mWidth / 16); // Tells about the total number of macro blocks in each row.
+    for (uint32_t i = 0; i < numMVs; i++, pInfo++) // numMVs is the total macroblock in the frame.
+    {
+
+        if (abs(pInfo->mv_x) > motionVectorThreshold || abs(pInfo->mv_y) > motionVectorThreshold)
+        {
+            auto tempY = floor(i / totalMacroblockInRow); // i represents current macroblock , To get the y offset of macroblock the current macroblock is divided by macroblock across width.
+            auto y = tempY * 16 + 8; // Here every macroblock is of 16x16 , So multiply it by 16. To get to the centre of the macroblock add it by 8.
+            auto tempX = floor(i % totalMacroblockInRow);
+            auto x = tempX * 16 + 8;
+            CircleOverlay circleOverlay;
+            circleOverlay.x1 = x;
+            circleOverlay.y1 = y;
+            circleOverlay.radius = 1;
+            circleOverlays.push_back(circleOverlay);
+        }
+    }
+
+    for (auto &circleOverlay : circleOverlays)
+    {
+        compositeOverlay.add(&circleOverlay);
+    }
+
+    if (circleOverlays.size())
+    {
+        DrawingOverlay drawingOverlay;
+        drawingOverlay.add(&compositeOverlay);
+        auto serializeSize = drawingOverlay.mGetSerializeSize();
+        serializeSize += 100;
+        auto motionVectorFrame = makeFrame(serializeSize, motionVectorFramePinId);
+        drawingOverlay.serialize(motionVectorFrame);
+        frames.insert(make_pair(motionVectorFramePinId, motionVectorFrame));
+    }
+}
 void H264EncoderV4L2Helper::capturePlaneDQCallback(AV4L2Buffer *buffer)
 {
     auto frame = frame_sp(frame_opool.construct(buffer->planesInfo[0].data, buffer->v4l2_buf.m.planes[0].bytesused), std::bind(&H264EncoderV4L2Helper::reuseCatureBuffer, this, std::placeholders::_1, buffer->getIndex(), mSelf));
-    mSendFrame(frame);
+    frame->setMetadata(h264Metadata);
+    frame_container frames;
+    frames.insert(make_pair(h264FrameOutputPinId, frame));
+
+    if (enableMotionVectors)
+    {
+        v4l2_ctrl_videoenc_outputbuf_metadata_MV enc_mv_metadata;
+        getMotionVectors(buffer->v4l2_buf.index, enc_mv_metadata);
+        serializeMotionVectors(enc_mv_metadata, frames);
+    }
+
+    mSendFrameContainer(frames);
     mConverter->releaseFrame();
 }