diff --git a/base/include/H264Decoder.h b/base/include/H264Decoder.h index c35cebae5..f44aeba1c 100644 --- a/base/include/H264Decoder.h +++ b/base/include/H264Decoder.h @@ -1,11 +1,18 @@ #pragma once #include "Module.h" +#include class H264DecoderProps : public ModuleProps { public: - H264DecoderProps() {} + H264DecoderProps(uint _lowerWaterMark = 300, uint _upperWaterMark = 350) + { + lowerWaterMark = _lowerWaterMark; + upperWaterMark = _upperWaterMark; + } + uint lowerWaterMark; + uint upperWaterMark; }; class H264Decoder : public Module @@ -26,10 +33,42 @@ class H264Decoder : public Module bool shouldTriggerSOS(); private: + void bufferDecodedFrames(frame_sp& frame); + void bufferBackwardEncodedFrames(frame_sp& frame, short naluType); + void bufferAndDecodeForwardEncodedFrames(frame_sp& frame, short naluType); + class Detail; boost::shared_ptr mDetail; bool mShouldTriggerSOS; framemetadata_sp mOutputMetadata; std::string mOutputPinId; H264DecoderProps mProps; -}; + + /* Used to buffer multiple complete GOPs + note that we decode frames from this queue in reverse play*/ + std::deque> backwardGopBuffer; + /* buffers the incomplete GOP */ + std::deque latestBackwardGop; + /* It buffers only one latest GOP + used in cases where partial GOP maybe in cache and rest of the GOP needs to be decoded + note that since there is no buffering in forward play, we directly decode frames from module queue*/ + std::deque latestForwardGop; + std::map decodedFramesCache; + void sendDecodedFrame(); + bool mDirection; + bool dirChangedToFwd = false; + bool dirChangedToBwd = false; + bool foundIFrameOfReverseGop = false; + bool flushDecoderFlag = false; + bool decodePreviousFramesOfTheForwardGop = false; + bool prevFrameInCache = false; + void decodeFrameFromBwdGOP(); + std::deque incomingFramesTSQ; + void clearIncompleteBwdGopTsFromIncomingTSQ(std::deque& latestGop); + void saveSpsPps(frame_sp frame); + void* prependSpsPps(frame_sp& iFrame, size_t& spsPpsFrameSize); + void dropFarthestFromCurrentTs(uint64_t ts); + frame_sp mHeaderFrame; + boost::asio::const_buffer spsBuffer; + boost::asio::const_buffer ppsBuffer; +}; \ No newline at end of file diff --git a/base/include/H264Metadata.h b/base/include/H264Metadata.h index d59d6d2b6..d315f1858 100644 --- a/base/include/H264Metadata.h +++ b/base/include/H264Metadata.h @@ -40,9 +40,12 @@ class H264Metadata : public FrameMetadata width = metadata.width; height = metadata.height; + direction = metadata.direction; + mp4Seek = metadata.mp4Seek; //setDataSize(); } - + bool direction = true; + bool mp4Seek = false; protected: void initData(int _width, int _height, MemType _memType = MemType::HOST) { diff --git a/base/src/H264Decoder.cpp b/base/src/H264Decoder.cpp index 6c1dbe5b8..6203be960 100644 --- a/base/src/H264Decoder.cpp +++ b/base/src/H264Decoder.cpp @@ -12,6 +12,7 @@ #include "Frame.h" #include "Logger.h" #include "Utils.h" +#include "H264Utils.h" class H264Decoder::Detail { @@ -27,35 +28,42 @@ class H264Decoder::Detail bool setMetadata(framemetadata_sp& metadata, frame_sp frame, std::function send, std::function makeFrame) { - if (metadata->getFrameType() == FrameMetadata::FrameType::H264_DATA) + auto type = H264Utils::getNALUType((char*)frame->data()); + if (type == H264Utils::H264_NAL_TYPE_IDR_SLICE || type == H264Utils::H264_NAL_TYPE_SEQ_PARAM) { - sps_pps_properties p; - H264ParserUtils::parse_sps(((const char*)frame->data()) + 5, frame->size() > 5 ? frame->size() - 5 : frame->size(), &p); - mWidth = p.width; - mHeight = p.height; + if (metadata->getFrameType() == FrameMetadata::FrameType::H264_DATA) + { + sps_pps_properties p; + H264ParserUtils::parse_sps(((const char*)frame->data()) + 5, frame->size() > 5 ? frame->size() - 5 : frame->size(), &p); + mWidth = p.width; + mHeight = p.height; - auto h264Metadata = framemetadata_sp(new H264Metadata(mWidth, mHeight)); - auto rawOutMetadata = FrameMetadataFactory::downcast(h264Metadata); - rawOutMetadata->setData(*rawOutMetadata); - } + auto h264Metadata = framemetadata_sp(new H264Metadata(mWidth, mHeight)); + auto rawOutMetadata = FrameMetadataFactory::downcast(h264Metadata); + rawOutMetadata->setData(*rawOutMetadata); +#ifdef ARM64 + helper.reset(new h264DecoderV4L2Helper()); + return helper->init(send, makeFrame); +#else + helper.reset(new H264DecoderNvCodecHelper(mWidth, mHeight)); + return helper->init(send, makeFrame); +#endif + } + else + { + throw AIPException(AIP_NOTIMPLEMENTED, "Unknown frame type"); + } + } else { - throw AIPException(AIP_NOTIMPLEMENTED, "Unknown frame type"); + return false; } - -#ifdef ARM64 - helper.reset(new h264DecoderV4L2Helper()); - return helper->init(send, makeFrame); -#else - helper.reset(new H264DecoderNvCodecHelper(mWidth, mHeight)); - return helper->init(send, makeFrame);// -#endif } - void compute(frame_sp& frame) + void compute(void* inputFrameBuffer, size_t inputFrameSize, uint64_t inputFrameTS) { - helper->process(frame); + helper->process(inputFrameBuffer, inputFrameSize, inputFrameTS); } #ifdef ARM64 @@ -139,7 +147,6 @@ bool H264Decoder::init() { return false; } - return true; } @@ -149,38 +156,433 @@ bool H264Decoder::term() auto eosFrame = frame_sp(new EoSFrame()); mDetail->closeAllThreads(eosFrame); #endif - mDetail.reset(); return Module::term(); } +void* H264Decoder::prependSpsPps(frame_sp& iFrame, size_t& spsPpsFrameSize) +{ + spsPpsFrameSize = iFrame->size() + spsBuffer.size() + ppsBuffer.size() + 8; + uint8_t* spsPpsFrameBuffer = new uint8_t[spsPpsFrameSize]; + char NaluSeprator[4] = { 00 ,00, 00 ,01 }; + auto nalu = reinterpret_cast(NaluSeprator); + memcpy(spsPpsFrameBuffer, nalu, 4); + spsPpsFrameBuffer += 4; + memcpy(spsPpsFrameBuffer, spsBuffer.data(), spsBuffer.size()); + spsPpsFrameBuffer += spsBuffer.size(); + memcpy(spsPpsFrameBuffer, nalu, 4); + spsPpsFrameBuffer += 4; + memcpy(spsPpsFrameBuffer, ppsBuffer.data(), ppsBuffer.size()); + spsPpsFrameBuffer += ppsBuffer.size(); + memcpy(spsPpsFrameBuffer, iFrame->data(), iFrame->size()); + spsPpsFrameBuffer = spsPpsFrameBuffer - spsBuffer.size() - ppsBuffer.size() - 8; + return spsPpsFrameBuffer; +} + +void H264Decoder::clearIncompleteBwdGopTsFromIncomingTSQ(std::deque& latestGop) +{ + while (!latestGop.empty()) + { + auto deleteItr = std::find(incomingFramesTSQ.begin(), incomingFramesTSQ.end(), latestGop.front()->timestamp); + if (deleteItr != incomingFramesTSQ.end()) + { + incomingFramesTSQ.erase(deleteItr); + latestGop.pop_front(); + } + } +} + +void H264Decoder::bufferBackwardEncodedFrames(frame_sp& frame, short naluType) +{ + if (dirChangedToBwd) + { + latestBackwardGop.clear(); + dirChangedToBwd = false; + } + // insert frames into the latest gop until I frame comes. + latestBackwardGop.emplace_back(frame); + // The latest GOP is complete when I Frame comes up, move the GOP to backwardGopBuffer where all the backward GOP's are buffered + if (naluType == H264Utils::H264_NAL_TYPE_IDR_SLICE || naluType == H264Utils::H264_NAL_TYPE_SEQ_PARAM) + { + foundIFrameOfReverseGop = true; + backwardGopBuffer.push_back(std::move(latestBackwardGop)); + } +} + +void H264Decoder::bufferAndDecodeForwardEncodedFrames(frame_sp& frame, short naluType) +{ + if (dirChangedToFwd) + { + // Whenever the direction changes to forward we just send all the backward buffered GOP's to decoded in a single step . The motive is to send the current forward frame to decoder in the same step. + while (!backwardGopBuffer.empty()) + { + decodeFrameFromBwdGOP(); + } + + // Whenever direction changes to forward , And the latestBackwardGop is incomplete , then delete the latest backward GOP and remove the frames from incomingFramesTSQ entry as well + if (!latestBackwardGop.empty()) + { + clearIncompleteBwdGopTsFromIncomingTSQ(latestBackwardGop); + } + dirChangedToFwd = false; + } + if(prevFrameInCache) + { + // previous Frame was In Cache & current is not + if (!latestForwardGop.empty()) + { + short naluTypeOfForwardGopFirstFrame = H264Utils::getNALUType((char*)latestForwardGop.front()->data()); + if (naluTypeOfForwardGopFirstFrame == H264Utils::H264_NAL_TYPE_IDR_SLICE || naluTypeOfForwardGopFirstFrame == H264Utils::H264_NAL_TYPE_SEQ_PARAM) + { + // Corner case: Forward :- current frame is not part of latestForwardGOP + if (latestForwardGop.front()->timestamp > frame->timestamp) + { + latestForwardGop.clear(); + } + } + + // Corner case: Forward:- When end of cache hits while in the middle of gop, before decoding the next P frame we need decode the previous frames of that GOP. + // There might be a case where we might have cleared the decoder, in order to start the decoder again we must prepend sps and pps to I frame if not present + if (!latestForwardGop.empty() && naluTypeOfForwardGopFirstFrame == H264Utils::H264_NAL_TYPE_IDR_SLICE) + { + auto iFrame = latestForwardGop.front(); + size_t spsPpsFrameSize; + auto spsPpsFrameBuffer = prependSpsPps(iFrame, spsPpsFrameSize); + mDetail->compute(spsPpsFrameBuffer, spsPpsFrameSize, iFrame->timestamp); + latestForwardGop.pop_front(); + for (auto itr = latestForwardGop.begin(); itr != latestForwardGop.end(); itr++) + { + if (itr->get()->timestamp < frame->timestamp) + { + mDetail->compute(itr->get()->data(), itr->get()->size(), itr->get()->timestamp); + } + } + } + else if (!latestForwardGop.empty() && naluTypeOfForwardGopFirstFrame == H264Utils::H264_NAL_TYPE_SEQ_PARAM) + { + for (auto itr = latestForwardGop.begin(); itr != latestForwardGop.end(); itr++) + { + if (itr->get()->timestamp < frame->timestamp) + { + mDetail->compute(itr->get()->data(), itr->get()->size(), itr->get()->timestamp); + } + } + } + } + } + prevFrameInCache = false; + + /* buffer fwd GOP and send the current frame */ + // new GOP starts + if (naluType == H264Utils::H264_NAL_TYPE_IDR_SLICE || naluType == H264Utils::H264_NAL_TYPE_SEQ_PARAM) + { + latestForwardGop.clear(); + } + latestForwardGop.emplace_back(frame); + + // If direction changed to forward in the middle of GOP (Even the latest gop of backward was half and not decoded) , Then we drop the P frames until next I frame. + // We also remove the entries of P frames from the incomingFramesTSQ. + short latestForwardGopFirstFrameNaluType = H264Utils::getNALUType((char*)latestForwardGop.begin()->get()->data()); + if (latestForwardGopFirstFrameNaluType != H264Utils::H264_NAL_TYPE_IDR_SLICE && latestForwardGopFirstFrameNaluType != H264Utils::H264_NAL_TYPE_SEQ_PARAM) + { + clearIncompleteBwdGopTsFromIncomingTSQ(latestForwardGop); + return; + } + + mDetail->compute(frame->data(), frame->size(), frame->timestamp); + return; +} + +void H264Decoder::decodeFrameFromBwdGOP() +{ + if (!backwardGopBuffer.empty() && H264Utils::getNALUType((char*)backwardGopBuffer.front().back()->data()) == H264Utils::H264_NAL_TYPE_IDR_SLICE && prevFrameInCache) + { + auto iFrame = backwardGopBuffer.front().back(); + size_t spsPpsFrameSize; + auto spsPpsFrameBuffer = prependSpsPps(iFrame, spsPpsFrameSize); + mDetail->compute(spsPpsFrameBuffer, spsPpsFrameSize, iFrame->timestamp); + backwardGopBuffer.front().pop_back(); + prevFrameInCache = false; + } + if (!backwardGopBuffer.empty() && !backwardGopBuffer.front().empty()) + { + // For reverse play we sent the frames to the decoder in reverse, As the last frame added in the deque should be sent first (Example : P,P,P,P,P,P,I) + auto itr = backwardGopBuffer.front().rbegin(); + mDetail->compute(itr->get()->data(), itr->get()->size(), itr->get()->timestamp); + backwardGopBuffer.front().pop_back(); + } + if (backwardGopBuffer.size() >= 1 && backwardGopBuffer.front().empty()) + { + backwardGopBuffer.pop_front(); + } + if (backwardGopBuffer.empty()) + { + foundIFrameOfReverseGop = false; + } +} + +void H264Decoder::saveSpsPps(frame_sp frame) +{ + auto mFrameBuffer = const_buffer(frame->data(), frame->size()); + auto ret = H264Utils::parseNalu(mFrameBuffer); + const_buffer tempSpsBuffer; + const_buffer tempPpsBuffer; + short typeFound; + tie(typeFound, tempSpsBuffer, tempPpsBuffer) = ret; + + if ((tempSpsBuffer.size() != 0) || (tempPpsBuffer.size() != 0)) + { + mHeaderFrame = frame; + spsBuffer = tempSpsBuffer; + ppsBuffer = tempPpsBuffer; + } +} + bool H264Decoder::process(frame_container& frames) { - auto frame = frames.cbegin()->second; - mDetail->compute(frame); + auto frame = frames.begin()->second; + auto frameMetadata = frame->getMetadata(); + auto h264Metadata = FrameMetadataFactory::downcast(frameMetadata); + + if (mDirection && !h264Metadata->direction) + { + dirChangedToBwd = true; + } + else if (!mDirection && h264Metadata->direction) + { + dirChangedToFwd = true; //rename to directionChangedToFwd + } + else + { + dirChangedToBwd = false; + dirChangedToFwd = false; + } + + /* Clear the latest forward gop whenever seek happens bcz there is no buffering for fwd play. + We dont clear backwardGOP because there might be a left over GOP to be decoded. */ + if (h264Metadata->mp4Seek) + { + latestForwardGop.clear(); + } + + mDirection = h264Metadata->direction; + short naluType = H264Utils::getNALUType((char*)frame->data()); + if (naluType == H264Utils::H264_NAL_TYPE_SEQ_PARAM) + { + saveSpsPps(frame); + } + // we get a repeated frame whenever direction changes i.e. the timestamp Q latest frame is repeated + if (!incomingFramesTSQ.empty() && incomingFramesTSQ.back() == frame->timestamp) + { + flushDecoderFlag = true; + } + + //Insert the frames time stamp in TS queue. We send the frames to next modules in the same order. + incomingFramesTSQ.push_back(frame->timestamp); + + //If the frame is already present in the decoded output cache then skip the frame decoding. + if (decodedFramesCache.find(frame->timestamp) != decodedFramesCache.end()) + { + //prepend sps and pps if 1st frame is I frame + if (!backwardGopBuffer.empty() && H264Utils::getNALUType((char*)backwardGopBuffer.front().back()->data()) == H264Utils::H264_NAL_TYPE_IDR_SLICE) + { + auto iFrame = backwardGopBuffer.front().back(); + size_t spsPpsFrameSize; + auto spsPpsFrameBuffer = prependSpsPps(iFrame, spsPpsFrameSize); + mDetail->compute(spsPpsFrameBuffer, spsPpsFrameSize, iFrame->timestamp); + backwardGopBuffer.front().pop_back(); + } + // the buffered GOPs in bwdGOPBuffer needs to need to be processed first + while (!backwardGopBuffer.empty()) + { + decodeFrameFromBwdGOP(); + } + + // if we seeked + if (h264Metadata->mp4Seek) + { + // flush the incomplete GOP + flushDecoderFlag = true; + clearIncompleteBwdGopTsFromIncomingTSQ(latestBackwardGop); + } + + // corner case: partial GOP already present in cache + if (!mDirection && latestBackwardGop.empty() && backwardGopBuffer.empty()) + { + auto eosFrame = frame_sp(new EmptyFrame()); + mDetail->compute(eosFrame->data(), eosFrame->size(), eosFrame->timestamp); + flushDecoderFlag = false; + } + + if (!latestBackwardGop.empty()) + { + // Corner case: backward :- (I,P,P,P) Here if first two frames are in the cache and last two frames are not in the cache , to decode the last two frames we buffer the full gop and later decode it. + bufferBackwardEncodedFrames(frame, naluType); + sendDecodedFrame(); + return true; + } + + if (mDirection && ((naluType == H264Utils::H264_NAL_TYPE_SEQ_PARAM) || (naluType == H264Utils::H264_NAL_TYPE_IDR_SLICE))) + { + latestForwardGop.clear(); + latestForwardGop.push_back(frame); + } + // dont buffer fwd GOP if I frame has not been recieved (possible in intra GOP direction change cases) + else if (mDirection && !latestForwardGop.empty() && (H264Utils::getNALUType((char*)latestForwardGop.front()->data()) == H264Utils::H264_NAL_TYPE_SEQ_PARAM || H264Utils::getNALUType((char*)latestForwardGop.front()->data()) == H264Utils::H264_NAL_TYPE_IDR_SLICE)) + { + flushDecoderFlag = false; + latestForwardGop.push_back(frame); + } + + // While in forward play, if cache has resumed in the middle of the GOP then to get the previous few frames we need to flush the decoder. + if (mDirection && !prevFrameInCache) + { + auto eosFrame = frame_sp(new EmptyFrame()); + mDetail->compute(eosFrame->data(), eosFrame->size(), eosFrame->timestamp); + flushDecoderFlag = false; + } + prevFrameInCache = true; + sendDecodedFrame(); + return true; + } + /* If frame is not in output cache, it needs to be buffered & decoded */ + if (mDirection) + { + //Buffers the latest GOP and send the current frame to decoder. + bufferAndDecodeForwardEncodedFrames(frame, naluType); + } + else + { + //Only buffering of backward GOP happens + bufferBackwardEncodedFrames(frame, naluType); + } + if (foundIFrameOfReverseGop) + { + // The I frame of backward GOP was found , now we send the frames to the decoder one by one in every step + decodeFrameFromBwdGOP(); + } + sendDecodedFrame(); + dropFarthestFromCurrentTs(frame->timestamp); return true; } +void H264Decoder::sendDecodedFrame() +{ + // not in output cache && flushdecoder flag is set + if (!incomingFramesTSQ.empty() && !decodedFramesCache.empty() && decodedFramesCache.find(incomingFramesTSQ.front()) == decodedFramesCache.end() && flushDecoderFlag && backwardGopBuffer.empty()) + { + // We send empty frame to the decoder , in order to flush out all the frames from decoder. + // This is to handle some cases whenever the direction change happens and to get out the latest few frames sent to decoder. + auto eosFrame = frame_sp(new EmptyFrame()); + mDetail->compute(eosFrame->data(), eosFrame->size(), eosFrame->timestamp); + flushDecoderFlag = false; + } + + // timestamp in output cache + if (!incomingFramesTSQ.empty() && !decodedFramesCache.empty() && decodedFramesCache.find(incomingFramesTSQ.front()) != decodedFramesCache.end()) + { + auto outFrame = decodedFramesCache[incomingFramesTSQ.front()]; + incomingFramesTSQ.pop_front(); + frame_container frames; + frames.insert(make_pair(mOutputPinId, outFrame)); + send(frames); + } +} + +void H264Decoder::bufferDecodedFrames(frame_sp& frame) +{ + decodedFramesCache.insert({ frame->timestamp, frame }); +} + +void H264Decoder::dropFarthestFromCurrentTs(uint64_t ts) +{ + if (decodedFramesCache.empty()) + { + return; + } + + /* dropping algo */ + int64_t begDistTS = ts - decodedFramesCache.begin()->first; + auto absBeginDistance = abs(begDistTS); + int64_t endDistTS = ts - decodedFramesCache.rbegin()->first; + auto absEndDistance = abs(endDistTS); + if (decodedFramesCache.size() >= mProps.upperWaterMark) + { + if (absEndDistance <= absBeginDistance) + { + auto itr = decodedFramesCache.begin(); + while (itr != decodedFramesCache.end()) + { + if (decodedFramesCache.size() >= mProps.lowerWaterMark) + { + boost::mutex::scoped_lock(m_mutex); + // Note - erase returns the iterator of next element after deletion. + // Dont drop the frames from cache which are present in the incomingFramesTSQ + if (std::find(incomingFramesTSQ.begin(), incomingFramesTSQ.end(), itr->first) != incomingFramesTSQ.end()) + { + itr++; + continue; + } + itr = decodedFramesCache.erase(itr); + } + else + { + return; + } + } + } + else + { + // delete from end using the fwd iterator. + auto itr = decodedFramesCache.end(); + --itr; + while (itr != decodedFramesCache.begin()) + { + if (decodedFramesCache.size() >= mProps.lowerWaterMark) + { + boost::mutex::scoped_lock(m_mutex); + // Note - erase returns the iterator of next element after deletion. + if (std::find(incomingFramesTSQ.begin(), incomingFramesTSQ.end(), itr->first) != incomingFramesTSQ.end()) + { + --itr; + continue; + } + itr = decodedFramesCache.erase(itr); + --itr; + } + else + { + return; + } + } + } + } +} + bool H264Decoder::processSOS(frame_sp& frame) { auto metadata = frame->getMetadata(); - mDetail->setMetadata(metadata, frame, + auto h264Metadata = FrameMetadataFactory::downcast(metadata); + mDirection = h264Metadata->direction; + auto ret = mDetail->setMetadata(metadata, frame, [&](frame_sp& outputFrame) { - frame_container frames; - frames.insert(make_pair(mOutputPinId, outputFrame)); - send(frames); + bufferDecodedFrames(outputFrame); }, [&]() -> frame_sp {return makeFrame(); } ); - mShouldTriggerSOS = false; - auto rawOutMetadata = FrameMetadataFactory::downcast(mOutputMetadata); + if (ret) + { + mShouldTriggerSOS = false; + auto rawOutMetadata = FrameMetadataFactory::downcast(mOutputMetadata); #ifdef ARM64 - RawImagePlanarMetadata OutputMetadata(mDetail->mWidth, mDetail->mHeight, ImageMetadata::ImageType::NV12, 128, CV_8U, FrameMetadata::MemType::DMABUF); + RawImagePlanarMetadata OutputMetadata(mDetail->mWidth, mDetail->mHeight, ImageMetadata::ImageType::NV12, 128, CV_8U, FrameMetadata::MemType::DMABUF); #else - RawImagePlanarMetadata OutputMetadata(mDetail->mWidth, mDetail->mHeight, ImageMetadata::YUV420, size_t(0), CV_8U, FrameMetadata::HOST); + RawImagePlanarMetadata OutputMetadata(mDetail->mWidth, mDetail->mHeight, ImageMetadata::YUV420, size_t(0), CV_8U, FrameMetadata::HOST); #endif - rawOutMetadata->setData(OutputMetadata); + rawOutMetadata->setData(OutputMetadata); + } + return true; } @@ -192,7 +594,7 @@ bool H264Decoder::shouldTriggerSOS() bool H264Decoder::processEOS(string& pinId) { auto frame = frame_sp(new EmptyFrame()); - mDetail->compute(frame); + mDetail->compute(frame->data(), frame->size(), frame->timestamp); mShouldTriggerSOS = true; return true; } \ No newline at end of file diff --git a/base/src/H264DecoderNvCodecHelper.cpp b/base/src/H264DecoderNvCodecHelper.cpp index f90af9231..6dd0e7d17 100644 --- a/base/src/H264DecoderNvCodecHelper.cpp +++ b/base/src/H264DecoderNvCodecHelper.cpp @@ -711,7 +711,7 @@ bool H264DecoderNvCodecHelper::init(std::function _send, std::f { makeFrame = _makeFrame; send = _send; - return false; + return true; } void H264DecoderNvCodecHelper::ConvertToPlanar(uint8_t* pHostFrame, int nWidth, int nHeight, int nBitDepth) { @@ -731,15 +731,17 @@ void H264DecoderNvCodecHelper::ConvertToPlanar(uint8_t* pHostFrame, int nWidth, } } -void H264DecoderNvCodecHelper::process(frame_sp& frame) +void H264DecoderNvCodecHelper::process(void* inputFrameBuffer, size_t inputFrameSize, uint64_t inputFrameTS) { + if(inputFrameSize) + framesTimestampEntry.push(inputFrameTS); uint8_t* inputBuffer = NULL; int inputBufferSize = 0; - frame_sp outputFrame = makeFrame(); - uint8_t** outBuffer = reinterpret_cast(outputFrame->data()); + frame_sp outputFrame; + uint8_t** outBuffer; - inputBuffer = static_cast(frame->data()); - inputBufferSize = frame->size(); + inputBuffer = static_cast(inputFrameBuffer); + inputBufferSize = inputFrameSize; int nFrameReturned = 0, nFrame = 0; bool bOutPlanar = true; @@ -749,7 +751,9 @@ void H264DecoderNvCodecHelper::process(frame_sp& frame) for (int i = 0; i < nFrameReturned; i++) { ConvertToPlanar(outBuffer[i], helper->GetWidth(), helper->GetHeight(), helper->GetBitDepth()); - + outputFrame = makeFrame(); + outputFrame->timestamp = framesTimestampEntry.front(); + framesTimestampEntry.pop(); memcpy(outputFrame->data(), outBuffer[i], outputFrame->size()); send(outputFrame); } diff --git a/base/src/H264DecoderNvCodecHelper.h b/base/src/H264DecoderNvCodecHelper.h index f7e117d5d..4ed30803e 100644 --- a/base/src/H264DecoderNvCodecHelper.h +++ b/base/src/H264DecoderNvCodecHelper.h @@ -12,6 +12,7 @@ #include #include "CommonDefs.h" #include "CudaCommon.h" +#include /** * @brief Exception class for error reporting from the decode API. @@ -237,9 +238,10 @@ class H264DecoderNvCodecHelper : public NvDecoder bool init(std::function send, std::function makeFrame); void ConvertToPlanar(uint8_t* pHostFrame, int nWidth, int nHeight, int nBitDepth); - void process(frame_sp& frame); + void process(void* inputFrameBuffer, size_t inputFrameSize, uint64_t inputFrameTS); std::function send; std::function makeFrame; private: boost::shared_ptr helper; + std::queue framesTimestampEntry; }; \ No newline at end of file diff --git a/base/src/H264DecoderV4L2Helper.cpp b/base/src/H264DecoderV4L2Helper.cpp index af9c65335..d976464ac 100644 --- a/base/src/H264DecoderV4L2Helper.cpp +++ b/base/src/H264DecoderV4L2Helper.cpp @@ -282,10 +282,10 @@ Buffer::fill_buffer_plane_format(uint32_t *num_planes, return 0; } -void h264DecoderV4L2Helper::read_input_chunk_frame_sp(frame_sp inpFrame, Buffer * buffer) +void h264DecoderV4L2Helper::read_input_chunk_frame_sp(void* inputFrameBuffer, size_t inputFrameSize, Buffer * buffer) { - memcpy(buffer->planes[0].data,inpFrame->data(),inpFrame->size()); - buffer->planes[0].bytesused = static_cast(inpFrame->size()); + memcpy(buffer->planes[0].data,inputFrameBuffer,inputFrameSize); + buffer->planes[0].bytesused = static_cast(inputFrameSize); } /** @@ -315,6 +315,8 @@ void h264DecoderV4L2Helper::read_input_chunk_frame_sp(frame_sp inpFrame, Buffer { return -1; } + outputFrame->timestamp = framesTimestampEntry.front(); + framesTimestampEntry.pop(); send(outputFrame); @@ -370,7 +372,7 @@ void h264DecoderV4L2Helper::read_input_chunk_frame_sp(frame_sp inpFrame, Buffer return ret_val; } - void h264DecoderV4L2Helper::query_set_capture(context_t * ctx ,int &f_d) + void h264DecoderV4L2Helper::query_set_capture(context_t * ctx) { struct v4l2_format format; struct v4l2_crop crop; @@ -637,13 +639,9 @@ void * h264DecoderV4L2Helper::capture_thread(void *arg) ** Format and buffers are now set on capture. */ - auto outputFrame = m_nThread->makeFrame(); - auto dmaOutFrame = static_cast(outputFrame->data()); - int f_d = dmaOutFrame->getFd(); - if (!ctx->in_error) { - m_nThread->query_set_capture(ctx, f_d); + m_nThread->query_set_capture(ctx); } /* Check for resolution event to again @@ -658,7 +656,7 @@ void * h264DecoderV4L2Helper::capture_thread(void *arg) switch (event.type) { case V4L2_EVENT_RESOLUTION_CHANGE: - m_nThread->query_set_capture(ctx, f_d); + m_nThread->query_set_capture(ctx); continue; } } @@ -728,7 +726,11 @@ void * h264DecoderV4L2Helper::capture_thread(void *arg) /* Blocklinear to Pitch transformation is required ** to dump the raw decoded buffer data. */ - + + auto outputFrame = m_nThread->makeFrame(); + + auto dmaOutFrame = static_cast(outputFrame->data()); + int f_d = dmaOutFrame->getFd(); ret_val = NvBufferTransform(decoded_buffer->planes[0].fd,f_d, &transform_params); if (ret_val == -1) { @@ -782,7 +784,7 @@ void * h264DecoderV4L2Helper::capture_thread(void *arg) return NULL; } - bool h264DecoderV4L2Helper::decode_process(context_t& ctx, frame_sp frame) + bool h264DecoderV4L2Helper::decode_process(context_t& ctx, void* inputFrameBuffer, size_t inputFrameSize) { bool allow_DQ = true; int ret_val; @@ -822,7 +824,7 @@ void * h264DecoderV4L2Helper::capture_thread(void *arg) if (ctx.decode_pixfmt == V4L2_PIX_FMT_H264) { - read_input_chunk_frame_sp(frame, buffer); + read_input_chunk_frame_sp(inputFrameBuffer, inputFrameSize, buffer); } else { @@ -1131,6 +1133,10 @@ bool h264DecoderV4L2Helper::init(std::function _send, std::func makeFrame = _makeFrame; mBuffer.reset(new Buffer()); send = _send; + return initializeDecoder(); +} +bool h264DecoderV4L2Helper::initializeDecoder() +{ int flags = 0; struct v4l2_capability caps; struct v4l2_buffer op_v4l2_buf; @@ -1302,10 +1308,22 @@ bool h264DecoderV4L2Helper::init(std::function _send, std::func typedef void * (*THREADFUNCPTR)(void *); pthread_create(&ctx.dec_capture_thread, NULL,h264DecoderV4L2Helper::capture_thread, (void *) (this)); + + return true; } -int h264DecoderV4L2Helper::process(frame_sp inputFrame) +int h264DecoderV4L2Helper::process(void* inputFrameBuffer, size_t inputFrameSize, uint64_t inputFrameTS) { uint32_t idx = 0; + if(inputFrameSize) + framesTimestampEntry.push(inputFrameTS); + + if(inputFrameSize && ctx.eos && ctx.got_eos) + { + ctx.eos = false; + ctx.got_eos = false; + initializeDecoder(); + } + while (!ctx.eos && !ctx.in_error && idx < ctx.op_num_buffers) { struct v4l2_buffer queue_v4l2_buf_op; @@ -1318,7 +1336,7 @@ int h264DecoderV4L2Helper::process(frame_sp inputFrame) buffer = ctx.op_buffers[idx]; if (ctx.decode_pixfmt == V4L2_PIX_FMT_H264) { - read_input_chunk_frame_sp(inputFrame, buffer); + read_input_chunk_frame_sp(inputFrameBuffer, inputFrameSize, buffer); } else { @@ -1351,7 +1369,7 @@ int h264DecoderV4L2Helper::process(frame_sp inputFrame) } // Dequeue and queue loop on output plane. - ctx.eos = decode_process(ctx,inputFrame); + ctx.eos = decode_process(ctx,inputFrameBuffer, inputFrameSize); /* For blocking mode, after getting EOS on output plane, ** dequeue all the queued buffers on output plane. @@ -1387,7 +1405,7 @@ int h264DecoderV4L2Helper::process(frame_sp inputFrame) } void h264DecoderV4L2Helper::closeAllThreads(frame_sp eosFrame) { - process(eosFrame); + process(eosFrame->data(), eosFrame->size(), 0); if (ctx.fd != -1) { if (ctx.dec_capture_thread) diff --git a/base/src/H264DecoderV4L2Helper.h b/base/src/H264DecoderV4L2Helper.h index eb8e9193e..3a556b06b 100644 --- a/base/src/H264DecoderV4L2Helper.h +++ b/base/src/H264DecoderV4L2Helper.h @@ -40,6 +40,7 @@ #include "Frame.h" #include #include +#include /** * @brief Class representing a buffer. @@ -192,7 +193,7 @@ class h264DecoderV4L2Helper * @param[in] stream Input stream * @param[in] buffer Buffer class pointer */ - void read_input_chunk_frame_sp(frame_sp inpFrame, Buffer *buffer); + void read_input_chunk_frame_sp(void* inputFrameBuffer, size_t inputFrameSize, Buffer *buffer); /** * @brief Writes a plane data of the buffer to a file. @@ -228,7 +229,7 @@ class h264DecoderV4L2Helper * * @param[in] ctx Pointer to the decoder context struct created. */ - void query_set_capture(context_t *ctx, int &fd); + void query_set_capture(context_t *ctx); /** * @brief Callback function on capture thread. @@ -257,7 +258,7 @@ class h264DecoderV4L2Helper * EOS is detected by the decoder and all the buffers are dequeued; * else the decode process continues running. */ - bool decode_process(context_t &ctx, frame_sp frame); + bool decode_process(context_t &ctx, void* inputFrameBuffer, size_t inputFrameSize); /** * @brief Dequeues an event. @@ -381,10 +382,12 @@ class h264DecoderV4L2Helper */ int subscribe_event(int fd, uint32_t type, uint32_t id, uint32_t flags); - int process(frame_sp inputFrame); + int process(void* inputFrameBuffer, size_t inputFrameSize, uint64_t inputFrameTS); bool init(std::function send, std::function makeFrame); + bool initializeDecoder(); + void closeAllThreads(frame_sp eosFrame); protected: boost::shared_ptr mBuffer; @@ -392,4 +395,5 @@ class h264DecoderV4L2Helper std::function makeFrame; std::function send; int ret = 0; + std::queue framesTimestampEntry; }; diff --git a/base/src/Mp4ReaderSource.cpp b/base/src/Mp4ReaderSource.cpp index b267b7fb3..3ab236841 100644 --- a/base/src/Mp4ReaderSource.cpp +++ b/base/src/Mp4ReaderSource.cpp @@ -256,6 +256,8 @@ class Mp4ReaderDetailAbs } LOG_TRACE << "changed direction frameIdx <" << mState.mFrameCounterIdx << "> totalFrames <" << mState.mFramesInVideo << ">"; mp4_demux_toggle_playback(mState.demux, mState.video.id); + mDirection = _direction; + setMetadata(); } } @@ -491,6 +493,7 @@ class Mp4ReaderDetailAbs mState.mFramesInVideo = mState.info.sample_count; mWidth = mState.info.video_width; mHeight = mState.info.video_height; + mDirection = mState.direction; mDurationInSecs = mState.info.duration / mState.info.timescale; mFPS = mState.mFramesInVideo / mDurationInSecs; } @@ -612,6 +615,8 @@ class Mp4ReaderDetailAbs // reset flags waitFlag = false; sentEOSSignal = false; + isMp4SeekFrame = true; + setMetadata(); return true; } @@ -673,6 +678,8 @@ class Mp4ReaderDetailAbs waitFlag = false; // prependSpsPps mState.shouldPrependSpsPps = true; + isMp4SeekFrame = true; + setMetadata(); return true; } @@ -976,11 +983,11 @@ class Mp4ReaderDetailAbs std::string mVideoPath = ""; int32_t mFrameCounterIdx; bool shouldPrependSpsPps = false; + bool foundFirstReverseIFrame = false; bool end = false; Mp4ReaderSourceProps props; float speed; bool direction; - //bool end; } mState; uint64_t openVideoStartingTS = 0; uint64_t reloadFileAfter = 0; @@ -993,6 +1000,7 @@ class Mp4ReaderDetailAbs uint64_t recheckDiskTS = 0; boost::shared_ptr cof; framemetadata_sp updatedEncodedImgMetadata; + framemetadata_sp mH264Metadata; /* mState.end = true is possible only in two cases: - if parseFS found no more relevant files on the disk @@ -1001,6 +1009,8 @@ class Mp4ReaderDetailAbs public: int mWidth = 0; int mHeight = 0; + bool mDirection; + bool isMp4SeekFrame = false; int ret; double mFPS = 0; double mDurationInSecs = 0; @@ -1057,12 +1067,6 @@ void Mp4ReaderDetailJpeg::setMetadata() } auto encodedMetadata = FrameMetadataFactory::downcast(metadata); encodedMetadata->setData(*encodedMetadata); - - auto mp4FrameMetadata = framemetadata_sp(new Mp4VideoMetadata("v_1_0")); - // set proto version in mp4videometadata - auto serFormatVersion = getSerFormatVersion(); - auto mp4VideoMetadata = FrameMetadataFactory::downcast(mp4FrameMetadata); - mp4VideoMetadata->setData(serFormatVersion); Mp4ReaderDetailAbs::setMetadata(); // set at Module level mSetMetadata(encodedImagePinId, metadata); @@ -1141,17 +1145,21 @@ bool Mp4ReaderDetailJpeg::produceFrames(frame_container& frames) void Mp4ReaderDetailH264::setMetadata() { - auto metadata = framemetadata_sp(new H264Metadata(mWidth, mHeight)); - if (!metadata->isSet()) + mH264Metadata = framemetadata_sp(new H264Metadata(mWidth, mHeight)); + + if (!mH264Metadata->isSet()) { return; } - auto h264Metadata = FrameMetadataFactory::downcast(metadata); + auto h264Metadata = FrameMetadataFactory::downcast(mH264Metadata); + h264Metadata->direction = mDirection; + h264Metadata->mp4Seek = isMp4SeekFrame; h264Metadata->setData(*h264Metadata); readSPSPPS(); + Mp4ReaderDetailAbs::setMetadata(); - mSetMetadata(h264ImagePinId, metadata); + mSetMetadata(h264ImagePinId, mH264Metadata); return; } @@ -1231,11 +1239,11 @@ bool Mp4ReaderDetailH264::produceFrames(frame_container& frames) return true; } - if (mState.shouldPrependSpsPps) + if (mState.shouldPrependSpsPps || (!mState.direction && !mState.foundFirstReverseIFrame)) { boost::asio::mutable_buffer tmpBuffer(imgFrame->data(), imgFrame->size()); auto type = H264Utils::getNALUType((char*)tmpBuffer.data()); - if (type != H264Utils::H264_NAL_TYPE_END_OF_SEQ) + if (type == H264Utils::H264_NAL_TYPE_IDR_SLICE) { auto tempFrame = makeFrame(imgSize + spsSize + ppsSize + 8, h264ImagePinId); uint8_t* tempFrameBuffer = reinterpret_cast(tempFrame->data()); @@ -1244,8 +1252,14 @@ bool Mp4ReaderDetailH264::produceFrames(frame_container& frames) memcpy(tempFrameBuffer, imgFrame->data(), imgSize); imgSize += spsSize + ppsSize + 8; imgFrame = tempFrame; + mState.foundFirstReverseIFrame = true; + mState.shouldPrependSpsPps = false; + } + else if (type == H264Utils::H264_NAL_TYPE_SEQ_PARAM) + { + mState.shouldPrependSpsPps = false; + mState.foundFirstReverseIFrame = true; } - mState.shouldPrependSpsPps = false; } auto trimmedImgFrame = makeFrameTrim(imgFrame, imgSize, h264ImagePinId); @@ -1256,6 +1270,7 @@ bool Mp4ReaderDetailH264::produceFrames(frame_container& frames) { frameData[3] = 0x1; frameData[spsSize + 7] = 0x1; + frameData[spsSize + ppsSize + 9] = 0x0; frameData[spsSize + ppsSize + 10] = 0x0; frameData[spsSize + ppsSize + 11] = 0x1; } @@ -1301,6 +1316,11 @@ bool Mp4ReaderDetailH264::produceFrames(frame_container& frames) } frames.insert(make_pair(metadataFramePinId, trimmedMetadataFrame)); } + if (isMp4SeekFrame) + { + isMp4SeekFrame = false; + setMetadata(); + } return true; } diff --git a/base/test/mp4_reverse_play_tests.cpp b/base/test/mp4_reverse_play_tests.cpp index 48aff5cd7..b5ff202f9 100644 --- a/base/test/mp4_reverse_play_tests.cpp +++ b/base/test/mp4_reverse_play_tests.cpp @@ -6,14 +6,18 @@ #include "Logger.h" #include "AIPExceptions.h" #include "PipeLine.h" - #include "Mp4ReaderSource.h" #include "Mp4VideoMetadata.h" #include "EncodedImageMetadata.h" #include "StatSink.h" - +#include "H264Metadata.h" #include "FrameContainerQueue.h" +#ifdef APRA_CUDA_ENABLED +#include "CudaCommon.h" +#include "H264Decoder.h" +#endif + BOOST_AUTO_TEST_SUITE(mp4_reverse_play) class TestModule : public Module @@ -91,7 +95,7 @@ class TestModule1 : public TestModule struct SetupPlaybackTests { SetupPlaybackTests(std::string videoPath, - bool reInitInterval, bool direction, bool parseFS) + bool reInitInterval, bool direction, bool parseFS, FrameMetadata::FrameType frameType) { LoggerProps loggerProps; loggerProps.logLevel = boost::log::trivial::severity_level::info; @@ -103,12 +107,20 @@ struct SetupPlaybackTests mp4ReaderProps.logHealthFrequency = 1000; mp4ReaderProps.fps = 100; mp4Reader = boost::shared_ptr(new Mp4ReaderSource(mp4ReaderProps)); - auto encodedImageMetadata = framemetadata_sp(new EncodedImageMetadata(0, 0)); - mp4Reader->addOutPutPin(encodedImageMetadata); + if (frameType == FrameMetadata::FrameType::ENCODED_IMAGE) + { + auto encodedImageMetadata = framemetadata_sp(new EncodedImageMetadata(0, 0)); + mp4Reader->addOutPutPin(encodedImageMetadata); + } + else if (frameType == FrameMetadata::FrameType::H264_DATA) + { + auto h264ImageMetadata = framemetadata_sp(new H264Metadata(0, 0)); + mp4Reader->addOutPutPin(h264ImageMetadata); + } auto mp4Metadata = framemetadata_sp(new Mp4VideoMetadata("v_2_0")); mp4Reader->addOutPutPin(mp4Metadata); - TestModuleProps sinkProps;// (30, 100, true); + TestModuleProps sinkProps; //sinkProps.logHealth = false; sinkProps.logHealthFrequency = 1; sink = boost::shared_ptr(new TestModule1(sinkProps)); @@ -122,10 +134,56 @@ struct SetupPlaybackTests boost::shared_ptr sink = nullptr; }; +#ifdef APRA_CUDA_ENABLED +struct SetupPlaybackDecoderTests +{ + SetupPlaybackDecoderTests(std::string videoPath, + bool reInitInterval, bool direction, bool parseFS) + { + LoggerProps loggerProps; + loggerProps.logLevel = boost::log::trivial::severity_level::info; + Logger::initLogger(loggerProps); + + bool readLoop = false; + auto mp4ReaderProps = Mp4ReaderSourceProps(videoPath, parseFS, reInitInterval, direction, readLoop, false); + mp4ReaderProps.logHealth = true; + mp4ReaderProps.logHealthFrequency = 1000; + mp4ReaderProps.fps = 100; + mp4Reader = boost::shared_ptr(new Mp4ReaderSource(mp4ReaderProps)); + + auto h264ImageMetadata = framemetadata_sp(new H264Metadata(0, 0)); + mp4Reader->addOutPutPin(h264ImageMetadata); + + auto mp4Metadata = framemetadata_sp(new Mp4VideoMetadata("v_2_0")); + mp4Reader->addOutPutPin(mp4Metadata); + + std::vector mImagePin; + mImagePin = mp4Reader->getAllOutputPinsByType(FrameMetadata::H264_DATA); + + decoder = boost::shared_ptr(new H264Decoder(H264DecoderProps())); + mp4Reader->setNext(decoder, mImagePin); + + TestModuleProps sinkProps; + //sinkProps.logHealth = false; + sinkProps.logHealthFrequency = 1; + sink = boost::shared_ptr(new TestModule1(sinkProps)); + decoder->setNext(sink); + + BOOST_TEST(mp4Reader->init()); + BOOST_TEST(decoder->init()); + BOOST_TEST(sink->init()); + } + + boost::shared_ptr mp4Reader; + boost::shared_ptr decoder; + boost::shared_ptr sink = nullptr; +}; +#endif + BOOST_AUTO_TEST_CASE(fwd) { std::string videoPath = "data/Mp4_videos/mp4_seek_tests/20220522/0016/1655895288956.mp4"; - SetupPlaybackTests f(videoPath, 0, true, true); + SetupPlaybackTests f(videoPath, 0, true, true, FrameMetadata::ENCODED_IMAGE); int ct = 0, total = 601; while (ct < total - 1) @@ -154,7 +212,7 @@ BOOST_AUTO_TEST_CASE(fwd) BOOST_AUTO_TEST_CASE(switch_playback) { std::string videoPath = "data/Mp4_videos/mp4_seek_tests/20220522/0016/1655895288956.mp4"; - SetupPlaybackTests f(videoPath, 0, true, true); + SetupPlaybackTests f(videoPath, 0, true, true, FrameMetadata::ENCODED_IMAGE); f.mp4Reader->step(); auto sinkQ = f.sink->getQue(); @@ -237,7 +295,7 @@ BOOST_AUTO_TEST_CASE(switch_playback) BOOST_AUTO_TEST_CASE(video_coverage) { std::string videoPath = "data/Mp4_videos/mp4_seek_tests/20220522/0023/1655919060000.mp4"; - SetupPlaybackTests f(videoPath, 0, true, true); + SetupPlaybackTests f(videoPath, 0, true, true, FrameMetadata::ENCODED_IMAGE); /* forward playback verification */ f.mp4Reader->step(); @@ -304,7 +362,7 @@ BOOST_AUTO_TEST_CASE(seek_in_revPlayback_prev_hr) std::string videoPath = "data/Mp4_videos/mp4_seek_tests/20220522/0023/1655919060000.mp4"; bool direction = false; - SetupPlaybackTests f(videoPath, 0, direction, true); + SetupPlaybackTests f(videoPath, 0, direction, true, FrameMetadata::ENCODED_IMAGE); auto sinkQ = f.sink->getQue(); // last frame @@ -335,7 +393,7 @@ BOOST_AUTO_TEST_CASE(seek_in_revPlayback_prev_day) std::string videoPath = "data/Mp4_videos/mp4_seek_tests/20220523/0001/1655926320000.mp4"; bool direction = false; - SetupPlaybackTests f(videoPath, 0, direction, true); + SetupPlaybackTests f(videoPath, 0, direction, true, FrameMetadata::ENCODED_IMAGE); auto sinkQ = f.sink->getQue(); // last frame @@ -365,7 +423,7 @@ BOOST_AUTO_TEST_CASE(seek_in_revPlay_prev_hr) std::string videoPath = "data/Mp4_videos/mp4_seek_tests/20220522/0016/1655895162221.mp4"; bool direction = false; - SetupPlaybackTests f(videoPath, 0, direction, true); + SetupPlaybackTests f(videoPath, 0, direction, true, FrameMetadata::ENCODED_IMAGE); auto sinkQ = f.sink->getQue(); // last frame @@ -388,7 +446,7 @@ BOOST_AUTO_TEST_CASE(seek_in_revPlay_fail_to_seek_infile_restore) std::string videoPath = "data/Mp4_videos/mp4_seek_tests/20220522/0023/1655919060000.mp4"; bool direction = false; - SetupPlaybackTests f(videoPath, 0, direction, true); + SetupPlaybackTests f(videoPath, 0, direction, true, FrameMetadata::ENCODED_IMAGE); auto sinkQ = f.sink->getQue(); // last frame @@ -429,7 +487,7 @@ BOOST_AUTO_TEST_CASE(seek_dir_change_trig_fresh_parse) std::string videoPath = "data/Mp4_videos/mp4_seek_tests/20220523/0001/1655926320000.mp4"; bool direction = true; - SetupPlaybackTests f(videoPath, 0, direction, true); + SetupPlaybackTests f(videoPath, 0, direction, true, FrameMetadata::ENCODED_IMAGE); auto sinkQ = f.sink->getQue(); // last frame // first @@ -493,7 +551,7 @@ BOOST_AUTO_TEST_CASE(step_only_parse_disabled_video_cov_with_reinitInterval) with reinitInterval */ std::string videoPath = "data/Mp4_videos/mp4_seek_tests/apra.mp4"; - SetupPlaybackTests f(videoPath, 10, true, false); + SetupPlaybackTests f(videoPath, 10, true, false, FrameMetadata::ENCODED_IMAGE); /* forward playback verification */ f.mp4Reader->step(); @@ -540,4 +598,558 @@ BOOST_AUTO_TEST_CASE(step_only_parse_disabled_video_cov_with_reinitInterval) BOOST_TEST(lastFrameTS == 1673855454254); } +BOOST_AUTO_TEST_CASE(fwd_h264) +{ + std::string videoPath = "./data/Mp4_videos/mp4_seeks_tests_h264/20230111/0012/1673420640350.mp4"; + SetupPlaybackTests f(videoPath, 0, true, true, FrameMetadata::H264_DATA); + + int ct = 0, total = 500; + while (ct < total - 1) + { + f.mp4Reader->step(); + auto sinkQ = f.sink->getQue(); + auto frames = sinkQ->try_pop(); + auto frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + LOG_INFO << "frame->timestamp <" << frame->timestamp << ">"; + ct++; + } + f.mp4Reader->step(); + auto sinkQ = f.sink->getQue(); + auto frames = sinkQ->try_pop(); + auto frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1673420645353); + LOG_INFO << "frame->timestamp <" << frame->timestamp << ">"; + + // new video open + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604318680); +} + +BOOST_AUTO_TEST_CASE(switch_playback_h264) +{ + std::string videoPath = "./data/Mp4_videos/mp4_seeks_tests_h264/20230501/0012/1685604361723.mp4"; + SetupPlaybackTests f(videoPath, 0, true, true, FrameMetadata::H264_DATA); + + f.mp4Reader->step(); + auto sinkQ = f.sink->getQue(); + auto frames = sinkQ->try_pop(); + auto frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604361723); + + LOG_INFO << "changing playback bwd>"; + f.mp4Reader->changePlayback(1, false); + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604361723); + + // new video open + new file parse happens + LOG_INFO << "new video opens"; + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604325685); + + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604325655); + + LOG_INFO << "chaning playback fwd>"; + f.mp4Reader->changePlayback(1, true); + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604325655); + + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604325685); + + // new video open + LOG_INFO << "new video opens<><>"; + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604361723); + LOG_INFO << "1 frame->timestamp <" << frame->timestamp << ">"; + + int nFramesInOpenVideo = 151, count = 1; + while (count < nFramesInOpenVideo - 1) + { + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + LOG_TRACE << "frameIdx/total <" << count << "/" << nFramesInOpenVideo << ">"; + LOG_TRACE << "frame->timestamp <" << frame->timestamp << ">"; + ++count; + } + // last frame of open video + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604366727); + + // new video open + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604896179); +} + +BOOST_AUTO_TEST_CASE(video_coverage_h264) +{ + std::string videoPath = "./data/Mp4_videos/mp4_seeks_tests_h264/20230501/0012/1685604361723.mp4"; + SetupPlaybackTests f(videoPath, 0, true, true, FrameMetadata::H264_DATA); + + /* forward playback verification */ + f.mp4Reader->step(); + auto sinkQ = f.sink->getQue(); + auto frames = sinkQ->try_pop(); + auto frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604361723); + + int nFramesInOpenVideo = 151, count = 1; + while (count < nFramesInOpenVideo - 1) + { + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + LOG_TRACE << "frameIdx/total <" << count << "/" << nFramesInOpenVideo << ">"; + LOG_TRACE << "frame->timestamp <" << frame->timestamp << ">"; + ++count; + } + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == (1685604361723 + 5004)); + + /* backward playback verification */ + LOG_INFO << "changing playback bwd>"; + f.mp4Reader->changePlayback(1, false); + f.mp4Reader->step(); + sinkQ = f.sink->getQue(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == (1685604361723 + 5004)); + + nFramesInOpenVideo = 151, count = 1; + while (count < nFramesInOpenVideo - 1) + { + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + LOG_TRACE << "frameIdx/total <" << count << "/" << nFramesInOpenVideo << ">"; + LOG_TRACE << "frame->timestamp <" << frame->timestamp << ">"; + ++count; + } + // first frame + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604361723); + + // new (prev) video open + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604325685); +} + +BOOST_AUTO_TEST_CASE(seek_in_revPlayback_prev_hr_h264) +{ + std::string videoPath = "./data/Mp4_videos/mp4_seeks_tests_h264/20230501/0012/1685604361723.mp4"; + bool direction = false; + + SetupPlaybackTests f(videoPath, 0, direction, true, FrameMetadata::H264_DATA); + auto sinkQ = f.sink->getQue(); + + // last frame + f.mp4Reader->step(); + auto frames = sinkQ->try_pop(); + auto frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604361723 + 5004); + + // 2nd I frame is at 1685604362731 + f.mp4Reader->randomSeek(1685604361723 + 1010, false); + f.mp4Reader->step(); + + // first frame + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604362731); + + int nFramesInOpenVideo = 30, count = 0; + while (count < nFramesInOpenVideo - 1) + { + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + LOG_TRACE << "frameIdx/total <" << count << "/" << nFramesInOpenVideo << ">"; + LOG_TRACE << "frame->timestamp <" << frame->timestamp << ">"; + ++count; + } + + //first frame + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604361723); + + // new (prev) video open + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604325685); +} + +BOOST_AUTO_TEST_CASE(seek_in_revPlay_fail_to_seek_infile_restore_h264) +{ + std::string videoPath = "./data/Mp4_videos/mp4_seeks_tests_h264/20230501/0012/1685604361723.mp4"; + bool direction = false; + + SetupPlaybackTests f(videoPath, 0, direction, true, FrameMetadata::H264_DATA); + auto sinkQ = f.sink->getQue(); + + // last frame + f.mp4Reader->step(); + auto frames = sinkQ->try_pop(); + auto frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame != nullptr); + BOOST_TEST(frame->timestamp == 1685604361723 + 5004); + + // nothing further on disk + f.mp4Reader->randomSeek(1673420640300, false); + f.mp4Reader->step(); + frames = sinkQ->pop(); + BOOST_TEST(frames.begin()->second->isEOS()); + auto eosFrame = dynamic_cast(frames.begin()->second.get()); + auto type = eosFrame->getEoSFrameType(); + BOOST_TEST(type == EoSFrame::EoSFrameType::MP4_SEEK_EOS); + + // last frame of the new (prev) video open + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1685604361723 + 4974); +} + +BOOST_AUTO_TEST_CASE(seek_dir_change_trig_fresh_parse_h264) +{ + std::string videoPath = "./data/Mp4_videos/mp4_seeks_tests_h264/20230501/0013/1685604896179.mp4"; + bool direction = true; + + SetupPlaybackTests f(videoPath, 0, direction, true, FrameMetadata::H264_DATA); + auto sinkQ = f.sink->getQue(); + + // first frame + f.mp4Reader->step(); + auto frames = sinkQ->try_pop(); + auto frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame != nullptr); + BOOST_TEST(frame->timestamp == 1685604896179); + + + f.mp4Reader->randomSeek(1685604896179 + 10, false); + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame != nullptr); + BOOST_TEST(frame->timestamp == 1685604896179); + + auto snap = f.mp4Reader->getCacheSnapShot(); + printCache(snap); + + // bwd seek -- first + f.mp4Reader->play(1, false); + f.mp4Reader->randomSeek(1673420640350 + 2, false); + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame != nullptr); + BOOST_TEST(frame->timestamp == 1673420640350); + + snap = f.mp4Reader->getCacheSnapShot(); + printCache(snap); + + // change direction - fwd - seek into --second + f.mp4Reader->play(1, true); + f.mp4Reader->randomSeek(1685604318680, false); // use play + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame != nullptr); + BOOST_TEST(frame->timestamp == 1685604318680); + + snap = f.mp4Reader->getCacheSnapShot(); + printCache(snap); + + // change direction - bwd - seek into --second + f.mp4Reader->play(1, false); + f.mp4Reader->randomSeek(1685604318680 + 10, false); // use play + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame != nullptr); + BOOST_TEST(frame->timestamp == 1685604318680); +} + +BOOST_AUTO_TEST_CASE(step_only_parse_disabled_video_cov_with_reinitInterval_h264) +{ + /* + video coverage test i.e. [st -> end (eof) | direction change | end -> st (eof)] + parse disabled + using only step + with reinitInterval + */ + std::string videoPath = "data/Mp4_videos/mp4_seeks_tests_h264/apraH264.mp4"; + SetupPlaybackTests f(videoPath, 10, true, false, FrameMetadata::H264_DATA); + + /* forward playback verification */ + f.mp4Reader->step(); + auto sinkQ = f.sink->getQue(); + auto frames = sinkQ->try_pop(); + auto frame = Module::getFrameByType(frames, FrameMetadata::FrameType::H264_DATA); + BOOST_TEST(frame->timestamp == 1673420640350); + + while (1) + { + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = frames.begin()->second; + if (frame->isEOS()) + { + auto eosFrame = dynamic_cast(frame.get()); + BOOST_TEST(eosFrame->getEoSFrameType() == EoSFrame::EoSFrameType::MP4_PLYB_EOS); + break; + } + } + LOG_INFO << "Reached EOF !"; + + /* backward playback verification */ + LOG_INFO << "changing playback bwd>"; + uint64_t lastFrameTS = 0; + f.mp4Reader->changePlayback(1, false); + while (1) + { + f.mp4Reader->step(); + frames = sinkQ->try_pop(); + frame = frames.begin()->second; + if (frame->isEOS()) + { + auto eosFrame = dynamic_cast(frame.get()); + BOOST_TEST(eosFrame->getEoSFrameType() == EoSFrame::EoSFrameType::MP4_PLYB_EOS); + break; + } + else + { + lastFrameTS = frame->timestamp; + } + } + LOG_INFO << "Reached EOF !"; + BOOST_TEST(lastFrameTS == 1673420640350); +} + +#ifdef APRA_CUDA_ENABLED + +BOOST_AUTO_TEST_CASE(fwd_h264_decoder_change_playback_bwd) +{ +#ifdef __x86_64__ + auto cuContext = apracucontext_sp(new ApraCUcontext()); + int major=0,minor=0; + if(!cuContext->getComputeCapability(major,minor)) + return; + LOG_INFO << "Compute Cap "<step(); + f.decoder->step(); + auto sinkQ = f.sink->getQue(); + frames = sinkQ->try_pop(); + } + frameTs = frames.begin()->second->timestamp; + BOOST_TEST(frameTs == 1673420640350); + + f.mp4Reader->changePlayback(1, false); + //since there is a delay in deocoder the first reverse frame will be sent once all the forward frames are processed. + while(!frames.empty()) + { + f.mp4Reader->step(); + f.decoder->step(); + auto sinkQ = f.sink->getQue(); + frames = sinkQ->try_pop(); + if (frameTs == frames.begin()->second->timestamp) + break; + frameTs = frames.begin()->second->timestamp; + } + BOOST_TEST(frameTs == frames.begin()->second->timestamp);//First reverse frame +} + +BOOST_AUTO_TEST_CASE(fwd_h264_decoder_change_playback) +{ +#ifdef __x86_64__ + auto cuContext = apracucontext_sp(new ApraCUcontext()); + int major=0,minor=0; + if(!cuContext->getComputeCapability(major,minor)) + return; + LOG_INFO << "Compute Cap "<step(); + f.decoder->step(); + auto sinkQ = f.sink->getQue(); + frames = sinkQ->try_pop(); + } + frameTs = frames.begin()->second->timestamp; + BOOST_TEST(frameTs == 1691502958947); + + f.mp4Reader->changePlayback(1, false); + //since there is a delay in deocoder the first reverse frame will be sent once all the forward frames are processed. + while (!frames.empty()) + { + f.mp4Reader->step(); + f.decoder->step(); + auto sinkQ = f.sink->getQue(); + frames = sinkQ->try_pop(); + if (frameTs == frames.begin()->second->timestamp)//since a repeated whenever direction changes + break; + frameTs = frames.begin()->second->timestamp; + } + + BOOST_TEST(frameTs == frames.begin()->second->timestamp);//First reverse frame + + f.mp4Reader->changePlayback(1, true); + + while (!frames.empty()) + { + f.mp4Reader->step(); + f.decoder->step(); + auto sinkQ = f.sink->getQue(); + frames = sinkQ->try_pop(); + if (frameTs == frames.begin()->second->timestamp) + break; + frameTs = frames.begin()->second->timestamp; + } + + BOOST_TEST(frameTs == frames.begin()->second->timestamp);//First forward frame +} + +BOOST_AUTO_TEST_CASE(reverse_play_deocoder) +{ +#ifdef __x86_64__ + auto cuContext = apracucontext_sp(new ApraCUcontext()); + int major=0,minor=0; + if(!cuContext->getComputeCapability(major,minor)) + return; + LOG_INFO << "Compute Cap "<step(); + f.decoder->step(); + auto sinkQ = f.sink->getQue(); + frames = sinkQ->try_pop(); + } + frameTs = frames.begin()->second->timestamp; + if (frameTs == 1691503018158);//last frame of the video. +} + +BOOST_AUTO_TEST_CASE(reverse_play_to_fwd_play_decoder) +{ +#ifdef __x86_64__ + auto cuContext = apracucontext_sp(new ApraCUcontext()); + int major=0,minor=0; + if(!cuContext->getComputeCapability(major,minor)) + return; + LOG_INFO << "Compute Cap "<step(); + f.decoder->step(); + auto sinkQ = f.sink->getQue(); + frames = sinkQ->try_pop(); + } + frameTs = frames.begin()->second->timestamp; + + if (frameTs == 1691503018158);//last frame of the video. + + for (int i = 0; i < 10; i++) + { + f.mp4Reader->step(); + f.decoder->step(); + auto sinkQ = f.sink->getQue(); + frames = sinkQ->try_pop(); + } + + f.mp4Reader->changePlayback(1, true);//IntraGop direction to fwd , then till next I frame all the p frames are skipped + + while (!frames.empty()) + { + f.mp4Reader->step(); + f.decoder->step(); + auto sinkQ = f.sink->getQue(); + frames = sinkQ->try_pop(); + if (frameTs == 1691503017167)//First forward frame and also I frame + break; + frameTs = frames.begin()->second->timestamp; + } +} +#endif + BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/base/test/mp4_seek_tests.cpp b/base/test/mp4_seek_tests.cpp index 0ac46db14..94380d59c 100644 --- a/base/test/mp4_seek_tests.cpp +++ b/base/test/mp4_seek_tests.cpp @@ -14,6 +14,11 @@ #include "FrameContainerQueue.h" #include "H264Metadata.h" +#ifdef APRA_CUDA_ENABLED +#include "CudaCommon.h" +#include "H264Decoder.h" +#endif + BOOST_AUTO_TEST_SUITE(mp4_seek_tests) struct SetupSeekTests @@ -52,6 +57,48 @@ struct SetupSeekTests BOOST_TEST(sink->init()); } +#ifdef APRA_CUDA_ENABLED + SetupSeekTests(std::string videoPath, + bool reInitInterval, bool direction, bool parseFS) + { + LoggerProps loggerProps; + loggerProps.logLevel = boost::log::trivial::severity_level::info; + Logger::initLogger(loggerProps); + + bool readLoop = false; + auto mp4ReaderProps = Mp4ReaderSourceProps(videoPath, parseFS, reInitInterval, direction, readLoop, false); + mp4ReaderProps.logHealth = true; + mp4ReaderProps.logHealthFrequency = 1000; + mp4ReaderProps.fps = 100; + mp4Reader = boost::shared_ptr(new Mp4ReaderSource(mp4ReaderProps)); + + auto h264ImageMetadata = framemetadata_sp(new H264Metadata(0, 0)); + mp4Reader->addOutPutPin(h264ImageMetadata); + + auto mp4Metadata = framemetadata_sp(new Mp4VideoMetadata("v_2_0")); + mp4Reader->addOutPutPin(mp4Metadata); + + std::vector mImagePin; + mImagePin = mp4Reader->getAllOutputPinsByType(FrameMetadata::H264_DATA); + + decoder = boost::shared_ptr(new H264Decoder(H264DecoderProps())); + mp4Reader->setNext(decoder, mImagePin); + + auto sinkProps = ExternalSinkProps(); + sinkProps.logHealth = true; + sinkProps.logHealthFrequency = 1000; + sink = boost::shared_ptr(new ExternalSink(sinkProps)); + decoder->setNext(sink); + + auto p = boost::shared_ptr(new PipeLine("mp4reader")); + p->appendModule(mp4Reader); + + BOOST_TEST(mp4Reader->init()); + BOOST_TEST(decoder->init()); + BOOST_TEST(sink->init()); + } +#endif + ~SetupSeekTests() { mp4Reader->term(); @@ -125,6 +172,9 @@ struct SetupSeekTests boost::shared_ptr p = nullptr; boost::shared_ptr mp4Reader; boost::shared_ptr sink; +#ifdef APRA_CUDA_ENABLED + boost::shared_ptr decoder; +#endif }; BOOST_AUTO_TEST_CASE(no_seek) @@ -775,7 +825,7 @@ BOOST_AUTO_TEST_CASE(seek_in_next_file_h264) auto imgFrame = frames.begin()->second; - /* ts of first frame of next file is 1655895288956 - video length 5secs. + /* ts of first frame of next file is 1685604361723 - video length 5secs. Seek 3 sec inside the file which is next to the currently open file. */ uint64_t skipTS = 1685604364723; s.mp4Reader->randomSeek(skipTS, false); @@ -1296,4 +1346,326 @@ BOOST_AUTO_TEST_CASE(read_loop_h264) BOOST_TEST(imgFrame->timestamp == 1673420640350); } +#ifdef APRA_CUDA_ENABLED +BOOST_AUTO_TEST_CASE(seek_inside_cache_fwd) +{ +#ifdef __x86_64__ + auto cuContext = apracucontext_sp(new ApraCUcontext()); + int major=0,minor=0; + if(!cuContext->getComputeCapability(major,minor)) + return; + LOG_INFO << "Compute Cap "<step(); + s.decoder->step(); + auto sinkQ = s.sink->getQue(); + frames = sinkQ->try_pop(); + } + frameTs = frames.begin()->second->timestamp; + + BOOST_TEST(frameTs == 1691502958947);//First frame + + for (int i = 0; i < 90; i++) + { + s.mp4Reader->step(); + s.decoder->step(); + auto sinkQ = s.sink->getQue(); + frames = sinkQ->try_pop(); + } + + uint64_t skipTS = 1691502961947; + s.mp4Reader->randomSeek(skipTS, false); + + while(!frames.empty()) + { + s.mp4Reader->step(); + s.decoder->step(); + auto sinkQ = s.sink->getQue(); + frames = sinkQ->try_pop(); + frameTs = frames.begin()->second->timestamp; + if (frameTs == 1691502962087) + break; + } + + BOOST_TEST(frameTs == 1691502962087);//seek frame + + s.decoder->term(); +} + +BOOST_AUTO_TEST_CASE(seek_outside_cache_fwd) +{ +#ifdef __x86_64__ + auto cuContext = apracucontext_sp(new ApraCUcontext()); + int major=0,minor=0; + if(!cuContext->getComputeCapability(major,minor)) + return; + LOG_INFO << "Compute Cap "<step(); + s.decoder->step(); + auto sinkQ = s.sink->getQue(); + frames = sinkQ->try_pop(); + } + frameTs = frames.begin()->second->timestamp; + + BOOST_TEST(frameTs == 1691502958947);//First frame + + uint64_t skipTS = 1691502963937; + s.mp4Reader->randomSeek(skipTS, false); + + while (!frames.empty()) + { + s.mp4Reader->step(); + s.decoder->step(); + auto sinkQ = s.sink->getQue(); + frames = sinkQ->try_pop(); + frameTs = frames.begin()->second->timestamp; + if (frameTs == 1691502964160) + break; + } + + BOOST_TEST(frameTs == 1691502964160); + + s.decoder->term(); +} + +BOOST_AUTO_TEST_CASE(seek_inside_cache_bwd) +{ +#ifdef __x86_64__ + auto cuContext = apracucontext_sp(new ApraCUcontext()); + int major=0,minor=0; + if(!cuContext->getComputeCapability(major,minor)) + return; + LOG_INFO << "Compute Cap "<step(); + s.decoder->step(); + auto sinkQ = s.sink->getQue(); + frames = sinkQ->try_pop(); + } + + uint64_t skipTS = 1691503016167; + s.mp4Reader->randomSeek(skipTS, false); + + while (!frames.empty()) + { + s.mp4Reader->step(); + s.decoder->step(); + auto sinkQ = s.sink->getQue(); + frames = sinkQ->try_pop(); + frameTs = frames.begin()->second->timestamp; + if (frameTs == 1691503017167) + break; + } + BOOST_TEST(frameTs == 1691503017167);//seeked frame + s.decoder->term(); +} + +BOOST_AUTO_TEST_CASE(seek_outside_cache_bwd) +{ +#ifdef __x86_64__ + auto cuContext = apracucontext_sp(new ApraCUcontext()); + int major=0,minor=0; + if(!cuContext->getComputeCapability(major,minor)) + return; + LOG_INFO << "Compute Cap "<step(); + s.decoder->step(); + auto sinkQ = s.sink->getQue(); + frames = sinkQ->try_pop(); + } + + uint64_t skipTS = 1691503002167; + s.mp4Reader->randomSeek(skipTS, false); + + while(!frames.empty())//Process all the buffered backward frames to get the seeked I frame. + { + s.mp4Reader->step(); + s.decoder->step(); + auto sinkQ = s.sink->getQue(); + frames = sinkQ->try_pop(); + frameTs = frames.begin()->second->timestamp; + if (frameTs == 1691503001585) + break; + } + BOOST_TEST(frameTs == 1691503001585);//seeked Frame + + s.decoder->term(); +} + +BOOST_AUTO_TEST_CASE(seek_inside_cache_and_dir_change_to_bwd) +{ +#ifdef __x86_64__ + auto cuContext = apracucontext_sp(new ApraCUcontext()); + int major=0,minor=0; + if(!cuContext->getComputeCapability(major,minor)) + return; + LOG_INFO << "Compute Cap "<step(); + s.decoder->step(); + auto sinkQ = s.sink->getQue(); + frames = sinkQ->try_pop(); + } + frameTs = frames.begin()->second->timestamp; + + BOOST_TEST(frameTs == 1691502958947);//First frame + + for (int i = 0; i < 90; i++) + { + s.mp4Reader->step(); + s.decoder->step(); + auto sinkQ = s.sink->getQue(); + frames = sinkQ->try_pop(); + } + + uint64_t skipTS = 1691502961947; + s.mp4Reader->randomSeek(skipTS, false); + s.mp4Reader->changePlayback(1, false); + + while (!frames.empty()) + { + s.mp4Reader->step(); + s.decoder->step(); + auto sinkQ = s.sink->getQue(); + frames = sinkQ->try_pop(); + frameTs = frames.begin()->second->timestamp; + if (frameTs == 1691502962042) + break; + } + BOOST_TEST(frameTs == 1691502962042); + + s.decoder->term(); +} + +BOOST_AUTO_TEST_CASE(seek_outside_cache_and_dir_change_to_fwd) +{ +#ifdef __x86_64__ + auto cuContext = apracucontext_sp(new ApraCUcontext()); + int major=0,minor=0; + if(!cuContext->getComputeCapability(major,minor)) + return; + LOG_INFO << "Compute Cap "<step(); + s.decoder->step(); + auto sinkQ = s.sink->getQue(); + frames = sinkQ->try_pop(); + } + frameTs = frames.begin()->second->timestamp; + + BOOST_TEST(frameTs == 1691503018158);//First frame + + uint64_t skipTS = 1691502963937; + s.mp4Reader->changePlayback(1, true); + s.mp4Reader->randomSeek(skipTS, false); + + + while (!frames.empty()) + { + s.mp4Reader->step(); + s.decoder->step(); + auto sinkQ = s.sink->getQue(); + frames = sinkQ->try_pop(); + frameTs = frames.begin()->second->timestamp; + if (frameTs == 1691502964160) + break; + } + BOOST_TEST(frameTs == 1691502964160);//seeked_i_Frame + s.decoder->term(); +} +#endif + BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/data/Mp4_videos/h264_reverse_play/20230708/0019/1691502958947.mp4 b/data/Mp4_videos/h264_reverse_play/20230708/0019/1691502958947.mp4 new file mode 100644 index 000000000..600aeb3f2 Binary files /dev/null and b/data/Mp4_videos/h264_reverse_play/20230708/0019/1691502958947.mp4 differ