From bd24a22bafa23c0a1d1800c03714d5535de6ea2d Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Thu, 23 Jun 2022 22:42:40 +0200 Subject: [PATCH 01/28] Stream buffer initial implementation --- Samples/MediaPlayerCPP/MainPage.xaml.cpp | 63 ++- Samples/MediaPlayerCPP/MainPage.xaml.h | 5 +- Samples/MediaPlayerCS/MainPage.xaml.cs | 70 +++- Source/FFmpegInteropX.vcxproj | 1 + Source/FFmpegMediaSource.cpp | 219 ++-------- Source/FFmpegMediaSource.h | 26 +- Source/FFmpegReader.cpp | 508 +++++++++++++++++++++-- Source/FFmpegReader.h | 247 ++++++++++- Source/MediaSampleProvider.cpp | 180 +++----- Source/MediaSampleProvider.h | 18 +- Source/MediaSourceConfig.h | 25 +- Source/NALPacketSampleProvider.cpp | 6 +- Source/NALPacketSampleProvider.h | 2 +- Source/StringUtils.h | 20 +- Source/SubtitleProvider.h | 245 +++++------ Source/SubtitleProviderBitmap.h | 280 +++++++++---- Source/UncompressedSampleProvider.cpp | 6 +- Source/UncompressedSampleProvider.h | 2 +- Source/UncompressedVideoSampleProvider.h | 4 +- 19 files changed, 1261 insertions(+), 666 deletions(-) diff --git a/Samples/MediaPlayerCPP/MainPage.xaml.cpp b/Samples/MediaPlayerCPP/MainPage.xaml.cpp index f28a125a67..1faeaccb3d 100644 --- a/Samples/MediaPlayerCPP/MainPage.xaml.cpp +++ b/Samples/MediaPlayerCPP/MainPage.xaml.cpp @@ -1,4 +1,4 @@ -//***************************************************************************** +//***************************************************************************** // // Copyright 2015 Microsoft Corporation // @@ -83,17 +83,31 @@ void MediaPlayerCPP::MainPage::OnButtonPressed(Windows::Media::SystemMediaTransp task MainPage::TryOpenLastFile() { - try - { - //Try open last file - auto file = co_await StorageApplicationPermissions::FutureAccessList->GetFileAsync( - StorageApplicationPermissions::FutureAccessList->Entries->GetAt(0).Token); - co_await OpenLocalFile(file); - } - catch (Exception^ ex) - { - DisplayErrorMessage(ex->Message); - } + try + { + //Try open last file + auto file = co_await StorageApplicationPermissions::FutureAccessList->GetFileAsync( + StorageApplicationPermissions::FutureAccessList->Entries->GetAt(0).Token); + co_await OpenLocalFile(file); + } + catch (Exception^ ex) + { + DisplayErrorMessage(ex->Message); + } +} + +task MainPage::TryOpenLastUri() +{ + try + { + //Try open last uri + auto uri = (String^)ApplicationData::Current->LocalSettings->Values->Lookup("LastUri"); + co_await OpenUriStream(uri); + } + catch (Exception^ ex) + { + DisplayErrorMessage(ex->Message); + } } void MainPage::OpenLocalFile(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e) @@ -192,6 +206,8 @@ task MainPage::OpenUriStream(Platform::String^ uri) // Close control panel after opening media Splitter->IsPaneOpen = false; + + ApplicationData::Current->LocalSettings->Values->Insert("LastUri", uri); } catch (Exception^ ex) { @@ -477,14 +493,33 @@ void MediaPlayerCPP::MainPage::OnKeyDown(Windows::UI::Core::CoreWindow^ sender, TryOpenLastFile(); } + if (args->VirtualKey == Windows::System::VirtualKey::Enter && (Window::Current->CoreWindow->GetKeyState(Windows::System::VirtualKey::Shift) & Windows::UI::Core::CoreVirtualKeyStates::Down) + == Windows::UI::Core::CoreVirtualKeyStates::Down && ApplicationData::Current->LocalSettings->Values->HasKey("LastUri")) + { + TryOpenLastUri(); + } + if (args->VirtualKey == Windows::System::VirtualKey::V) { if (playbackItem && playbackItem->VideoTracks->Size > 1) { - playbackItem->VideoTracks->SelectedIndex = - (playbackItem->VideoTracks->SelectedIndex + 1) % playbackItem->VideoTracks->Size; + bool reverse = (Window::Current->CoreWindow->GetKeyState(Windows::System::VirtualKey::Shift) & Windows::UI::Core::CoreVirtualKeyStates::Down) == Windows::UI::Core::CoreVirtualKeyStates::Down; + int index = reverse ? + (playbackItem->VideoTracks->SelectedIndex - 1) % playbackItem->VideoTracks->Size : + (playbackItem->VideoTracks->SelectedIndex + 1) % playbackItem->VideoTracks->Size; + playbackItem->VideoTracks->SelectedIndex = index; } } + + if (args->VirtualKey == Windows::System::VirtualKey::Right && FFmpegMSS && mediaPlayer->CanSeek) + { + mediaPlayer->Position = TimeSpan{ min(mediaPlayer->Position.Duration + 50000000, mediaPlayer->NaturalDuration.Duration) }; + } + + if (args->VirtualKey == Windows::System::VirtualKey::Left && FFmpegMSS && mediaPlayer->CanSeek) + { + mediaPlayer->Position = TimeSpan{ max(mediaPlayer->Position.Duration - 50000000, 0) }; + } } diff --git a/Samples/MediaPlayerCPP/MainPage.xaml.h b/Samples/MediaPlayerCPP/MainPage.xaml.h index caaafe6094..3eb79400df 100644 --- a/Samples/MediaPlayerCPP/MainPage.xaml.h +++ b/Samples/MediaPlayerCPP/MainPage.xaml.h @@ -1,4 +1,4 @@ -//***************************************************************************** +//***************************************************************************** // // Copyright 2015 Microsoft Corporation // @@ -43,7 +43,8 @@ namespace MediaPlayerCPP void OpenLocalFile(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e); task OpenLocalFile(); task OpenLocalFile(Windows::Storage::StorageFile^ file); - task TryOpenLastFile(); + task TryOpenLastFile(); + task TryOpenLastUri(); void URIBoxKeyUp(Platform::Object^ sender, Windows::UI::Xaml::Input::KeyRoutedEventArgs^ e); task OpenUriStream(Platform::String^ uri); void MediaFailed(Platform::Object^ sender, Windows::UI::Xaml::ExceptionRoutedEventArgs^ e); diff --git a/Samples/MediaPlayerCS/MainPage.xaml.cs b/Samples/MediaPlayerCS/MainPage.xaml.cs index 0e5649ac1e..2a37e303ef 100644 --- a/Samples/MediaPlayerCS/MainPage.xaml.cs +++ b/Samples/MediaPlayerCS/MainPage.xaml.cs @@ -1,4 +1,4 @@ -//***************************************************************************** +//***************************************************************************** // // Copyright 2015 Microsoft Corporation // @@ -92,12 +92,21 @@ private async void MainPage_KeyDown(CoreWindow sender, KeyEventArgs args) { await TryOpenLastFile(); } + if (args.VirtualKey == VirtualKey.Enter && (Window.Current.CoreWindow.GetKeyState(VirtualKey.Shift) & CoreVirtualKeyStates.Down) + == CoreVirtualKeyStates.Down && ApplicationData.Current.LocalSettings.Values.ContainsKey("LastUri")) + { + await TryOpenLastUri(); + } + if (args.VirtualKey == VirtualKey.V) { if (playbackItem != null && playbackItem.VideoTracks.Count > 1) { - playbackItem.VideoTracks.SelectedIndex = + bool reverse = (Window.Current.CoreWindow.GetKeyState(VirtualKey.Control) & CoreVirtualKeyStates.Down) == CoreVirtualKeyStates.Down; + int index = reverse ? + (playbackItem.VideoTracks.SelectedIndex - 1) % playbackItem.VideoTracks.Count : (playbackItem.VideoTracks.SelectedIndex + 1) % playbackItem.VideoTracks.Count; + playbackItem.VideoTracks.SelectedIndex = index; } } } @@ -151,6 +160,18 @@ private async Task TryOpenLastFile() { } } + private async Task TryOpenLastUri() + { + try + { + //Try open last uri + var uri = (string)ApplicationData.Current.LocalSettings.Values["LastUri"]; + await OpenStreamUri(uri); + } + catch (Exception) + { + } + } public MediaSourceConfig Config { get; set; } @@ -225,30 +246,37 @@ private async void URIBoxKeyUp(object sender, KeyRoutedEventArgs e) // Mark event as handled to prevent duplicate event to re-triggered e.Handled = true; - try - { - // Set FFmpeg specific options. List of options can be found in https://www.ffmpeg.org/ffmpeg-protocols.html + await OpenStreamUri(uri); + } + } - // Below are some sample options that you can set to configure RTSP streaming - // Config.FFmpegOptions.Add("rtsp_flags", "prefer_tcp"); - // Config.FFmpegOptions.Add("stimeout", 100000); + private async Task OpenStreamUri(string uri) + { + try + { + // Set FFmpeg specific options. List of options can be found in https://www.ffmpeg.org/ffmpeg-protocols.html - // Instantiate FFmpegMediaSource using the URI - mediaPlayer.Source = null; - FFmpegMSS = await FFmpegMediaSource.CreateFromUriAsync(uri, Config); + // Below are some sample options that you can set to configure RTSP streaming + // Config.FFmpegOptions.Add("rtsp_flags", "prefer_tcp"); + // Config.FFmpegOptions.Add("stimeout", 100000); - var source = FFmpegMSS.CreateMediaPlaybackItem(); + // Instantiate FFmpegMediaSource using the URI + mediaPlayer.Source = null; + FFmpegMSS = await FFmpegMediaSource.CreateFromUriAsync(uri, Config); - // Pass MediaStreamSource to Media Element - mediaPlayer.Source = source; + var source = FFmpegMSS.CreateMediaPlaybackItem(); - // Close control panel after opening media - Splitter.IsPaneOpen = false; - } - catch (Exception ex) - { - await DisplayErrorMessage(ex.Message); - } + // Pass MediaStreamSource to Media Element + mediaPlayer.Source = source; + + // Close control panel after opening media + Splitter.IsPaneOpen = false; + + ApplicationData.Current.LocalSettings.Values["LastUri"] = uri; + } + catch (Exception ex) + { + await DisplayErrorMessage(ex.Message); } } diff --git a/Source/FFmpegInteropX.vcxproj b/Source/FFmpegInteropX.vcxproj index 4875e7cfd3..e9e213b2f3 100644 --- a/Source/FFmpegInteropX.vcxproj +++ b/Source/FFmpegInteropX.vcxproj @@ -83,6 +83,7 @@ /bigobj /await /d2CoroOptsWorkaround %(AdditionalOptions) 28204;4635 true + stdcpp17 shcore.lib;runtimeobject.lib;mfuuid.lib;dxguid.lib;%(AdditionalDependencies) diff --git a/Source/FFmpegMediaSource.cpp b/Source/FFmpegMediaSource.cpp index 204e0b8822..c95299a9a9 100644 --- a/Source/FFmpegMediaSource.cpp +++ b/Source/FFmpegMediaSource.cpp @@ -111,6 +111,8 @@ FFmpegMediaSource::~FFmpegMediaSource() if (m_pReader != nullptr) { + m_pReader->Stop(); + m_pReader->Flush(); m_pReader = nullptr; } @@ -708,7 +710,7 @@ HRESULT FFmpegMediaSource::InitFFmpegContext() if (SUCCEEDED(hr)) { - m_pReader = ref new FFmpegReader(avFormatCtx, &sampleProviders); + m_pReader = ref new FFmpegReader(avFormatCtx, &sampleProviders, config); if (m_pReader == nullptr) { hr = E_OUTOFMEMORY; @@ -943,9 +945,6 @@ HRESULT FFmpegMediaSource::InitFFmpegContext() // Convert media duration from AV_TIME_BASE to TimeSpan unit mediaDuration = { LONGLONG(avFormatCtx->duration * 10000000 / double(AV_TIME_BASE)) }; - // Assign initial BufferTime to MediaStreamSource - mss->BufferTime = fileStreamData ? config->DefaultBufferTime : config->DefaultBufferTimeUri; - if (Windows::Foundation::Metadata::ApiInformation::IsPropertyPresent("Windows.Media.Core.MediaStreamSource", "MaxSupportedPlaybackRate")) { mss->MaxSupportedPlaybackRate = config->MaxSupportedPlaybackRate; @@ -965,6 +964,7 @@ HRESULT FFmpegMediaSource::InitFFmpegContext() startingRequestedToken = mss->Starting += ref new TypedEventHandler(this, &FFmpegMediaSource::OnStarting); sampleRequestedToken = mss->SampleRequested += ref new TypedEventHandler(this, &FFmpegMediaSource::OnSampleRequested); switchStreamRequestedToken = mss->SwitchStreamsRequested += ref new TypedEventHandler(this, &FFmpegMediaSource::OnSwitchStreamsRequested); + mss->Paused += ref new Windows::Foundation::TypedEventHandler(this, &FFmpegInteropX::FFmpegMediaSource::OnPaused); } } @@ -1557,6 +1557,13 @@ HRESULT FFmpegMediaSource::ParseOptions(PropertySet^ ffmpegOptions) return hr; } + +void FFmpegInteropX::FFmpegMediaSource::OnPaused(Windows::Media::Core::MediaStreamSource^ sender, Platform::Object^ args) +{ + m_pReader->Stop(); +} + + void FFmpegMediaSource::OnStarting(MediaStreamSource^ sender, MediaStreamSourceStartingEventArgs^ args) { mutexGuard.lock(); @@ -1618,6 +1625,8 @@ void FFmpegMediaSource::OnStarting(MediaStreamSource^ sender, MediaStreamSourceS } } + m_pReader->Start(); + isFirstSeek = false; isFirstSeekAfterStreamSwitch = false; mutexGuard.unlock(); @@ -1638,7 +1647,7 @@ void FFmpegMediaSource::OnSampleRequested(Windows::Media::Core::MediaStreamSourc CheckVideoDeviceChanged(); auto sample = currentVideoStream->GetNextSample(); args->Request->Sample = sample; - } + } else { args->Request->Sample = nullptr; @@ -1734,6 +1743,8 @@ void FFmpegMediaSource::CheckVideoDeviceChanged() void FFmpegMediaSource::OnSwitchStreamsRequested(MediaStreamSource^ sender, MediaStreamSourceSwitchStreamsRequestedEventArgs^ args) { mutexGuard.lock(); + m_pReader->Stop(); + m_pReader->Flush(); if (currentAudioStream && args->Request->OldStreamDescriptor == currentAudioStream->StreamDescriptor) { @@ -1742,12 +1753,14 @@ void FFmpegMediaSource::OnSwitchStreamsRequested(MediaStreamSource^ sender, Medi currentAudioStream->DisableFilters(); } currentAudioStream->DisableStream(); + currentAudioStream->Flush(true); currentAudioStream = nullptr; } if (currentVideoStream && args->Request->OldStreamDescriptor == currentVideoStream->StreamDescriptor) { currentVideoStream->DisableStream(); - currentVideoStream = nullptr; + currentAudioStream->Flush(true); + currentVideoStream = nullptr; } for each (auto stream in audioStreams) @@ -1778,182 +1791,19 @@ void FFmpegMediaSource::OnSwitchStreamsRequested(MediaStreamSource^ sender, Medi HRESULT FFmpegMediaSource::Seek(TimeSpan position, TimeSpan& actualPosition, bool allowFastSeek) { - auto hr = S_OK; - - // Select the first valid stream either from video or audio - auto stream = currentVideoStream ? currentVideoStream : currentAudioStream; - - if (stream) - { - int64_t seekTarget = stream->ConvertPosition(position); - auto diffActual = position - actualPosition; - auto diffLast = position - lastPosition; - bool isSeekBeforeStreamSwitch = PlaybackSession && config->FastSeekSmartStreamSwitching && diffActual.Duration > 0 && diffActual.Duration < 5000000 && diffLast.Duration > 0 && diffLast.Duration < 10000000; - - if (currentVideoStream && config->FastSeek && allowFastSeek && PlaybackSession && !isSeekBeforeStreamSwitch && !isFirstSeekAfterStreamSwitch) - { - // fast seek - auto playbackPosition = PlaybackSession ? lastPosition : currentVideoStream->LastSampleTimestamp; - bool seekForward; - TimeSpan referenceTime; - - // decide seek direction - if (isLastSeekForward && position > lastSeekStart && position <= lastSeekActual) - { - seekForward = true; - referenceTime = lastSeekStart + ((position - lastSeekStart) * 0.2); - DebugMessage(L" - ### Forward seeking continue\n"); - } - else if (!isLastSeekForward && position < lastSeekStart && position >= lastSeekActual) - { - seekForward = false; - referenceTime = lastSeekStart + ((position - lastSeekStart) * 0.2); - DebugMessage(L" - ### Backward seeking continue\n"); - } - else if (position >= playbackPosition) - { - seekForward = true; - referenceTime = playbackPosition + ((position - playbackPosition) * 0.2); - DebugMessage(L" - ### Forward seeking\n"); - } - else - { - seekForward = false; - referenceTime = playbackPosition + ((position - playbackPosition) * 0.2); - DebugMessage(L" - ### Backward seeking\n"); - } - - int64_t min = INT64_MIN; - int64_t max = INT64_MAX; - if (seekForward) - { - min = stream->ConvertPosition(referenceTime); - } - else - { - max = stream->ConvertPosition(referenceTime); - } - - if (avformat_seek_file(avFormatCtx, stream->StreamIndex, min, seekTarget, max, 0) < 0) - { - hr = E_FAIL; - DebugMessage(L" - ### Error while seeking\n"); - } - else - { - // Flush all active streams - FlushStreams(); - - // get and apply keyframe position for fast seeking - TimeSpan timestampVideo; - TimeSpan timestampVideoDuration; - hr = currentVideoStream->GetNextPacketTimestamp(timestampVideo, timestampVideoDuration); - - while (hr == S_OK && seekForward && timestampVideo < referenceTime) - { - // our min position was not respected. try again with higher min and target. - min += stream->ConvertDuration(TimeSpan{ 50000000 }); - seekTarget += stream->ConvertDuration(TimeSpan{ 50000000 }); - - if (avformat_seek_file(avFormatCtx, stream->StreamIndex, min, seekTarget, max, 0) < 0) - { - hr = E_FAIL; - DebugMessage(L" - ### Error while seeking\n"); - } - else - { - // Flush all active streams - FlushStreams(); - - // get updated timestamp - hr = currentVideoStream->GetNextPacketTimestamp(timestampVideo, timestampVideoDuration); - } - } - - if (hr == S_OK) - { - actualPosition = timestampVideo; - - // remember last seek direction - isLastSeekForward = seekForward; - lastSeekStart = position; - lastSeekActual = actualPosition; - - if (currentAudioStream) - { - // if we have audio, we need to seek back a bit more to get 100% clean audio - TimeSpan timestampAudio; - TimeSpan timestampAudioDuration; - hr = currentAudioStream->GetNextPacketTimestamp(timestampAudio, timestampAudioDuration); - if (hr == S_OK) - { - // audio stream should start one sample before video - auto audioTarget = timestampVideo - timestampAudioDuration; - auto audioPreroll = timestampAudio - timestampVideo; - if (audioPreroll.Duration > 0 && config->FastSeekCleanAudio) - { - seekTarget = stream->ConvertPosition(audioTarget - audioPreroll); - if (av_seek_frame(avFormatCtx, stream->StreamIndex, seekTarget, AVSEEK_FLAG_BACKWARD | AVSEEK_FLAG_ANY) < 0) - { - hr = E_FAIL; - DebugMessage(L" - ### Error while seeking\n"); - } - else - { - FlushStreams(); - - // Now drop all packets until desired keyframe position - currentVideoStream->SkipPacketsUntilTimestamp(timestampVideo); - currentAudioStream->SkipPacketsUntilTimestamp(audioTarget); - - auto sample = currentAudioStream->GetNextSample(); - if (sample) - { - actualPosition = sample->Timestamp + sample->Duration; - } - } - } - else if (audioPreroll.Duration <= 0) - { - // Negative audio preroll. Just drop all packets until target position. - currentAudioStream->SkipPacketsUntilTimestamp(audioTarget); - - hr = currentAudioStream->GetNextPacketTimestamp(timestampAudio, timestampAudioDuration); - if (hr == S_OK && (config->FastSeekCleanAudio || (timestampAudio + timestampAudioDuration) <= timestampVideo)) - { - // decode one audio sample to get clean output - auto sample = currentAudioStream->GetNextSample(); - if (sample) - { - actualPosition = sample->Timestamp + sample->Duration; - } - } - } - } - } - } - } - } - else - { - if (av_seek_frame(avFormatCtx, stream->StreamIndex, seekTarget, AVSEEK_FLAG_BACKWARD) < 0) - { - hr = E_FAIL; - DebugMessage(L" - ### Error while seeking\n"); - } - else - { - // Flush all active streams - FlushStreams(); - } - } - } - else - { - hr = E_FAIL; - } - - return hr; + auto diffCurrent = position - currentPosition; + auto diffLast = position - lastPosition; + bool isSeekBeforeStreamSwitch = allowFastSeek && config->FastSeekSmartStreamSwitching && !isFirstSeekAfterStreamSwitch && diffCurrent.Duration > 0 && diffCurrent.Duration < 5000000 && diffLast.Duration > 0 && diffLast.Duration < 10000000; + + bool fastSeek = allowFastSeek && config->FastSeek && currentVideoStream && PlaybackSession && !isFirstSeekAfterStreamSwitch; + if (isSeekBeforeStreamSwitch) + { + return S_OK; + } + else + { + return m_pReader->Seek(position, actualPosition, lastPosition, fastSeek, currentVideoStream, currentAudioStream); + } } CoreDispatcher^ FFmpegInteropX::FFmpegMediaSource::GetCurrentDispatcher() @@ -1980,8 +1830,8 @@ CoreDispatcher^ FFmpegInteropX::FFmpegMediaSource::GetCurrentDispatcher() void FFmpegMediaSource::OnPositionChanged(Windows::Media::Playback::MediaPlaybackSession^ sender, Platform::Object^ args) { mutexGuard.lock(); - lastPosition = actualPosition; - actualPosition = sender->Position; + lastPosition = currentPosition; + currentPosition = sender->Position; mutexGuard.unlock(); } @@ -2056,3 +1906,4 @@ static int64_t FileStreamSeek(void* ptr, int64_t pos, int whence) } } + diff --git a/Source/FFmpegMediaSource.h b/Source/FFmpegMediaSource.h index 047e4cb0f9..ef98d8d48d 100644 --- a/Source/FFmpegMediaSource.h +++ b/Source/FFmpegMediaSource.h @@ -1,4 +1,4 @@ -//***************************************************************************** +//***************************************************************************** // // Copyright 2015 Microsoft Corporation // @@ -270,18 +270,6 @@ namespace FFmpegInteropX void InitializePlaybackItem(MediaPlaybackItem^ playbackitem); bool CheckUseHardwareAcceleration(AVCodecContext* avCodecCtx, HardwareAccelerationStatus^ status, HardwareDecoderStatus& hardwareDecoderStatus, int maxProfile, int maxLevel); - void FlushStreams() - { - // Flush all active streams - for each (auto stream in sampleProviders) - { - if (stream && stream->IsEnabled) - { - stream->Flush(); - } - } - } - internal: static FFmpegMediaSource^ CreateFromStream(IRandomAccessStream^ stream, MediaSourceConfig^ config, CoreDispatcher^ dispatcher); static FFmpegMediaSource^ CreateFromUri(String^ uri, MediaSourceConfig^ config, CoreDispatcher^ dispatcher); @@ -343,6 +331,9 @@ namespace FFmpegInteropX CoreDispatcher^ dispatcher; MediaPlaybackSession^ session; EventRegistrationToken sessionPositionEvent; + TimeSpan currentPosition; + TimeSpan lastPosition; + bool isFirstSeekAfterStreamSwitch; String^ videoCodecName; String^ audioCodecName; @@ -357,16 +348,9 @@ namespace FFmpegInteropX HANDLE deviceHandle; IMFDXGIDeviceManager* deviceManager; - bool isFirstSeekAfterStreamSwitch; - bool isLastSeekForward; - TimeSpan lastSeekStart; - TimeSpan lastSeekActual; - - TimeSpan actualPosition; - TimeSpan lastPosition; - static CoreDispatcher^ GetCurrentDispatcher(); void OnPositionChanged(Windows::Media::Playback::MediaPlaybackSession^ sender, Platform::Object^ args); + void OnPaused(Windows::Media::Core::MediaStreamSource^ sender, Platform::Object^ args); }; } diff --git a/Source/FFmpegReader.cpp b/Source/FFmpegReader.cpp index bd8e95a3fa..9d39caca94 100644 --- a/Source/FFmpegReader.cpp +++ b/Source/FFmpegReader.cpp @@ -18,61 +18,501 @@ #include "pch.h" #include "FFmpegReader.h" +#include "UncompressedSampleProvider.h" + +using namespace Concurrency; using namespace FFmpegInteropX; -FFmpegReader::FFmpegReader(AVFormatContext* avFormatCtx, std::vector* initProviders) - : m_pAvFormatCtx(avFormatCtx) +FFmpegReader::FFmpegReader(AVFormatContext* avFormatCtx, std::vector* initProviders, MediaSourceConfig^ config) + : avFormatCtx(avFormatCtx) , sampleProviders(initProviders) + , config(config) { } -FFmpegReader::~FFmpegReader() +void FFmpegReader::Start() { + { + std::lock_guard lock(mutex); + if (!isReading) + { + readTask = create_task([this] () { this->ReadDataLoop(); }); + isReading = true; + } + } } -// Read the next packet from the stream and push it into the appropriate -// sample provider -int FFmpegReader::ReadPacket() +void FFmpegReader::Stop() { - int ret; - AVPacket *avPacket = av_packet_alloc(); - if (!avPacket) - { - return E_OUTOFMEMORY; - } + bool wait = false; + { + std::lock_guard lock(mutex); + if (isReading) + { + isReading = false; + wait = true; + } + } - ret = av_read_frame(m_pAvFormatCtx, avPacket); - if (ret < 0) - { - av_packet_free(&avPacket); - return ret; - } + if (wait) + { + readTask.wait(); + } - if (avPacket->stream_index >= (int)sampleProviders->size()) - { - // new stream detected. if this is a subtitle stream, we could create it now. - av_packet_free(&avPacket); - return ret; - } +} + +void FFmpegReader::FlushCodecs() +{ + std::lock_guard lock(mutex); + for (auto stream : *sampleProviders) + { + stream->Flush(false); + } +} + +void FFmpegReader::Flush() +{ + std::lock_guard lock(mutex); + for (auto stream : *sampleProviders) + { + stream->Flush(true); + } + result = 0; +} + +HRESULT FFmpegReader::Seek(TimeSpan position, TimeSpan& actualPosition, TimeSpan currentPosition, bool fastSeek, MediaSampleProvider^ videoStream, MediaSampleProvider^ audioStream) +{ + auto hr = S_OK; + if (result != 0) + { + fastSeek = false; + result = 0; + } + + // Select the first valid stream either from video or audio + auto stream = videoStream ? videoStream : audioStream; + + if (stream) + { + int64_t seekTarget = stream->ConvertPosition(position); + + TimeSpan videoSkipPts, audioSkipPts; + if (TrySeekBuffered(position, actualPosition, fastSeek, videoStream, audioStream)) + { + // Flush all active stream codecs but keep bufferss + FlushCodecs(); + } + else if (fastSeek) + { + hr = SeekFast(position, actualPosition, currentPosition, videoStream, audioStream); + } + else + { + if (av_seek_frame(avFormatCtx, stream->StreamIndex, seekTarget, AVSEEK_FLAG_BACKWARD) < 0) + { + hr = E_FAIL; + DebugMessage(L" - ### Error while seeking\n"); + } + else + { + // Flush all active streams with buffers + Flush(); + } + } + } + else + { + hr = E_FAIL; + } + + return hr; +} + +HRESULT FFmpegReader::SeekFast(TimeSpan position, TimeSpan& actualPosition, TimeSpan currentPosition, MediaSampleProvider^ videoStream, MediaSampleProvider^ audioStream) +{ + HRESULT hr = S_OK; + int64_t seekTarget = videoStream->ConvertPosition(position); + bool isUriSource = avFormatCtx->url; + + // fast seek + bool seekForward; + TimeSpan referenceTime; + + // decide seek direction + if (isLastSeekForward && position > lastSeekStart && position <= lastSeekActual) + { + seekForward = true; + referenceTime = lastSeekStart + ((position - lastSeekStart) * 0.2); + DebugMessage(L" - ### Forward seeking continue\n"); + } + else if (!isLastSeekForward && position < lastSeekStart && position >= lastSeekActual) + { + seekForward = false; + referenceTime = lastSeekStart + ((position - lastSeekStart) * 0.2); + DebugMessage(L" - ### Backward seeking continue\n"); + } + else if (position >= currentPosition) + { + seekForward = true; + referenceTime = currentPosition + ((position - currentPosition) * 0.2); + DebugMessage(L" - ### Forward seeking\n"); + } + else + { + seekForward = false; + referenceTime = currentPosition + ((position - currentPosition) * 0.2); + DebugMessage(L" - ### Backward seeking\n"); + } + + int64_t min = INT64_MIN; + int64_t max = INT64_MAX; + if (seekForward) + { + min = videoStream->ConvertPosition(referenceTime); + } + else + { + max = videoStream->ConvertPosition(referenceTime); + } + + if (avformat_seek_file(avFormatCtx, videoStream->StreamIndex, min, seekTarget, max, 0) < 0) + { + hr = E_FAIL; + DebugMessage(L" - ### Error while seeking\n"); + } + else + { + // Flush all active streams with buffers + Flush(); + + // get and apply keyframe position for fast seeking + TimeSpan timestampVideo; + TimeSpan timestampVideoDuration; + hr = videoStream->GetNextPacketTimestamp(timestampVideo, timestampVideoDuration); + + if (hr == S_FALSE && isUriSource) + { + if (dynamic_cast(videoStream)) + { + auto sample = videoStream->GetNextSample(); + if (sample) + { + timestampVideo = sample->Timestamp; + timestampVideoDuration = sample->Duration; + timestampVideo += timestampVideoDuration; + hr = S_OK; + } + } + else + { + //hr = audioStream->GetNextPacketTimestamp(timestampVideo, timestampVideoDuration); + } + } + + while (hr == S_OK && seekForward && timestampVideo < referenceTime && !isUriSource) + { + // our min position was not respected. try again with higher min and target. + min += videoStream->ConvertDuration(TimeSpan{ 50000000 }); + seekTarget += videoStream->ConvertDuration(TimeSpan{ 50000000 }); + + if (avformat_seek_file(avFormatCtx, videoStream->StreamIndex, min, seekTarget, max, 0) < 0) + { + hr = E_FAIL; + DebugMessage(L" - ### Error while seeking\n"); + } + else + { + // Flush all active streams with buffers + Flush(); + + // get updated timestamp + hr = videoStream->GetNextPacketTimestamp(timestampVideo, timestampVideoDuration); + } + } + + if (hr == S_OK) + { + // reduce playback delay + timestampVideo += timestampVideoDuration; + + actualPosition = timestampVideo; + + // remember last seek direction + isLastSeekForward = seekForward; + lastSeekStart = position; + lastSeekActual = actualPosition; + + if (audioStream) + { + // if we have audio, we need to seek back a bit more to get 100% clean audio + TimeSpan timestampAudio; + TimeSpan timestampAudioDuration; + hr = audioStream->GetNextPacketTimestamp(timestampAudio, timestampAudioDuration); + if (hr == S_OK) + { + // audio stream should start one sample before video + auto audioTarget = timestampVideo - timestampAudioDuration; + auto audioPreroll = timestampAudio - timestampVideo; + if (audioPreroll.Duration > 0 && config->FastSeekCleanAudio && !isUriSource) + { + seekTarget = videoStream->ConvertPosition(audioTarget - audioPreroll); + if (av_seek_frame(avFormatCtx, videoStream->StreamIndex, seekTarget, AVSEEK_FLAG_BACKWARD | AVSEEK_FLAG_ANY) < 0) + { + hr = E_FAIL; + DebugMessage(L" - ### Error while seeking\n"); + } + else + { + // Flush all active streams with buffers + Flush(); + + // Now drop all packets until desired keyframe position + videoStream->SkipPacketsUntilTimestamp(timestampVideo); + audioStream->SkipPacketsUntilTimestamp(audioTarget); + + auto sample = audioStream->GetNextSample(); + /*if (sample) + { + actualPosition = sample->Timestamp + sample->Duration; + }*/ + } + } + else if (audioPreroll.Duration <= 0) + { + // Negative audio preroll. Just drop all packets until target position. + audioStream->SkipPacketsUntilTimestamp(audioTarget); - if (avPacket->stream_index < 0) + hr = audioStream->GetNextPacketTimestamp(timestampAudio, timestampAudioDuration); + if (hr == S_OK && (config->FastSeekCleanAudio || (timestampAudio + timestampAudioDuration) <= timestampVideo)) + { + // decode one audio sample to get clean output + auto sample = audioStream->GetNextSample(); + /*if (sample) + { + actualPosition = sample->Timestamp + sample->Duration; + }*/ + } + } + } + } + } + } + + return hr; +} + +bool FFmpegReader::TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, bool fastSeek, MediaSampleProvider^ videoStream, MediaSampleProvider^ audioStream) +{ + bool result = true;; + int vIndex; int aIndex; + + TimeSpan targetPosition = position; + + if (videoStream) + { + auto pts = videoStream->ConvertPosition(targetPosition); + vIndex = videoStream->buffer->TryFindPacketIndex(pts, true); + result &= vIndex >= 0; + + if (result && fastSeek) + { + auto targetPacket = videoStream->buffer->PeekPacketIndex(vIndex); + if (targetPacket->pts != AV_NOPTS_VALUE) + { + targetPosition = videoStream->ConvertPosition(targetPacket->pts); + } + } + } + + if (result && audioStream) + { + auto pts = audioStream->ConvertPosition(targetPosition); + aIndex = audioStream->buffer->TryFindPacketIndex(pts, false); + result &= aIndex >= 0; + } + + if (result) + { + // Flush all active streams but keep buffers + FlushCodecs(); + + if (videoStream) + { + videoStream->buffer->DropPackets(vIndex); + } + + if (audioStream) + { + if (config->FastSeekCleanAudio && aIndex > 0) + { + aIndex--; + audioStream->buffer->DropPackets(aIndex); + + // decode one audio sample to get clean output + auto sample = audioStream->GetNextSample(); + if (sample) + { + actualPosition = sample->Timestamp + sample->Duration; + } + } + else + { + audioStream->buffer->DropPackets(aIndex); + } + } + + actualPosition = targetPosition; + } + + return result; +} + +void FFmpegReader::ReadDataLoop() +{ + int ret = 0; + bool sleep = false; + bool force = false; + + while (true) { - av_packet_free(&avPacket); - return E_FAIL; + { + std::lock_guard lock(mutex); + force = forceReadStream >= 0; + sleep = false; + for (auto stream : *sampleProviders) + { + sleep &= stream->buffer->IsFull(stream); + } + if (!isReading) + { + break; + } + } + + if (!sleep || force) + { + ret = ReadPacket(); + if (ret < 0) + { + break; + } + } + else + { + Sleep(100); + } } - MediaSampleProvider^ provider = sampleProviders->at(avPacket->stream_index); - if (provider) { - provider->QueuePacket(avPacket); + std::lock_guard lock(mutex); + isReading = false; + forceReadStream = -1; + result = ret; } - else +} + +FFmpegReader::~FFmpegReader() +{ +} + +// Read the next packet from the stream and push it into the appropriate +// sample provider +int FFmpegReader::ReadPacket() +{ + int ret; + AVPacket* avPacket = av_packet_alloc(); + + if (!avPacket) + { + ret = E_OUTOFMEMORY; + } + + ret = av_read_frame(avFormatCtx, avPacket); + if (ret < 0) + { + std::lock_guard lock(mutex); + av_packet_free(&avPacket); + } + else + { + { + std::lock_guard lock(mutex); + + if (avPacket->stream_index == forceReadStream) + { + forceReadStream = -1; + } + + if (avPacket->stream_index >= (int)sampleProviders->size()) + { + // new stream detected. if this is a subtitle stream, we could create it now. + av_packet_free(&avPacket); + } + + if (avPacket->stream_index < 0) + { + av_packet_free(&avPacket); + } + + MediaSampleProvider^ provider = sampleProviders->at(avPacket->stream_index); + if (provider) + { + provider->QueuePacket(avPacket); + } + else + { + DebugMessage(L"Ignoring unused stream\n"); + av_packet_free(&avPacket); + } + } + } + + result = ret; + return result; +} + + +int FFmpegReader::ReadPacketForStream(StreamBuffer^ buffer) +{ + bool manual; + { + std::lock_guard lock(mutex); + manual = !isReading; + } + + while (true) { - DebugMessage(L"Ignoring unused stream\n"); - av_packet_free(&avPacket); + { + std::lock_guard lock(mutex); + + if (!(buffer->IsEmpty())) + { + forceReadStream = -1; + break; + } + else if (result < 0) + { + break; + } + else + { + forceReadStream = buffer->StreamIndex; + } + } + + //task_completion_event tce; + + if (manual) + { + ReadPacket(); + } + else + { + Sleep(10); + } } - return ret; + return result; } diff --git a/Source/FFmpegReader.h b/Source/FFmpegReader.h index cd196e6681..36c93e1670 100644 --- a/Source/FFmpegReader.h +++ b/Source/FFmpegReader.h @@ -18,22 +18,255 @@ #pragma once +#include +#include + +#include "MediaSourceConfig.h" #include "MediaSampleProvider.h" namespace FFmpegInteropX { + using namespace Concurrency; ref class FFmpegReader { - public: - virtual ~FFmpegReader(); - int ReadPacket(); + internal: + FFmpegReader(AVFormatContext* avFormatCtx, std::vector* sampleProviders, MediaSourceConfig^ config); + + int ReadPacket(); + int ReadPacketForStream(StreamBuffer^ buffer); + void Start(); + void Stop(); + void Flush(); + HRESULT Seek(TimeSpan position, TimeSpan& actualPosition, TimeSpan currentPosition, bool allowFastSeek, MediaSampleProvider^ videoStream, MediaSampleProvider^ audioStream); - internal: - FFmpegReader(AVFormatContext* avFormatCtx, std::vector* sampleProviders); private: - AVFormatContext* m_pAvFormatCtx; + + ~FFmpegReader(); + bool TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, bool fastSeek, MediaSampleProvider^ videoStream, MediaSampleProvider^ audioStream); + HRESULT SeekFast(TimeSpan position, TimeSpan& actualPosition, TimeSpan currentPosition, MediaSampleProvider^ videoStream, MediaSampleProvider^ audioStream); + void ReadDataLoop(); + void FlushCodecs(); + + AVFormatContext* avFormatCtx; std::vector* sampleProviders; - }; + MediaSourceConfig^ config; + + std::mutex mutex; + bool isReading; + int forceReadStream; + int result; + task readTask; + + bool isFirstSeekAfterStreamSwitch; + bool isLastSeekForward; + TimeSpan lastSeekStart; + TimeSpan lastSeekActual; + }; + + ref class StreamBuffer + { + internal: + StreamBuffer(int streamIndex, MediaSourceConfig^ config) + : config(config) + { + StreamIndex = streamIndex; + + } + + property int StreamIndex; + + void QueuePacket(AVPacket* packet) + { + std::lock_guard lock(mutex); + buffer.push_back(packet); + bufferSize += packet->size; + } + + bool ReadUntilNotEmpty(FFmpegReader^ reader) + { + while (IsEmpty()) + { + if (reader->ReadPacketForStream(this) < 0) + { + DebugMessage(L"GetNextPacket reaching EOF\n"); + break; + } + } + return !IsEmpty(); + } + + bool SkipUntilTimestamp(FFmpegReader^ reader, LONGLONG target) + { + bool foundPacket = false; + + while (!foundPacket) + { + if (ReadUntilNotEmpty(reader)) + { + // peek next packet and check pts value + auto packet = PeekPacket(); + + auto pts = packet->pts != AV_NOPTS_VALUE ? packet->pts : packet->dts; + if (pts != AV_NOPTS_VALUE && packet->duration != AV_NOPTS_VALUE) + { + auto packetEnd = pts + packet->duration; + if (packet->duration > 0 ? packetEnd <= target : packetEnd < target) + { + DropPackets(1); + } + else + { + foundPacket = true; + break; + } + } + else + { + break; + } + } + else + { + // no more packet found + break; + } + } + + return foundPacket; + } + + bool IsEmpty() + { + std::lock_guard lock(mutex); + return buffer.empty(); + } + + bool IsFull(MediaSampleProvider^ sampleProvider) + { + std::lock_guard lock(mutex); + auto maxSize = config->ReadAheadBufferSize; + auto maxDuration = config->ReadAheadBufferDuration; + + bool full = maxSize >= 0 && (long long)bufferSize > maxSize; + if (!full && maxDuration.Duration >= 0 && buffer.size() > 1) + { + auto firstPacket = buffer.front(); + auto lastPacket = buffer.back(); + auto firstPts = firstPacket->pts != AV_NOPTS_VALUE ? firstPacket->pts : firstPacket->dts; + auto lastPts = lastPacket->pts != AV_NOPTS_VALUE ? lastPacket->pts : lastPacket->dts; + + if (firstPts != AV_NOPTS_VALUE && lastPts != AV_NOPTS_VALUE) + { + auto duration = sampleProvider->ConvertDuration(lastPts - firstPts); + full = duration > maxDuration; + } + } + return full; + } + + AVPacket* PopPacket() + { + std::lock_guard lock(mutex); + + AVPacket* packet = NULL; + if (!buffer.empty()) + { + packet = buffer.front(); + buffer.pop_front(); + bufferSize -= packet->size; + } + + return packet; + } + + AVPacket* PeekPacket() + { + std::lock_guard lock(mutex); + + AVPacket* packet = NULL; + if (!buffer.empty()) + { + packet = buffer.front(); + } + + return packet; + } + + AVPacket* PeekPacketIndex(int index) + { + std::lock_guard lock(mutex); + return buffer.at(index); + } + + int TryFindPacketIndex(LONGLONG pts, bool requireKeyFrame) + { + std::lock_guard lock(mutex); + + bool hasEnd = false; + int index = 0; + int result = -1; + for (auto packet : buffer) + { + if (packet->pts <= pts) + { + if (!requireKeyFrame || packet->flags & AV_PKT_FLAG_KEY) + { + result = index; + if (packet->pts + packet->duration >= pts) + { + hasEnd = true; + break; + } + } + } + else + { + hasEnd = true; + break; + } + index++; + } + + if (result >= 0 && !hasEnd) + { + result = -1; + } + + return result; + } + + void Flush() + { + std::lock_guard lock(mutex); + while (!buffer.empty()) + { + auto packet = buffer.front(); + bufferSize -= packet->size; + buffer.pop_front(); + + av_packet_free(&packet); + } + } + + void DropPackets(int count) + { + std::lock_guard lock(mutex); + for (int i = 0; i < count; i++) + { + auto packet = buffer.front(); + bufferSize -= packet->size; + buffer.pop_front(); + + av_packet_free(&packet); + } + } + + private: + std::deque buffer; + std::mutex mutex; + size_t bufferSize; + MediaSourceConfig^ config; + }; } diff --git a/Source/MediaSampleProvider.cpp b/Source/MediaSampleProvider.cpp index a697a02e45..9bcc6af581 100644 --- a/Source/MediaSampleProvider.cpp +++ b/Source/MediaSampleProvider.cpp @@ -40,6 +40,7 @@ MediaSampleProvider::MediaSampleProvider( , m_config(config) , m_streamIndex(streamIndex) , hardwareDecoderStatus(hardwareDecoderStatus) + , buffer(ref new StreamBuffer(streamIndex, config)) { DebugMessage(L"MediaSampleProvider\n"); @@ -265,43 +266,33 @@ HRESULT MediaSampleProvider::GetNextPacket(AVPacket** avPacket, LONGLONG & packe { HRESULT hr = S_OK; - // Continue reading until there is an appropriate packet in the stream - while (m_packetQueue.empty()) - { - if (m_pReader->ReadPacket() < 0) - { - DebugMessage(L"GetNextPacket reaching EOF\n"); - break; - } - } - - if (!m_packetQueue.empty()) - { - // read next packet and set pts values - auto packet = PopPacket(); - *avPacket = packet; - - packetDuration = packet->duration; - - if (packet->pts != AV_NOPTS_VALUE) - { - packetPts = packet->pts; - // Set the PTS for the next sample if it doesn't one. - m_nextPacketPts = packetPts + packetDuration; - } - else if (m_isDiscontinuous && packet->dts != AV_NOPTS_VALUE) - { - packetPts = packet->dts; - // Use DTS instead of PTS after a seek, if PTS is not available (e.g. some WMV files) - m_nextPacketPts = packetPts + packetDuration; - } - else - { - packetPts = m_nextPacketPts; - // Set the PTS for the next sample if it doesn't one. - m_nextPacketPts += packetDuration; - } - } + if (buffer->ReadUntilNotEmpty(m_pReader)) + { + auto packet = buffer->PopPacket(); + + *avPacket = packet; + + packetDuration = packet->duration; + + if (packet->pts != AV_NOPTS_VALUE) + { + packetPts = packet->pts; + // Set the PTS for the next sample if it doesn't one. + m_nextPacketPts = packetPts + packetDuration; + } + else if (m_isDiscontinuous && packet->dts != AV_NOPTS_VALUE) + { + packetPts = packet->dts; + // Use DTS instead of PTS after a seek, if PTS is not available (e.g. some WMV files) + m_nextPacketPts = packetPts + packetDuration; + } + else + { + packetPts = m_nextPacketPts; + // Set the PTS for the next sample if it doesn't one. + m_nextPacketPts += packetDuration; + } + } else { hr = S_FALSE; @@ -314,28 +305,18 @@ HRESULT MediaSampleProvider::GetNextPacketTimestamp(TimeSpan& timestamp, TimeSpa { HRESULT hr = S_FALSE; - // Continue reading until there is an appropriate packet in the stream - while (m_packetQueue.empty()) - { - if (m_pReader->ReadPacket() < 0) - { - DebugMessage(L"GetNextPacketTimestamp reaching EOF\n"); - break; - } - } - - if (!m_packetQueue.empty()) - { - // peek next packet and set pts value - auto packet = m_packetQueue.front(); - auto pts = packet->pts != AV_NOPTS_VALUE ? packet->pts : packet->dts; - if (pts != AV_NOPTS_VALUE) - { - timestamp = ConvertPosition(pts); - packetDuration = ConvertDuration(packet->duration); - hr = S_OK; - } - } + if (buffer->ReadUntilNotEmpty(m_pReader)) + { + // peek next packet and set pts value + auto packet = buffer->PeekPacket(); + auto pts = packet->pts != AV_NOPTS_VALUE ? packet->pts : packet->dts; + if (pts != AV_NOPTS_VALUE) + { + timestamp = ConvertPosition(pts); + packetDuration = ConvertDuration(packet->duration); + hr = S_OK; + } + } return hr; } @@ -343,52 +324,11 @@ HRESULT MediaSampleProvider::GetNextPacketTimestamp(TimeSpan& timestamp, TimeSpa HRESULT MediaSampleProvider::SkipPacketsUntilTimestamp(TimeSpan timestamp) { HRESULT hr = S_OK; - bool foundPacket = false; - - while (hr == S_OK && !foundPacket) - { - // Continue reading until there is an appropriate packet in the stream - while (m_packetQueue.empty()) - { - if (m_pReader->ReadPacket() < 0) - { - DebugMessage(L"SkipPacketsUntilTimestamp reaching EOF\n"); - break; - } - } - if (!m_packetQueue.empty()) - { - // peek next packet and check pts value - auto packet = m_packetQueue.front(); - - auto pts = packet->pts != AV_NOPTS_VALUE ? packet->pts : packet->dts; - if (pts != AV_NOPTS_VALUE && packet->duration != AV_NOPTS_VALUE) - { - auto packetEnd = ConvertPosition(pts + packet->duration); - if (packet->duration > 0 ? packetEnd <= timestamp : packetEnd < timestamp) - { - m_packetQueue.pop(); - av_packet_free(&packet); - } - else - { - foundPacket = true; - break; - } - } - else - { - hr = S_FALSE; - break; - } - } - else - { - // no more packet found - hr = S_FALSE; - } - } + if (!buffer->SkipUntilTimestamp(m_pReader, ConvertPosition(timestamp))) + { + hr = S_FALSE; + } return hr; } @@ -399,7 +339,7 @@ void MediaSampleProvider::QueuePacket(AVPacket *packet) if (m_isEnabled) { - m_packetQueue.push(packet); + buffer->QueuePacket(packet); } else { @@ -407,32 +347,17 @@ void MediaSampleProvider::QueuePacket(AVPacket *packet) } } -AVPacket* MediaSampleProvider::PopPacket() -{ - DebugMessage(L" - PopPacket\n"); - AVPacket* result = NULL; - - if (!m_packetQueue.empty()) - { - result = m_packetQueue.front(); - m_packetQueue.pop(); - } - - return result; -} - -void MediaSampleProvider::Flush() +void MediaSampleProvider::Flush(bool flushBuffers) { DebugMessage(L"Flush\n"); - while (!m_packetQueue.empty()) - { - AVPacket *avPacket = PopPacket(); - av_packet_free(&avPacket); - } if (m_pAvCodecCtx) { avcodec_flush_buffers(m_pAvCodecCtx); } + if (flushBuffers) + { + buffer->Flush(); + } m_isDiscontinuous = true; IsCleanSample = false; } @@ -447,7 +372,6 @@ void MediaSampleProvider::EnableStream() void MediaSampleProvider::DisableStream() { DebugMessage(L"DisableStream\n"); - Flush(); m_isEnabled = false; m_pAvStream->discard = AVDISCARD_ALL; } @@ -510,7 +434,7 @@ void MediaSampleProvider::SetCommonVideoEncodingProperties(VideoEncodingProperti void MediaSampleProvider::Detach() { - Flush(); + Flush(false); m_pReader = nullptr; avcodec_close(m_pAvCodecCtx); avcodec_free_context(&m_pAvCodecCtx); @@ -520,4 +444,4 @@ void free_buffer(void *lpVoid) { auto buffer = (AVBufferRef *)lpVoid; av_buffer_unref(&buffer); -} \ No newline at end of file +} diff --git a/Source/MediaSampleProvider.h b/Source/MediaSampleProvider.h index e445c234f6..774d17d34c 100644 --- a/Source/MediaSampleProvider.h +++ b/Source/MediaSampleProvider.h @@ -29,14 +29,15 @@ namespace FFmpegInteropX using namespace Windows::Media::MediaProperties; using namespace Windows::Graphics::DirectX::Direct3D11; - ref class FFmpegReader; + ref class FFmpegReader; + ref class StreamBuffer; ref class MediaSampleProvider abstract { public: virtual ~MediaSampleProvider(); virtual MediaStreamSample^ GetNextSample(); - virtual void Flush(); + virtual void Flush(bool flushBuffers); property IMediaStreamDescriptor^ StreamDescriptor { @@ -105,15 +106,14 @@ namespace FFmpegInteropX void InitializeNameLanguageCodec(); virtual void InitializeStreamInfo(); virtual void QueuePacket(AVPacket* packet); - AVPacket* PopPacket(); HRESULT GetNextPacket(AVPacket** avPacket, LONGLONG& packetPts, LONGLONG& packetDuration); virtual HRESULT CreateNextSampleBuffer(IBuffer^* pBuffer, int64_t& samplePts, int64_t& sampleDuration, IDirect3DSurface^* surface) = 0; HRESULT GetNextPacketTimestamp(TimeSpan& timestamp, TimeSpan& packetDuration); HRESULT SkipPacketsUntilTimestamp(TimeSpan timestamp); virtual IMediaStreamDescriptor^ CreateStreamDescriptor() = 0; virtual HRESULT SetSampleProperties(MediaStreamSample^ sample) { return S_OK; }; // can be overridded for setting extended properties - void EnableStream(); - void DisableStream(); + virtual void EnableStream(); + virtual void DisableStream(); virtual void SetFilters(String^ filterDefinition) { };// override for setting effects in sample providers virtual void DisableFilters() {};//override for disabling filters in sample providers; virtual void SetCommonVideoEncodingProperties(VideoEncodingProperties^ videoEncodingProperties, bool isCompressedFormat); @@ -168,9 +168,8 @@ namespace FFmpegInteropX HardwareDecoderStatus hardwareDecoderStatus); private: - std::queue m_packetQueue; - int64 m_nextPacketPts; - IMediaStreamDescriptor^ m_streamDescriptor; + int64 m_nextPacketPts; + IMediaStreamDescriptor^ m_streamDescriptor; HardwareDecoderStatus hardwareDecoderStatus; internal: @@ -179,7 +178,8 @@ namespace FFmpegInteropX // externally MediaSourceConfig^ m_config; FFmpegReader^ m_pReader; - AVFormatContext* m_pAvFormatCtx; + StreamBuffer^ buffer; + AVFormatContext* m_pAvFormatCtx; AVCodecContext* m_pAvCodecCtx; AVStream* m_pAvStream; IStreamInfo^ streamInfo; diff --git a/Source/MediaSourceConfig.h b/Source/MediaSourceConfig.h index 0c448c7448..756dc8be07 100644 --- a/Source/MediaSourceConfig.h +++ b/Source/MediaSourceConfig.h @@ -47,8 +47,8 @@ namespace FFmpegInteropX FFmpegOptions = ref new PropertySet(); - DefaultBufferTime = TimeSpan{ 0 }; - DefaultBufferTimeUri = TimeSpan{ 50000000 }; + ReadAheadBufferSize = 100 * 1024 * 1024; + ReadAheadBufferDuration = TimeSpan { 600000000 }; AutoSelectForcedSubtitles = true; OverrideSubtitleStyles = false; @@ -185,12 +185,23 @@ namespace FFmpegInteropX ///The default BufferTime that gets assigned to the MediaStreamSource for IRandomAccessStream sources. - ///A value of 0 is recommended for local files, to avoid framework bugs and unneccessary memory consumption. - property TimeSpan DefaultBufferTime; + ///Deprecated due to framework bugs and memory consumption. Use ReadAheadBufferSize and ReadAheadBufferDuration instead. + [WFM::Deprecated("Use ReadAheadBufferSize and ReadAheadBufferDuration.", WFM::DeprecationType::Deprecate, 0x0)] + property TimeSpan DefaultBufferTime; ///The default BufferTime that gets assigned to the MediaStreamSource for URI sources. - ///Default is 5 seconds. You might want to use higher values, especially for DASH stream sources. - property TimeSpan DefaultBufferTimeUri; + ///Deprecated due to framework bugs and memory consumption. Use ReadAheadBufferSize and ReadAheadBufferDuration instead. + [WFM::Deprecated("Use ReadAheadBufferSize and ReadAheadBufferDuration.", WFM::DeprecationType::Deprecate, 0x0)] + property TimeSpan DefaultBufferTimeUri; + + + ///The maximum number of bytes to buffer ahead per stream. + ///This value can be changed any time during playback. + property long long ReadAheadBufferSize; + + ///The maximum duration to buffer ahead per stream. + ///This value can be changed any time during playback. + property TimeSpan ReadAheadBufferDuration; ///Automatically select subtitles when they have the 'forced' flag set. @@ -293,4 +304,4 @@ namespace FFmpegInteropX }; -} \ No newline at end of file +} diff --git a/Source/NALPacketSampleProvider.cpp b/Source/NALPacketSampleProvider.cpp index 247979016d..915a492c20 100644 --- a/Source/NALPacketSampleProvider.cpp +++ b/Source/NALPacketSampleProvider.cpp @@ -37,9 +37,9 @@ NALPacketSampleProvider::~NALPacketSampleProvider() { } -void NALPacketSampleProvider::Flush() +void NALPacketSampleProvider::Flush(bool flushBuffers) { - CompressedSampleProvider::Flush(); + CompressedSampleProvider::Flush(flushBuffers); m_bHasSentExtradata = false; } @@ -127,4 +127,4 @@ HRESULT NALPacketSampleProvider::WriteNALPacket(AVPacket* avPacket, IBuffer^* pB { // Write out the NAL packet as-is return CompressedSampleProvider::CreateBufferFromPacket(avPacket, pBuffer); -} \ No newline at end of file +} diff --git a/Source/NALPacketSampleProvider.h b/Source/NALPacketSampleProvider.h index 35f1717715..37b1b92cc6 100644 --- a/Source/NALPacketSampleProvider.h +++ b/Source/NALPacketSampleProvider.h @@ -26,7 +26,7 @@ namespace FFmpegInteropX { public: virtual ~NALPacketSampleProvider(); - virtual void Flush() override; + virtual void Flush(bool flushBuffers) override; internal: NALPacketSampleProvider( diff --git a/Source/StringUtils.h b/Source/StringUtils.h index 07267c30d4..6070018095 100644 --- a/Source/StringUtils.h +++ b/Source/StringUtils.h @@ -22,10 +22,11 @@ namespace FFmpegInteropX { if (!char_array) return std::wstring(L""); - std::string s_str = std::string(char_array); - std::wstring_convert> myconv; - std::wstring wid_str = myconv.from_bytes(s_str); - return wid_str; + auto required_size = MultiByteToWideChar(CP_UTF8, 0, char_array, -1, NULL, 0); + wchar_t* buffer = (wchar_t*)calloc(required_size, sizeof(wchar_t)); + auto result = MultiByteToWideChar(CP_UTF8, 0, char_array, -1, buffer, required_size); + std::wstring wid_str = std::wstring(buffer); + return wid_str; } static String^ Utf8ToPlatformString(const char* char_array) @@ -39,13 +40,14 @@ namespace FFmpegInteropX return ref new Platform::String(input.c_str(), (unsigned int)input.length()); } - static std::string PlatformStringToUtf8String(String^ value) { - std::wstring strW(value->Begin(), value->Length()); - std::wstring_convert> myconv; - return myconv.to_bytes(strW); + size_t required_size = WideCharToMultiByte(CP_UTF8, 0, value->Data(), -1, NULL, 0, NULL, NULL); + char* buffer = (char*)calloc(required_size, sizeof(char)); + auto result = WideCharToMultiByte(CP_UTF8, 0, value->Data(), -1, buffer, required_size, NULL, NULL); + std::string s_str = std::string(buffer); + return s_str; } }; -} \ No newline at end of file +} diff --git a/Source/SubtitleProvider.h b/Source/SubtitleProvider.h index f04e238123..97c92d5dbc 100644 --- a/Source/SubtitleProvider.h +++ b/Source/SubtitleProvider.h @@ -48,16 +48,13 @@ namespace FFmpegInteropX if (!m_config->IsExternalSubtitleParser) { - if (Windows::Foundation::Metadata::ApiInformation::IsEnumNamedValuePresent("Windows.Media.Core.TimedMetadataKind", "ImageSubtitle") && - timedMetadataKind == TimedMetadataKind::ImageSubtitle) - { - SubtitleTrack->CueEntered += ref new Windows::Foundation::TypedEventHandler(this, &FFmpegInteropX::SubtitleProvider::OnCueEntered); - } SubtitleTrack->TrackFailed += ref new Windows::Foundation::TypedEventHandler(this, &FFmpegInteropX::SubtitleProvider::OnTrackFailed); } InitializeStreamInfo(); + m_pAvStream->discard = AVDISCARD_DEFAULT; + return S_OK; } @@ -70,99 +67,96 @@ namespace FFmpegInteropX virtual void QueuePacket(AVPacket *packet) override { - if (m_isEnabled) - { - try - { - TimeSpan position = ConvertPosition(packet->pts); - TimeSpan duration = ConvertDuration(packet->duration); - bool isDurationFixed = false; - - auto cue = CreateCue(packet, &position, &duration); - if (cue && position.Duration >= 0) - { - // apply subtitle delay - position += SubtitleDelay; - if (position.Duration < 0) - { - negativePositionCues.emplace_back(cue, position.Duration); - position.Duration = 0; - } - - // clip previous extended duration cue, if there is one - if (lastExtendedDurationCue && m_config->PreventModifiedSubtitleDurationOverlap && - lastExtendedDurationCue->StartTime + lastExtendedDurationCue->Duration > position) - { - auto diff = position - (lastExtendedDurationCue->StartTime + lastExtendedDurationCue->Duration); - auto newDuration = lastExtendedDurationCue->Duration + diff; - if (newDuration.Duration > 0) - { - lastExtendedDurationCue->Duration = newDuration; - if (!m_config->IsExternalSubtitleParser) - { - pendingChangedDurationCues.push_back(lastExtendedDurationCue); - } - } - else - { - // weird subtitle timings, just leave it as is - } - } - - lastExtendedDurationCue = nullptr; - - if (duration.Duration < 0) - { - duration.Duration = InfiniteDuration; - } - else - { - if (m_config->AdditionalSubtitleDuration.Duration != 0) - { - duration.Duration += m_config->AdditionalSubtitleDuration.Duration; - lastExtendedDurationCue = cue; - } - if (duration.Duration < m_config->MinimumSubtitleDuration.Duration) - { - duration.Duration = m_config->MinimumSubtitleDuration.Duration; - lastExtendedDurationCue = cue; - } - } - - cue->StartTime = position; - cue->Duration = duration; - AddCue(cue); - - if (!m_config->IsExternalSubtitleParser) - { - isPreviousCueInfiniteDuration = duration.Duration >= InfiniteDuration; - } - else - { - // fixup infinite duration cues for external subs - if (isPreviousCueInfiniteDuration) - { - infiniteDurationCue->Duration = cue->StartTime - infiniteDurationCue->StartTime; - } - - if (duration.Duration >= InfiniteDuration) - { - isPreviousCueInfiniteDuration = true; - infiniteDurationCue = cue; - } - else - { - isPreviousCueInfiniteDuration = false; - infiniteDurationCue = nullptr; - } - } - } - } - catch (...) - { - OutputDebugString(L"Failed to create subtitle cue."); - } - } + try + { + TimeSpan position = ConvertPosition(packet->pts); + TimeSpan duration = ConvertDuration(packet->duration); + bool isDurationFixed = false; + + auto cue = CreateCue(packet, &position, &duration); + if (cue && position.Duration >= 0) + { + // apply subtitle delay + position += SubtitleDelay; + if (position.Duration < 0) + { + negativePositionCues.emplace_back(cue, position.Duration); + position.Duration = 0; + } + + // clip previous extended duration cue, if there is one + if (lastExtendedDurationCue && m_config->PreventModifiedSubtitleDurationOverlap && + lastExtendedDurationCue->StartTime + lastExtendedDurationCue->Duration > position) + { + auto diff = position - (lastExtendedDurationCue->StartTime + lastExtendedDurationCue->Duration); + auto newDuration = lastExtendedDurationCue->Duration + diff; + if (newDuration.Duration > 0) + { + lastExtendedDurationCue->Duration = newDuration; + if (!m_config->IsExternalSubtitleParser) + { + pendingChangedDurationCues.push_back(lastExtendedDurationCue); + } + } + else + { + // weird subtitle timings, just leave it as is + } + } + + lastExtendedDurationCue = nullptr; + + if (duration.Duration < 0) + { + duration.Duration = InfiniteDuration; + } + else + { + if (m_config->AdditionalSubtitleDuration.Duration != 0) + { + duration.Duration += m_config->AdditionalSubtitleDuration.Duration; + lastExtendedDurationCue = cue; + } + if (duration.Duration < m_config->MinimumSubtitleDuration.Duration) + { + duration.Duration = m_config->MinimumSubtitleDuration.Duration; + lastExtendedDurationCue = cue; + } + } + + cue->StartTime = position; + cue->Duration = duration; + AddCue(cue); + + if (!m_config->IsExternalSubtitleParser) + { + isPreviousCueInfiniteDuration = duration.Duration >= InfiniteDuration; + } + else + { + // fixup infinite duration cues for external subs + if (isPreviousCueInfiniteDuration) + { + infiniteDurationCue->Duration = cue->StartTime - infiniteDurationCue->StartTime; + } + + if (duration.Duration >= InfiniteDuration) + { + isPreviousCueInfiniteDuration = true; + infiniteDurationCue = cue; + } + else + { + isPreviousCueInfiniteDuration = false; + infiniteDurationCue = nullptr; + } + } + } + } + catch (...) + { + OutputDebugString(L"Failed to create subtitle cue."); + } av_packet_free(&packet); } @@ -264,7 +258,7 @@ namespace FFmpegInteropX void DispatchCueToTrack(IMediaCue^ cue) { - if (m_config->IsExternalSubtitleParser) + if (m_config->IsExternalSubtitleParser || !IsEnabled) { SubtitleTrack->AddCue(cue); } @@ -299,33 +293,6 @@ namespace FFmpegInteropX mutex.unlock(); } - void OnCueEntered(Windows::Media::Core::TimedMetadataTrack ^sender, Windows::Media::Core::MediaCueEventArgs ^args) - { - mutex.lock(); - try - { - //cleanup old cues to free memory - std::vector remove; - for each (auto cue in SubtitleTrack->Cues) - { - if (cue->StartTime + cue->Duration < args->Cue->StartTime) - { - remove.push_back(cue); - } - } - - for each (auto cue in remove) - { - SubtitleTrack->RemoveCue(cue); - } - } - catch (...) - { - OutputDebugString(L"Failed to cleanup old cues."); - } - mutex.unlock(); - } - void StartTimer() { if (dispatcher != nullptr && IsEnabled) @@ -485,11 +452,24 @@ namespace FFmpegInteropX public: - void Flush() override + void EnableStream() override + { + DebugMessage(L"EnableStream\n"); + m_isEnabled = true; + } + + void DisableStream() override + { + DebugMessage(L"DisableStream\n"); + m_isEnabled = false; + } + + void Flush(bool flushBuffers) override { - if (!m_config->IsExternalSubtitleParser) + CompressedSampleProvider::Flush(flushBuffers); + + if (!m_config->IsExternalSubtitleParser && flushBuffers) { - CompressedSampleProvider::Flush(); mutex.lock(); @@ -521,10 +501,11 @@ namespace FFmpegInteropX } } - private: - + internal: std::recursive_mutex mutex; - int cueCount; + + private: + int cueCount; std::vector pendingCues; std::vector pendingRefCues; std::vector pendingChangedDurationCues; diff --git a/Source/SubtitleProviderBitmap.h b/Source/SubtitleProviderBitmap.h index 8327a5d2b7..8f33fbec71 100644 --- a/Source/SubtitleProviderBitmap.h +++ b/Source/SubtitleProviderBitmap.h @@ -19,6 +19,19 @@ namespace FFmpegInteropX { } + virtual HRESULT Initialize() override + { + auto hr = SubtitleProvider::Initialize(); + + if (SUCCEEDED(hr)) + { + SubtitleTrack->CueEntered += ref new Windows::Foundation::TypedEventHandler(this, &FFmpegInteropX::SubtitleProviderBitmap::OnCueEntered); + SubtitleTrack->CueExited += ref new Windows::Foundation::TypedEventHandler(this, &FFmpegInteropX::SubtitleProviderBitmap::OnCueExited); + } + + return S_OK; + } + virtual void NotifyVideoFrameSize(int width, int height, double aspectRatio) override { videoWidth = width; @@ -35,113 +48,203 @@ namespace FFmpegInteropX virtual IMediaCue^ CreateCue(AVPacket* packet, TimeSpan* position, TimeSpan *duration) override { - // only decode image subtitles if the stream is selected - if (!IsEnabled) - { - return nullptr; - } + AVSubtitle* subtitle = (AVSubtitle*)malloc(sizeof(AVSubtitle)); + if (!subtitle) + { + return nullptr; + } - AVSubtitle subtitle; int gotSubtitle = 0; - auto result = avcodec_decode_subtitle2(m_pAvCodecCtx, &subtitle, &gotSubtitle, packet); + auto result = avcodec_decode_subtitle2(m_pAvCodecCtx, subtitle, &gotSubtitle, packet); if (result > 0 && gotSubtitle) { - if (subtitle.start_display_time > 0) + if (subtitle->start_display_time > 0) { - position->Duration += (long long)10000 * subtitle.start_display_time; + position->Duration += (long long)10000 * subtitle->start_display_time; } - duration->Duration = (long long)10000 * subtitle.end_display_time; + duration->Duration = (long long)10000 * subtitle->end_display_time; - if (subtitle.num_rects <= 0) + if (subtitle->num_rects <= 0) { - if (!dummyBitmap) - { - dummyBitmap = ref new SoftwareBitmap(BitmapPixelFormat::Bgra8, 16, 16, BitmapAlphaMode::Premultiplied); - } - // inserty dummy cue ImageCue^ cue = ref new ImageCue(); - cue->SoftwareBitmap = dummyBitmap; - avsubtitle_free(&subtitle); + cue->SoftwareBitmap = GetDummyBitmap(); + avsubtitle_free(subtitle); + delete subtitle; return cue; } - - int width, height, offsetX, offsetY; - TimedTextSize cueSize; - TimedTextPoint cuePosition; - if (subtitle.num_rects > 0 && CheckSize(subtitle, width, height, offsetX, offsetY, cueSize, cuePosition)) - { - using namespace Windows::Graphics::Imaging; - - auto bitmap = ref new SoftwareBitmap(BitmapPixelFormat::Bgra8, width, height, BitmapAlphaMode::Straight); - { - auto buffer = bitmap->LockBuffer(BitmapBufferAccessMode::Write); - auto reference = buffer->CreateReference(); - - // Query the IBufferByteAccess interface. - Microsoft::WRL::ComPtr bufferByteAccess; - reinterpret_cast(reference)->QueryInterface(IID_PPV_ARGS(&bufferByteAccess)); - - // Retrieve the buffer data. - byte* pixels = nullptr; - unsigned int capacity; - bufferByteAccess->GetBuffer(&pixels, &capacity); - - auto plane = buffer->GetPlaneDescription(0); - - for (unsigned int i = 0; i < subtitle.num_rects; i++) - { - auto rect = subtitle.rects[i]; - - for (int y = 0; y < rect->h; y++) - { - for (int x = 0; x < rect->w; x++) - { - auto inPointer = rect->data[0] + y * rect->linesize[0] + x; - auto color = inPointer[0]; - if (color < rect->nb_colors) - { - auto rgba = ((uint32*)rect->data[1])[color]; - auto outPointer = pixels + plane.StartIndex + plane.Stride * ((y + rect->y) - offsetY) + 4 * ((x + rect->x) - offsetX); - ((uint32*)outPointer)[0] = rgba; - } - else - { - OutputDebugString(L"Error: Illegal subtitle color."); - } - } - } - } - } - - ImageCue^ cue = ref new ImageCue(); - cue->SoftwareBitmap = SoftwareBitmap::Convert(bitmap, BitmapPixelFormat::Bgra8, BitmapAlphaMode::Premultiplied); - cue->Position = cuePosition; - cue->Extent = cueSize; - - avsubtitle_free(&subtitle); - - return cue; - } - else if (subtitle.num_rects > 0) - { - OutputDebugString(L"Error: Invalid subtitle size received."); - } - - avsubtitle_free(&subtitle); - } + else + { + int width, height, offsetX, offsetY; + TimedTextSize cueSize; + TimedTextPoint cuePosition; + if (subtitle->num_rects > 0 && CheckSize(subtitle, width, height, offsetX, offsetY, cueSize, cuePosition)) + { + auto id = nextId++.ToString(); + map[id] = subtitle; + + ImageCue^ cue = ref new ImageCue(); + cue->Id = id; + return cue; + } + else if (subtitle->num_rects > 0) + { + avsubtitle_free(subtitle); + delete subtitle; + OutputDebugString(L"Error: Invalid subtitle size received."); + } + } + } else if (result <= 0) { + avsubtitle_free(subtitle); + delete subtitle; OutputDebugString(L"Failed to decode subtitle."); } return nullptr; } + public: + + void Flush(bool flushBuffers) override + { + SubtitleProvider::Flush(flushBuffers); + + if (!m_config->IsExternalSubtitleParser && flushBuffers) + { + for (auto entry : map) + { + auto subtitle = entry.second; + avsubtitle_free(subtitle); + delete subtitle; + } + map.clear(); + } + } + private: - bool CheckSize(AVSubtitle& subtitle, int& width, int& height, int& offsetX, int& offsetY, TimedTextSize& cueSize, TimedTextPoint& cuePosition) + void OnCueEntered(Windows::Media::Core::TimedMetadataTrack^ sender, Windows::Media::Core::MediaCueEventArgs^ args) + { + mutex.lock(); + try + { + //cleanup old cues to free memory + std::vector remove; + for each (auto cue in SubtitleTrack->Cues) + { + if (cue->StartTime + cue->Duration < args->Cue->StartTime) + { + remove.push_back(cue); + } + } + + for each (auto cue in remove) + { + SubtitleTrack->RemoveCue(cue); + } + + auto cue = dynamic_cast(args->Cue); + if (cue && cue->Id && !cue->SoftwareBitmap) + { + auto subtitle = map.at(cue->Id); + + int width, height, offsetX, offsetY; + TimedTextSize cueSize; + TimedTextPoint cuePosition; + if (subtitle->num_rects > 0 && CheckSize(subtitle, width, height, offsetX, offsetY, cueSize, cuePosition)) + { + using namespace Windows::Graphics::Imaging; + + auto bitmap = ref new SoftwareBitmap(BitmapPixelFormat::Bgra8, width, height, BitmapAlphaMode::Straight); + { + auto buffer = bitmap->LockBuffer(BitmapBufferAccessMode::Write); + auto reference = buffer->CreateReference(); + + // Query the IBufferByteAccess interface. + Microsoft::WRL::ComPtr bufferByteAccess; + reinterpret_cast(reference)->QueryInterface(IID_PPV_ARGS(&bufferByteAccess)); + + // Retrieve the buffer data. + byte* pixels = nullptr; + unsigned int capacity; + bufferByteAccess->GetBuffer(&pixels, &capacity); + + auto plane = buffer->GetPlaneDescription(0); + + for (unsigned int i = 0; i < subtitle->num_rects; i++) + { + auto rect = subtitle->rects[i]; + + for (int y = 0; y < rect->h; y++) + { + for (int x = 0; x < rect->w; x++) + { + auto inPointer = rect->data[0] + y * rect->linesize[0] + x; + auto color = inPointer[0]; + if (color < rect->nb_colors) + { + auto rgba = ((uint32*)rect->data[1])[color]; + auto outPointer = pixels + plane.StartIndex + plane.Stride * ((y + rect->y) - offsetY) + 4 * ((x + rect->x) - offsetX); + ((uint32*)outPointer)[0] = rgba; + } + else + { + OutputDebugString(L"Error: Illegal subtitle color."); + } + } + } + } + } + + cue->SoftwareBitmap = SoftwareBitmap::Convert(bitmap, BitmapPixelFormat::Bgra8, BitmapAlphaMode::Premultiplied); + cue->Position = cuePosition; + cue->Extent = cueSize; + } + else if (subtitle->num_rects > 0) + { + OutputDebugString(L"Error: Invalid subtitle size received."); + } + } + } + catch (...) + { + OutputDebugString(L"Failed to render cue."); + } + mutex.unlock(); + } + + void OnCueExited(Windows::Media::Core::TimedMetadataTrack^ sender, Windows::Media::Core::MediaCueEventArgs^ args) + { + mutex.lock(); + try + { + auto cue = dynamic_cast(args->Cue); + if (cue && cue->Id && cue->SoftwareBitmap) + { + cue->SoftwareBitmap = GetDummyBitmap(); + } + } + catch (...) + { + OutputDebugString(L"Failed to cleanup old cues."); + } + mutex.unlock(); + } + + SoftwareBitmap^ GetDummyBitmap() + { + if (!dummyBitmap) + { + dummyBitmap = ref new SoftwareBitmap(BitmapPixelFormat::Bgra8, 16, 16, BitmapAlphaMode::Premultiplied); + } + + return dummyBitmap; + } + + bool CheckSize(AVSubtitle* subtitle, int& width, int& height, int& offsetX, int& offsetY, TimedTextSize& cueSize, TimedTextPoint& cuePosition) { if (!GetInitialSize()) { @@ -150,9 +253,9 @@ namespace FFmpegInteropX // get actual extent of subtitle rects int minX = subtitleWidth, minY = subtitleHeight, maxW = 0, maxH = 0; - for (unsigned int i = 0; i < subtitle.num_rects; i++) + for (unsigned int i = 0; i < subtitle->num_rects; i++) { - auto rect = subtitle.rects[i]; + auto rect = subtitle->rects[i]; minX = min(minX, rect->x); minY = min(minY, rect->y); maxW = max(maxW, rect->x + rect->w); @@ -233,6 +336,7 @@ namespace FFmpegInteropX int subtitleHeight; int optimalHeight; SoftwareBitmap^ dummyBitmap; - + std::map map; + int nextId; }; -} \ No newline at end of file +} diff --git a/Source/UncompressedSampleProvider.cpp b/Source/UncompressedSampleProvider.cpp index 4d644d7a41..1140ffc510 100644 --- a/Source/UncompressedSampleProvider.cpp +++ b/Source/UncompressedSampleProvider.cpp @@ -197,10 +197,10 @@ HRESULT UncompressedSampleProvider::FeedPacketToDecoder(int64_t& firstPacketPos) return hr; } -void UncompressedSampleProvider::Flush() +void UncompressedSampleProvider::Flush(bool flushBuffers) { - MediaSampleProvider::Flush(); + MediaSampleProvider::Flush(flushBuffers); // after seek we need to get first packet pts again hasNextFramePts = false; -} \ No newline at end of file +} diff --git a/Source/UncompressedSampleProvider.h b/Source/UncompressedSampleProvider.h index bde625b2b0..44c910eda1 100644 --- a/Source/UncompressedSampleProvider.h +++ b/Source/UncompressedSampleProvider.h @@ -48,7 +48,7 @@ namespace FFmpegInteropX public: - virtual void Flush() override; + virtual void Flush(bool flushBuffers) override; private: int64 nextFramePts; diff --git a/Source/UncompressedVideoSampleProvider.h b/Source/UncompressedVideoSampleProvider.h index 40c91aee41..948cd2eda7 100644 --- a/Source/UncompressedVideoSampleProvider.h +++ b/Source/UncompressedVideoSampleProvider.h @@ -48,10 +48,10 @@ namespace FFmpegInteropX { public: virtual ~UncompressedVideoSampleProvider(); - virtual void Flush() override + virtual void Flush(bool flushBuffers) override { hasFirstInterlacedFrame = false; - UncompressedSampleProvider::Flush(); + UncompressedSampleProvider::Flush(flushBuffers); } internal: From 3163dad97de8351dfbf94e9ca4791f7c646d879b Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Thu, 23 Jun 2022 20:38:47 +0200 Subject: [PATCH 02/28] Reorganize stream buffer files --- Source/FFmpegInteropX.vcxproj | 1 + Source/FFmpegInteropX.vcxproj.filters | 1 + Source/FFmpegReader.cpp | 89 +++++----- Source/FFmpegReader.h | 221 +----------------------- Source/MediaSampleProvider.cpp | 1 + Source/StreamBuffer.h | 236 ++++++++++++++++++++++++++ 6 files changed, 293 insertions(+), 256 deletions(-) create mode 100644 Source/StreamBuffer.h diff --git a/Source/FFmpegInteropX.vcxproj b/Source/FFmpegInteropX.vcxproj index e9e213b2f3..c3c0f6b0a1 100644 --- a/Source/FFmpegInteropX.vcxproj +++ b/Source/FFmpegInteropX.vcxproj @@ -131,6 +131,7 @@ + diff --git a/Source/FFmpegInteropX.vcxproj.filters b/Source/FFmpegInteropX.vcxproj.filters index aa39608f05..ed0e1c4e1c 100644 --- a/Source/FFmpegInteropX.vcxproj.filters +++ b/Source/FFmpegInteropX.vcxproj.filters @@ -189,6 +189,7 @@ Helpers + diff --git a/Source/FFmpegReader.cpp b/Source/FFmpegReader.cpp index 9d39caca94..ded5b9a409 100644 --- a/Source/FFmpegReader.cpp +++ b/Source/FFmpegReader.cpp @@ -18,6 +18,7 @@ #include "pch.h" #include "FFmpegReader.h" +#include "StreamBuffer.h" #include "UncompressedSampleProvider.h" using namespace Concurrency; @@ -25,22 +26,22 @@ using namespace Concurrency; using namespace FFmpegInteropX; FFmpegReader::FFmpegReader(AVFormatContext* avFormatCtx, std::vector* initProviders, MediaSourceConfig^ config) - : avFormatCtx(avFormatCtx) - , sampleProviders(initProviders) + : avFormatCtx(avFormatCtx) + , sampleProviders(initProviders) , config(config) { } void FFmpegReader::Start() { - { - std::lock_guard lock(mutex); - if (!isReading) - { - readTask = create_task([this] () { this->ReadDataLoop(); }); - isReading = true; - } - } + { + std::lock_guard lock(mutex); + if (!isReading) + { + readTask = create_task([this] () { this->ReadDataLoop(); }); + isReading = true; + } + } } void FFmpegReader::Stop() @@ -365,51 +366,51 @@ bool FFmpegReader::TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, actualPosition = targetPosition; } - return result; + return result; } void FFmpegReader::ReadDataLoop() { - int ret = 0; - bool sleep = false; + int ret = 0; + bool sleep = false; bool force = false; - while (true) - { - { - std::lock_guard lock(mutex); + while (true) + { + { + std::lock_guard lock(mutex); force = forceReadStream >= 0; sleep = false; for (auto stream : *sampleProviders) { sleep &= stream->buffer->IsFull(stream); } - if (!isReading) - { + if (!isReading) + { break; - } - } + } + } - if (!sleep || force) - { + if (!sleep || force) + { ret = ReadPacket(); if (ret < 0) { break; } - } - else - { - Sleep(100); - } - } - - { - std::lock_guard lock(mutex); - isReading = false; + } + else + { + Sleep(100); + } + } + + { + std::lock_guard lock(mutex); + isReading = false; forceReadStream = -1; result = ret; - } + } } FFmpegReader::~FFmpegReader() @@ -481,16 +482,16 @@ int FFmpegReader::ReadPacketForStream(StreamBuffer^ buffer) manual = !isReading; } - while (true) - { - { - std::lock_guard lock(mutex); + while (true) + { + { + std::lock_guard lock(mutex); - if (!(buffer->IsEmpty())) - { + if (!(buffer->IsEmpty())) + { forceReadStream = -1; break; - } + } else if (result < 0) { break; @@ -499,7 +500,7 @@ int FFmpegReader::ReadPacketForStream(StreamBuffer^ buffer) { forceReadStream = buffer->StreamIndex; } - } + } //task_completion_event tce; @@ -511,8 +512,8 @@ int FFmpegReader::ReadPacketForStream(StreamBuffer^ buffer) { Sleep(10); } - } + } - return result; + return result; } diff --git a/Source/FFmpegReader.h b/Source/FFmpegReader.h index 36c93e1670..8a7b94acc5 100644 --- a/Source/FFmpegReader.h +++ b/Source/FFmpegReader.h @@ -28,20 +28,22 @@ namespace FFmpegInteropX { using namespace Concurrency; - ref class FFmpegReader - { + ref class StreamBuffer; + + ref class FFmpegReader + { internal: FFmpegReader(AVFormatContext* avFormatCtx, std::vector* sampleProviders, MediaSourceConfig^ config); int ReadPacket(); int ReadPacketForStream(StreamBuffer^ buffer); - void Start(); - void Stop(); + void Start(); + void Stop(); void Flush(); HRESULT Seek(TimeSpan position, TimeSpan& actualPosition, TimeSpan currentPosition, bool allowFastSeek, MediaSampleProvider^ videoStream, MediaSampleProvider^ audioStream); - private: + private: ~FFmpegReader(); bool TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, bool fastSeek, MediaSampleProvider^ videoStream, MediaSampleProvider^ audioStream); @@ -49,8 +51,8 @@ namespace FFmpegInteropX void ReadDataLoop(); void FlushCodecs(); - AVFormatContext* avFormatCtx; - std::vector* sampleProviders; + AVFormatContext* avFormatCtx; + std::vector* sampleProviders; MediaSourceConfig^ config; std::mutex mutex; @@ -64,209 +66,4 @@ namespace FFmpegInteropX TimeSpan lastSeekStart; TimeSpan lastSeekActual; }; - - ref class StreamBuffer - { - internal: - StreamBuffer(int streamIndex, MediaSourceConfig^ config) - : config(config) - { - StreamIndex = streamIndex; - - } - - property int StreamIndex; - - void QueuePacket(AVPacket* packet) - { - std::lock_guard lock(mutex); - buffer.push_back(packet); - bufferSize += packet->size; - } - - bool ReadUntilNotEmpty(FFmpegReader^ reader) - { - while (IsEmpty()) - { - if (reader->ReadPacketForStream(this) < 0) - { - DebugMessage(L"GetNextPacket reaching EOF\n"); - break; - } - } - return !IsEmpty(); - } - - bool SkipUntilTimestamp(FFmpegReader^ reader, LONGLONG target) - { - bool foundPacket = false; - - while (!foundPacket) - { - if (ReadUntilNotEmpty(reader)) - { - // peek next packet and check pts value - auto packet = PeekPacket(); - - auto pts = packet->pts != AV_NOPTS_VALUE ? packet->pts : packet->dts; - if (pts != AV_NOPTS_VALUE && packet->duration != AV_NOPTS_VALUE) - { - auto packetEnd = pts + packet->duration; - if (packet->duration > 0 ? packetEnd <= target : packetEnd < target) - { - DropPackets(1); - } - else - { - foundPacket = true; - break; - } - } - else - { - break; - } - } - else - { - // no more packet found - break; - } - } - - return foundPacket; - } - - bool IsEmpty() - { - std::lock_guard lock(mutex); - return buffer.empty(); - } - - bool IsFull(MediaSampleProvider^ sampleProvider) - { - std::lock_guard lock(mutex); - auto maxSize = config->ReadAheadBufferSize; - auto maxDuration = config->ReadAheadBufferDuration; - - bool full = maxSize >= 0 && (long long)bufferSize > maxSize; - if (!full && maxDuration.Duration >= 0 && buffer.size() > 1) - { - auto firstPacket = buffer.front(); - auto lastPacket = buffer.back(); - auto firstPts = firstPacket->pts != AV_NOPTS_VALUE ? firstPacket->pts : firstPacket->dts; - auto lastPts = lastPacket->pts != AV_NOPTS_VALUE ? lastPacket->pts : lastPacket->dts; - - if (firstPts != AV_NOPTS_VALUE && lastPts != AV_NOPTS_VALUE) - { - auto duration = sampleProvider->ConvertDuration(lastPts - firstPts); - full = duration > maxDuration; - } - } - return full; - } - - AVPacket* PopPacket() - { - std::lock_guard lock(mutex); - - AVPacket* packet = NULL; - if (!buffer.empty()) - { - packet = buffer.front(); - buffer.pop_front(); - bufferSize -= packet->size; - } - - return packet; - } - - AVPacket* PeekPacket() - { - std::lock_guard lock(mutex); - - AVPacket* packet = NULL; - if (!buffer.empty()) - { - packet = buffer.front(); - } - - return packet; - } - - AVPacket* PeekPacketIndex(int index) - { - std::lock_guard lock(mutex); - return buffer.at(index); - } - - int TryFindPacketIndex(LONGLONG pts, bool requireKeyFrame) - { - std::lock_guard lock(mutex); - - bool hasEnd = false; - int index = 0; - int result = -1; - for (auto packet : buffer) - { - if (packet->pts <= pts) - { - if (!requireKeyFrame || packet->flags & AV_PKT_FLAG_KEY) - { - result = index; - if (packet->pts + packet->duration >= pts) - { - hasEnd = true; - break; - } - } - } - else - { - hasEnd = true; - break; - } - index++; - } - - if (result >= 0 && !hasEnd) - { - result = -1; - } - - return result; - } - - void Flush() - { - std::lock_guard lock(mutex); - while (!buffer.empty()) - { - auto packet = buffer.front(); - bufferSize -= packet->size; - buffer.pop_front(); - - av_packet_free(&packet); - } - } - - void DropPackets(int count) - { - std::lock_guard lock(mutex); - for (int i = 0; i < count; i++) - { - auto packet = buffer.front(); - bufferSize -= packet->size; - buffer.pop_front(); - - av_packet_free(&packet); - } - } - - private: - std::deque buffer; - std::mutex mutex; - size_t bufferSize; - MediaSourceConfig^ config; - }; } diff --git a/Source/MediaSampleProvider.cpp b/Source/MediaSampleProvider.cpp index 9bcc6af581..c2c1dc00e9 100644 --- a/Source/MediaSampleProvider.cpp +++ b/Source/MediaSampleProvider.cpp @@ -20,6 +20,7 @@ #include "MediaSampleProvider.h" #include "FFmpegMediaSource.h" #include "FFmpegReader.h" +#include "StreamBuffer.h" #include "LanguageTagConverter.h" #include "AvCodecContextHelpers.h" diff --git a/Source/StreamBuffer.h b/Source/StreamBuffer.h new file mode 100644 index 0000000000..9de8aaf6a4 --- /dev/null +++ b/Source/StreamBuffer.h @@ -0,0 +1,236 @@ +//***************************************************************************** +// +// Copyright 2015 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http ://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//***************************************************************************** + +#pragma once + +#include +#include + +#include "FFmpegReader.h" +#include "MediaSourceConfig.h" +#include "MediaSampleProvider.h" + +namespace FFmpegInteropX +{ + ref class FFmpegReader; + + ref class StreamBuffer + { + internal: + StreamBuffer(int streamIndex, MediaSourceConfig^ config) + : config(config) + { + StreamIndex = streamIndex; + + } + + property int StreamIndex; + + void QueuePacket(AVPacket* packet) + { + std::lock_guard lock(mutex); + buffer.push_back(packet); + bufferSize += packet->size; + } + + bool ReadUntilNotEmpty(FFmpegReader^ reader) + { + while (IsEmpty()) + { + if (reader->ReadPacketForStream(this) < 0) + { + DebugMessage(L"GetNextPacket reaching EOF\n"); + break; + } + } + return !IsEmpty(); + } + + bool SkipUntilTimestamp(FFmpegReader^ reader, LONGLONG target) + { + bool foundPacket = false; + + while (!foundPacket) + { + if (ReadUntilNotEmpty(reader)) + { + // peek next packet and check pts value + auto packet = PeekPacket(); + + auto pts = packet->pts != AV_NOPTS_VALUE ? packet->pts : packet->dts; + if (pts != AV_NOPTS_VALUE && packet->duration != AV_NOPTS_VALUE) + { + auto packetEnd = pts + packet->duration; + if (packet->duration > 0 ? packetEnd <= target : packetEnd < target) + { + DropPackets(1); + } + else + { + foundPacket = true; + break; + } + } + else + { + break; + } + } + else + { + // no more packet found + break; + } + } + + return foundPacket; + } + + bool IsEmpty() + { + std::lock_guard lock(mutex); + return buffer.empty(); + } + + bool IsFull(MediaSampleProvider^ sampleProvider) + { + std::lock_guard lock(mutex); + auto maxSize = config->ReadAheadBufferSize; + auto maxDuration = config->ReadAheadBufferDuration; + + bool full = maxSize >= 0 && (long long)bufferSize > maxSize; + if (!full && maxDuration.Duration >= 0 && buffer.size() > 1) + { + auto firstPacket = buffer.front(); + auto lastPacket = buffer.back(); + auto firstPts = firstPacket->pts != AV_NOPTS_VALUE ? firstPacket->pts : firstPacket->dts; + auto lastPts = lastPacket->pts != AV_NOPTS_VALUE ? lastPacket->pts : lastPacket->dts; + + if (firstPts != AV_NOPTS_VALUE && lastPts != AV_NOPTS_VALUE) + { + auto duration = sampleProvider->ConvertDuration(lastPts - firstPts); + full = duration > maxDuration; + } + } + return full; + } + + AVPacket* PopPacket() + { + std::lock_guard lock(mutex); + + AVPacket* packet = NULL; + if (!buffer.empty()) + { + packet = buffer.front(); + buffer.pop_front(); + bufferSize -= packet->size; + } + + return packet; + } + + AVPacket* PeekPacket() + { + std::lock_guard lock(mutex); + + AVPacket* packet = NULL; + if (!buffer.empty()) + { + packet = buffer.front(); + } + + return packet; + } + + AVPacket* PeekPacketIndex(int index) + { + std::lock_guard lock(mutex); + return buffer.at(index); + } + + int TryFindPacketIndex(LONGLONG pts, bool requireKeyFrame) + { + std::lock_guard lock(mutex); + + bool hasEnd = false; + int index = 0; + int result = -1; + for (auto packet : buffer) + { + if (packet->pts <= pts) + { + if (!requireKeyFrame || packet->flags & AV_PKT_FLAG_KEY) + { + result = index; + if (packet->pts + packet->duration >= pts) + { + hasEnd = true; + break; + } + } + } + else + { + hasEnd = true; + break; + } + index++; + } + + if (result >= 0 && !hasEnd) + { + result = -1; + } + + return result; + } + + void Flush() + { + std::lock_guard lock(mutex); + while (!buffer.empty()) + { + auto packet = buffer.front(); + bufferSize -= packet->size; + buffer.pop_front(); + + av_packet_free(&packet); + } + } + + void DropPackets(int count) + { + std::lock_guard lock(mutex); + for (int i = 0; i < count; i++) + { + auto packet = buffer.front(); + bufferSize -= packet->size; + buffer.pop_front(); + + av_packet_free(&packet); + } + } + + private: + std::deque buffer; + std::mutex mutex; + size_t bufferSize; + MediaSourceConfig^ config; + }; +} From c9fd9152327eb997298da7660d9cb3ba62f50c64 Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Thu, 23 Jun 2022 20:39:02 +0200 Subject: [PATCH 03/28] Cleanup solution and editor config --- .editorconfig | 2 +- FFmpegInteropX.sln | 11 ++++++++--- Samples/MediaPlayerCS/MediaPlayerCS.csproj | 2 +- Tests/UnitTest.csproj | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.editorconfig b/.editorconfig index 6cd5928d78..fc68c1084e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,7 +7,7 @@ root = true charset = utf-8 indent_style = space indent_size = 4 -end_of_line = lf +end_of_line =crlf trim_trailing_whitespace = true insert_final_newline = true diff --git a/FFmpegInteropX.sln b/FFmpegInteropX.sln index 2843d9e0f0..c625b1b5a7 100644 --- a/FFmpegInteropX.sln +++ b/FFmpegInteropX.sln @@ -1,9 +1,9 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.28307.136 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.32002.261 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FFmpegInterop", "Source\FFmpegInteropX.vcxproj", "{9CFA3B3E-B7AF-4629-84E2-C962C5B046B1}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "FFmpegInteropX", "Source\FFmpegInteropX.vcxproj", "{9CFA3B3E-B7AF-4629-84E2-C962C5B046B1}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "MediaPlayerCPP", "Samples\MediaPlayerCPP\MediaPlayerCPP.vcxproj", "{9C0EB7EC-6FED-46E7-9CC9-F36132050AA1}" EndProject @@ -14,6 +14,11 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MediaPlayerCS", "Samples\Me EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnitTest", "Tests\UnitTest.csproj", "{F1ECBD62-71FE-47FB-B588-1E331A375B59}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{F6A9745D-66E3-42BC-8124-BA173E9E781C}" + ProjectSection(SolutionItems) = preProject + .editorconfig = .editorconfig + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM = Debug|ARM diff --git a/Samples/MediaPlayerCS/MediaPlayerCS.csproj b/Samples/MediaPlayerCS/MediaPlayerCS.csproj index d5194451a6..c1e4d9dd17 100644 --- a/Samples/MediaPlayerCS/MediaPlayerCS.csproj +++ b/Samples/MediaPlayerCS/MediaPlayerCS.csproj @@ -148,7 +148,7 @@ {9cfa3b3e-b7af-4629-84e2-c962c5b046b1} - FFmpegInterop + FFmpegInteropX diff --git a/Tests/UnitTest.csproj b/Tests/UnitTest.csproj index bee4becf5f..15a95511b5 100644 --- a/Tests/UnitTest.csproj +++ b/Tests/UnitTest.csproj @@ -144,7 +144,7 @@ {9cfa3b3e-b7af-4629-84e2-c962c5b046b1} - FFmpegInterop + FFmpegInteropX From 8f37cc752ae4dcedfa0eca2e648e1abbd276fbb1 Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Thu, 23 Jun 2022 21:46:27 +0200 Subject: [PATCH 04/28] Fine tuning --- Samples/MediaPlayerCPP/MainPage.xaml.cpp | 8 ++++---- Samples/MediaPlayerCS/MainPage.xaml.cs | 10 ++++++++++ Source/FFmpegMediaSource.cpp | 3 +++ Source/FFmpegReader.cpp | 22 ++++++++++++---------- Source/StringUtils.h | 2 +- 5 files changed, 30 insertions(+), 15 deletions(-) diff --git a/Samples/MediaPlayerCPP/MainPage.xaml.cpp b/Samples/MediaPlayerCPP/MainPage.xaml.cpp index 1faeaccb3d..d6bfdce74f 100644 --- a/Samples/MediaPlayerCPP/MainPage.xaml.cpp +++ b/Samples/MediaPlayerCPP/MainPage.xaml.cpp @@ -511,14 +511,14 @@ void MediaPlayerCPP::MainPage::OnKeyDown(Windows::UI::Core::CoreWindow^ sender, } } - if (args->VirtualKey == Windows::System::VirtualKey::Right && FFmpegMSS && mediaPlayer->CanSeek) + if (args->VirtualKey == Windows::System::VirtualKey::Right && FFmpegMSS && mediaPlayer->PlaybackSession->CanSeek) { - mediaPlayer->Position = TimeSpan{ min(mediaPlayer->Position.Duration + 50000000, mediaPlayer->NaturalDuration.Duration) }; + mediaPlayer->PlaybackSession->Position = TimeSpan{ mediaPlayer->PlaybackSession->Position.Duration + 50000000 }; } - if (args->VirtualKey == Windows::System::VirtualKey::Left && FFmpegMSS && mediaPlayer->CanSeek) + if (args->VirtualKey == Windows::System::VirtualKey::Left && FFmpegMSS && mediaPlayer->PlaybackSession->CanSeek) { - mediaPlayer->Position = TimeSpan{ max(mediaPlayer->Position.Duration - 50000000, 0) }; + mediaPlayer->PlaybackSession->Position = TimeSpan{ mediaPlayer->PlaybackSession->Position.Duration - 50000000 }; } } diff --git a/Samples/MediaPlayerCS/MainPage.xaml.cs b/Samples/MediaPlayerCS/MainPage.xaml.cs index 2a37e303ef..b4632e86a3 100644 --- a/Samples/MediaPlayerCS/MainPage.xaml.cs +++ b/Samples/MediaPlayerCS/MainPage.xaml.cs @@ -109,6 +109,16 @@ private async void MainPage_KeyDown(CoreWindow sender, KeyEventArgs args) playbackItem.VideoTracks.SelectedIndex = index; } } + + if (args.VirtualKey == VirtualKey.Right && FFmpegMSS != null && mediaPlayer.PlaybackSession.CanSeek) + { + mediaPlayer.PlaybackSession.Position += TimeSpan.FromSeconds(5); + } + + if (args.VirtualKey == VirtualKey.Left && FFmpegMSS != null && mediaPlayer.PlaybackSession.CanSeek) + { + mediaPlayer.PlaybackSession.Position -= TimeSpan.FromSeconds(5); + } } private async void CodecChecker_CodecRequired(object sender, CodecRequiredEventArgs args) diff --git a/Source/FFmpegMediaSource.cpp b/Source/FFmpegMediaSource.cpp index c95299a9a9..b87009feb6 100644 --- a/Source/FFmpegMediaSource.cpp +++ b/Source/FFmpegMediaSource.cpp @@ -945,6 +945,9 @@ HRESULT FFmpegMediaSource::InitFFmpegContext() // Convert media duration from AV_TIME_BASE to TimeSpan unit mediaDuration = { LONGLONG(avFormatCtx->duration * 10000000 / double(AV_TIME_BASE)) }; + // Assign initial BufferTime to MediaStreamSource + mss->BufferTime = TimeSpan{ 0 }; + if (Windows::Foundation::Metadata::ApiInformation::IsPropertyPresent("Windows.Media.Core.MediaStreamSource", "MaxSupportedPlaybackRate")) { mss->MaxSupportedPlaybackRate = config->MaxSupportedPlaybackRate; diff --git a/Source/FFmpegReader.cpp b/Source/FFmpegReader.cpp index ded5b9a409..a0c0f54d41 100644 --- a/Source/FFmpegReader.cpp +++ b/Source/FFmpegReader.cpp @@ -68,7 +68,8 @@ void FFmpegReader::FlushCodecs() std::lock_guard lock(mutex); for (auto stream : *sampleProviders) { - stream->Flush(false); + if (stream) + stream->Flush(false); } } @@ -77,7 +78,8 @@ void FFmpegReader::Flush() std::lock_guard lock(mutex); for (auto stream : *sampleProviders) { - stream->Flush(true); + if (stream) + stream->Flush(true); } result = 0; } @@ -234,9 +236,6 @@ HRESULT FFmpegReader::SeekFast(TimeSpan position, TimeSpan& actualPosition, Time if (hr == S_OK) { - // reduce playback delay - timestampVideo += timestampVideoDuration; - actualPosition = timestampVideo; // remember last seek direction @@ -273,10 +272,10 @@ HRESULT FFmpegReader::SeekFast(TimeSpan position, TimeSpan& actualPosition, Time audioStream->SkipPacketsUntilTimestamp(audioTarget); auto sample = audioStream->GetNextSample(); - /*if (sample) + if (sample) { actualPosition = sample->Timestamp + sample->Duration; - }*/ + } } } else if (audioPreroll.Duration <= 0) @@ -289,10 +288,10 @@ HRESULT FFmpegReader::SeekFast(TimeSpan position, TimeSpan& actualPosition, Time { // decode one audio sample to get clean output auto sample = audioStream->GetNextSample(); - /*if (sample) + if (sample) { actualPosition = sample->Timestamp + sample->Duration; - }*/ + } } } } @@ -383,7 +382,10 @@ void FFmpegReader::ReadDataLoop() sleep = false; for (auto stream : *sampleProviders) { - sleep &= stream->buffer->IsFull(stream); + if (stream) + { + sleep &= stream->buffer->IsFull(stream); + } } if (!isReading) { diff --git a/Source/StringUtils.h b/Source/StringUtils.h index 6070018095..2d8feee11e 100644 --- a/Source/StringUtils.h +++ b/Source/StringUtils.h @@ -42,7 +42,7 @@ namespace FFmpegInteropX static std::string PlatformStringToUtf8String(String^ value) { - size_t required_size = WideCharToMultiByte(CP_UTF8, 0, value->Data(), -1, NULL, 0, NULL, NULL); + int required_size = WideCharToMultiByte(CP_UTF8, 0, value->Data(), -1, NULL, 0, NULL, NULL); char* buffer = (char*)calloc(required_size, sizeof(char)); auto result = WideCharToMultiByte(CP_UTF8, 0, value->Data(), -1, buffer, required_size, NULL, NULL); std::string s_str = std::string(buffer); From c1a0a1e6a4f1fdba6c9922aee107127d8b1249ff Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Fri, 24 Jun 2022 21:43:13 +0200 Subject: [PATCH 05/28] Change to task based synchronization --- Source/FFmpegMediaSource.cpp | 8 -- Source/FFmpegMediaSource.h | 1 - Source/FFmpegReader.cpp | 145 ++++++++++++++++++++++++++------- Source/FFmpegReader.h | 5 ++ Source/MediaSampleProvider.cpp | 5 ++ Source/MediaSampleProvider.h | 1 + Source/MediaSourceConfig.h | 2 +- 7 files changed, 127 insertions(+), 40 deletions(-) diff --git a/Source/FFmpegMediaSource.cpp b/Source/FFmpegMediaSource.cpp index b87009feb6..6704b82604 100644 --- a/Source/FFmpegMediaSource.cpp +++ b/Source/FFmpegMediaSource.cpp @@ -967,7 +967,6 @@ HRESULT FFmpegMediaSource::InitFFmpegContext() startingRequestedToken = mss->Starting += ref new TypedEventHandler(this, &FFmpegMediaSource::OnStarting); sampleRequestedToken = mss->SampleRequested += ref new TypedEventHandler(this, &FFmpegMediaSource::OnSampleRequested); switchStreamRequestedToken = mss->SwitchStreamsRequested += ref new TypedEventHandler(this, &FFmpegMediaSource::OnSwitchStreamsRequested); - mss->Paused += ref new Windows::Foundation::TypedEventHandler(this, &FFmpegInteropX::FFmpegMediaSource::OnPaused); } } @@ -1560,13 +1559,6 @@ HRESULT FFmpegMediaSource::ParseOptions(PropertySet^ ffmpegOptions) return hr; } - -void FFmpegInteropX::FFmpegMediaSource::OnPaused(Windows::Media::Core::MediaStreamSource^ sender, Platform::Object^ args) -{ - m_pReader->Stop(); -} - - void FFmpegMediaSource::OnStarting(MediaStreamSource^ sender, MediaStreamSourceStartingEventArgs^ args) { mutexGuard.lock(); diff --git a/Source/FFmpegMediaSource.h b/Source/FFmpegMediaSource.h index ef98d8d48d..36fbcee526 100644 --- a/Source/FFmpegMediaSource.h +++ b/Source/FFmpegMediaSource.h @@ -350,7 +350,6 @@ namespace FFmpegInteropX static CoreDispatcher^ GetCurrentDispatcher(); void OnPositionChanged(Windows::Media::Playback::MediaPlaybackSession^ sender, Platform::Object^ args); - void OnPaused(Windows::Media::Core::MediaStreamSource^ sender, Platform::Object^ args); }; } diff --git a/Source/FFmpegReader.cpp b/Source/FFmpegReader.cpp index a0c0f54d41..25076a74d7 100644 --- a/Source/FFmpegReader.cpp +++ b/Source/FFmpegReader.cpp @@ -17,6 +17,7 @@ //***************************************************************************** #include "pch.h" +#include #include "FFmpegReader.h" #include "StreamBuffer.h" #include "UncompressedSampleProvider.h" @@ -31,14 +32,14 @@ FFmpegReader::FFmpegReader(AVFormatContext* avFormatCtx, std::vector lock(mutex); - if (!isReading) + if (!isReading && (config->ReadAheadBufferSize > 0 || config->ReadAheadBufferDuration.Duration > 0) && !config->IsFrameGrabber) { readTask = create_task([this] () { this->ReadDataLoop(); }); + sleepEvent = task_completion_event(); isReading = true; } } @@ -52,13 +53,26 @@ void FFmpegReader::Stop() if (isReading) { isReading = false; + sleepEvent.set(); wait = true; } } if (wait) { - readTask.wait(); + try + { + readTask.wait(); + } + catch (...) + { + DebugMessage(L"Failed to wait for task. Probably destructor called from UI thread.\n"); + + while (!readTask.is_done()) + { + Sleep(1); + } + } } } @@ -86,6 +100,8 @@ void FFmpegReader::Flush() HRESULT FFmpegReader::Seek(TimeSpan position, TimeSpan& actualPosition, TimeSpan currentPosition, bool fastSeek, MediaSampleProvider^ videoStream, MediaSampleProvider^ audioStream) { + Stop(); + auto hr = S_OK; if (result != 0) { @@ -372,28 +388,29 @@ void FFmpegReader::ReadDataLoop() { int ret = 0; bool sleep = false; - bool force = false; + + // Create a call object that prints characters that it receives + // to the console. + call breakSleep([this] (int) + { + sleepEvent.set(); + }); + timer sleepTimer(10000u, 0, &breakSleep, false); while (true) { { std::lock_guard lock(mutex); - force = forceReadStream >= 0; - sleep = false; - for (auto stream : *sampleProviders) - { - if (stream) - { - sleep &= stream->buffer->IsFull(stream); - } - } if (!isReading) { break; } + + sleep = CheckNeedsSleep(sleep); } - if (!sleep || force) + + if (!sleep) { ret = ReadPacket(); if (ret < 0) @@ -403,18 +420,65 @@ void FFmpegReader::ReadDataLoop() } else { - Sleep(100); + auto sleepTask = create_task(sleepEvent); + sleepTimer.start(); + sleepTask.wait(); + sleepEvent = task_completion_event(); } } - + { std::lock_guard lock(mutex); + if (forceReadStream != -1) + { + waitStreamEvent.set(); + } isReading = false; forceReadStream = -1; result = ret; + lastStream = nullptr; + fullStream = nullptr; } } + +bool FFmpegReader::CheckNeedsSleep(bool wasSleeping) +{ + bool force = forceReadStream >= 0; + if (force) + { + return false; + } + + bool sleep = wasSleeping; + + // check if we need to start sleeping + if (!sleep && lastStream && lastStream->IsBufferFull()) + { + sleep = true; + fullStream = lastStream; + } + + // check if we can stop sleeping + if (sleep && !fullStream && !fullStream->IsBufferFull()) + { + sleep = false; + fullStream = nullptr; + for (auto stream : *sampleProviders) + { + if (stream && stream->IsBufferFull()) + { + sleep = true; + fullStream = stream; + break; + } + } + } + + return sleep; +} + + FFmpegReader::~FFmpegReader() { } @@ -434,7 +498,6 @@ int FFmpegReader::ReadPacket() ret = av_read_frame(avFormatCtx, avPacket); if (ret < 0) { - std::lock_guard lock(mutex); av_packet_free(&avPacket); } else @@ -445,6 +508,7 @@ int FFmpegReader::ReadPacket() if (avPacket->stream_index == forceReadStream) { forceReadStream = -1; + waitStreamEvent.set(); } if (avPacket->stream_index >= (int)sampleProviders->size()) @@ -462,6 +526,7 @@ int FFmpegReader::ReadPacket() if (provider) { provider->QueuePacket(avPacket); + lastStream = provider; } else { @@ -471,6 +536,7 @@ int FFmpegReader::ReadPacket() } } + std::lock_guard lock(mutex); result = ret; return result; } @@ -484,14 +550,13 @@ int FFmpegReader::ReadPacketForStream(StreamBuffer^ buffer) manual = !isReading; } - while (true) + if (manual) { + // no read-ahead used + while (true) { - std::lock_guard lock(mutex); - if (!(buffer->IsEmpty())) { - forceReadStream = -1; break; } else if (result < 0) @@ -500,19 +565,39 @@ int FFmpegReader::ReadPacketForStream(StreamBuffer^ buffer) } else { - forceReadStream = buffer->StreamIndex; + ReadPacket(); } } + } + else + { + // read-ahead active + while (true) + { + task waitStreamTask; + { + std::lock_guard lock(mutex); - //task_completion_event tce; + if (!(buffer->IsEmpty())) + { + forceReadStream = -1; + break; + } + else if (result < 0) + { + break; + } + else + { + forceReadStream = buffer->StreamIndex; + waitStreamEvent = task_completion_event(); + waitStreamTask = create_task(waitStreamEvent); - if (manual) - { - ReadPacket(); - } - else - { - Sleep(10); + sleepEvent.set(); + } + } + + waitStreamTask.wait(); } } diff --git a/Source/FFmpegReader.h b/Source/FFmpegReader.h index 8a7b94acc5..5d1425765c 100644 --- a/Source/FFmpegReader.h +++ b/Source/FFmpegReader.h @@ -50,16 +50,21 @@ namespace FFmpegInteropX HRESULT SeekFast(TimeSpan position, TimeSpan& actualPosition, TimeSpan currentPosition, MediaSampleProvider^ videoStream, MediaSampleProvider^ audioStream); void ReadDataLoop(); void FlushCodecs(); + bool CheckNeedsSleep(bool wasSleeping); AVFormatContext* avFormatCtx; std::vector* sampleProviders; MediaSourceConfig^ config; + MediaSampleProvider^ lastStream; + MediaSampleProvider^ fullStream; std::mutex mutex; bool isReading; int forceReadStream; int result; task readTask; + task_completion_event sleepEvent; + task_completion_event waitStreamEvent; bool isFirstSeekAfterStreamSwitch; bool isLastSeekForward; diff --git a/Source/MediaSampleProvider.cpp b/Source/MediaSampleProvider.cpp index c2c1dc00e9..d6c404e67c 100644 --- a/Source/MediaSampleProvider.cpp +++ b/Source/MediaSampleProvider.cpp @@ -348,6 +348,11 @@ void MediaSampleProvider::QueuePacket(AVPacket *packet) } } +bool MediaSampleProvider::IsBufferFull() +{ + return buffer->IsFull(this); +} + void MediaSampleProvider::Flush(bool flushBuffers) { DebugMessage(L"Flush\n"); diff --git a/Source/MediaSampleProvider.h b/Source/MediaSampleProvider.h index 774d17d34c..e3daa4bdc7 100644 --- a/Source/MediaSampleProvider.h +++ b/Source/MediaSampleProvider.h @@ -107,6 +107,7 @@ namespace FFmpegInteropX virtual void InitializeStreamInfo(); virtual void QueuePacket(AVPacket* packet); HRESULT GetNextPacket(AVPacket** avPacket, LONGLONG& packetPts, LONGLONG& packetDuration); + bool IsBufferFull(); virtual HRESULT CreateNextSampleBuffer(IBuffer^* pBuffer, int64_t& samplePts, int64_t& sampleDuration, IDirect3DSurface^* surface) = 0; HRESULT GetNextPacketTimestamp(TimeSpan& timestamp, TimeSpan& packetDuration); HRESULT SkipPacketsUntilTimestamp(TimeSpan timestamp); diff --git a/Source/MediaSourceConfig.h b/Source/MediaSourceConfig.h index 756dc8be07..d8617d81ac 100644 --- a/Source/MediaSourceConfig.h +++ b/Source/MediaSourceConfig.h @@ -108,7 +108,7 @@ namespace FFmpegInteropX AutoCorrectAnsiSubtitles = true; AnsiSubtitleEncoding = CharacterEncoding::GetSystemDefault(); - FastSeek = false; + FastSeek = true; FastSeekCleanAudio = true; FastSeekSmartStreamSwitching = true; From 4419163f7b3bf0c78aefa13c8ab1fc81661c3400 Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Fri, 24 Jun 2022 22:07:37 +0200 Subject: [PATCH 06/28] Fix bug in TexturePool --- Source/D3D11VideoSampleProvider.h | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Source/D3D11VideoSampleProvider.h b/Source/D3D11VideoSampleProvider.h index d9b8199db2..fb7faf4e84 100644 --- a/Source/D3D11VideoSampleProvider.h +++ b/Source/D3D11VideoSampleProvider.h @@ -120,7 +120,10 @@ namespace FFmpegInteropX { if (sample->Direct3D11Surface) { - samples.insert(reinterpret_cast(sample)); + // AddRef the sample on native interface to prevent it from being collected before Processed is called + auto sampleNative = reinterpret_cast(sample); + sampleNative->AddRef(); + sample->Processed += ref new Windows::Foundation::TypedEventHandler(this, &D3D11VideoSampleProvider::OnProcessed); } @@ -145,7 +148,9 @@ namespace FFmpegInteropX texturePool->ReturnTexture(texture); } - samples.erase(reinterpret_cast(sender)); + // Release the sample's native interface + auto sampleNative = reinterpret_cast(sender); + sampleNative->Release(); SAFE_RELEASE(surface); SAFE_RELEASE(texture); @@ -379,7 +384,6 @@ namespace FFmpegInteropX } TexturePool^ texturePool; - std::set samples; }; } From cf36315059a7bcd643a4f1c3b761271fd495e156 Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Sun, 26 Jun 2022 22:11:20 +0200 Subject: [PATCH 07/28] Fix flush wrong stream after switch streams --- Source/FFmpegMediaSource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/FFmpegMediaSource.cpp b/Source/FFmpegMediaSource.cpp index 6704b82604..6ebbb0b9ce 100644 --- a/Source/FFmpegMediaSource.cpp +++ b/Source/FFmpegMediaSource.cpp @@ -1754,7 +1754,7 @@ void FFmpegMediaSource::OnSwitchStreamsRequested(MediaStreamSource^ sender, Medi if (currentVideoStream && args->Request->OldStreamDescriptor == currentVideoStream->StreamDescriptor) { currentVideoStream->DisableStream(); - currentAudioStream->Flush(true); + currentVideoStream->Flush(true); currentVideoStream = nullptr; } From 6b19a04384f7d31bf012dc3072afe737e906a840 Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Thu, 30 Jun 2022 21:32:01 +0200 Subject: [PATCH 08/28] Change read ahead to non blocking --- Source/D3D11VideoSampleProvider.h | 4 +- Source/FFmpegReader.cpp | 166 +++++++++++++++--------------- Source/FFmpegReader.h | 10 +- 3 files changed, 93 insertions(+), 87 deletions(-) diff --git a/Source/D3D11VideoSampleProvider.h b/Source/D3D11VideoSampleProvider.h index 20e14c3515..70c0285b63 100644 --- a/Source/D3D11VideoSampleProvider.h +++ b/Source/D3D11VideoSampleProvider.h @@ -52,9 +52,9 @@ namespace FFmpegInteropX ReleaseTrackedSamples(); } - virtual void Flush() override + virtual void Flush(bool flushBuffers) override { - UncompressedVideoSampleProvider::Flush(); + UncompressedVideoSampleProvider::Flush(flushBuffers); ReturnTrackedSamples(); } diff --git a/Source/FFmpegReader.cpp b/Source/FFmpegReader.cpp index fa759a25a2..9f8cac1db5 100644 --- a/Source/FFmpegReader.cpp +++ b/Source/FFmpegReader.cpp @@ -17,7 +17,6 @@ //***************************************************************************** #include "pch.h" -#include #include "FFmpegReader.h" #include "StreamBuffer.h" #include "UncompressedSampleProvider.h" @@ -33,15 +32,31 @@ FFmpegReader::FFmpegReader(AVFormatContext* avFormatCtx, std::vector lock(mutex); - if (!isReading && (config->ReadAheadBufferSize > 0 || config->ReadAheadBufferDuration.Duration > 0) && !config->IsFrameGrabber) + if (!isEnabled && (config->ReadAheadBufferSize > 0 || config->ReadAheadBufferDuration.Duration > 0) && !config->IsFrameGrabber) { - readTask = create_task([this] () { this->ReadDataLoop(); }); - sleepEvent = task_completion_event(); - isReading = true; + sleepTimerTarget = new call([this](int value) { OnTimer(value); }); + if (!sleepTimerTarget) + { + return; + } + + sleepTimer = new timer(100u, 0, sleepTimerTarget, true); + if (!sleepTimer) + { + delete sleepTimerTarget; + return; + } + + readTask = create_task([this]() { this->ReadDataLoop(); }); + isEnabled = true; } } } @@ -51,11 +66,14 @@ void FFmpegReader::Stop() bool wait = false; { std::lock_guard lock(mutex); - if (isReading) + if (isEnabled) { - isReading = false; - sleepEvent.set(); + isEnabled = false; wait = true; + + sleepTimer->stop(); + delete sleepTimer; + delete sleepTimerTarget; } } @@ -76,6 +94,13 @@ void FFmpegReader::Stop() } } + if (forceReadStream != -1) + { + waitStreamEvent.set(); + } + forceReadStream = -1; + lastStream = nullptr; + fullStream = nullptr; } void FFmpegReader::FlushCodecs() @@ -96,7 +121,7 @@ void FFmpegReader::Flush() if (stream) stream->Flush(true); } - result = 0; + readResult = 0; } HRESULT FFmpegReader::Seek(TimeSpan position, TimeSpan& actualPosition, TimeSpan currentPosition, bool fastSeek, MediaSampleProvider^ videoStream, MediaSampleProvider^ audioStream) @@ -104,10 +129,10 @@ HRESULT FFmpegReader::Seek(TimeSpan position, TimeSpan& actualPosition, TimeSpan Stop(); auto hr = S_OK; - if (result != 0) + if (readResult != 0) { fastSeek = false; - result = 0; + readResult = 0; } // Select the first valid stream either from video or audio @@ -120,8 +145,7 @@ HRESULT FFmpegReader::Seek(TimeSpan position, TimeSpan& actualPosition, TimeSpan TimeSpan videoSkipPts, audioSkipPts; if (TrySeekBuffered(position, actualPosition, fastSeek, videoStream, audioStream)) { - // Flush all active stream codecs but keep bufferss - FlushCodecs(); + // all good } else if (fastSeek) { @@ -390,59 +414,43 @@ void FFmpegReader::ReadDataLoop() int ret = 0; bool sleep = false; - // Create a call object that prints characters that it receives - // to the console. - call breakSleep([this] (int) - { - sleepEvent.set(); - }); - timer sleepTimer(10000u, 0, &breakSleep, false); - while (true) { { std::lock_guard lock(mutex); - if (!isReading) + if (!isEnabled) { break; } sleep = CheckNeedsSleep(sleep); - } - - - if (!sleep) - { - ret = ReadPacket(); - if (ret < 0) + if (sleep) { + isSleeping = true; + sleepTimer->start(); break; } } - else + + ret = ReadPacket(); + if (ret < 0) { - auto sleepTask = create_task(sleepEvent); - sleepTimer.start(); - sleepTask.wait(); - sleepEvent = task_completion_event(); + break; } } - +} + +void FFmpegInteropX::FFmpegReader::OnTimer(int value) +{ + std::lock_guard lock(mutex); + if (isEnabled) { - std::lock_guard lock(mutex); - if (forceReadStream != -1) - { - waitStreamEvent.set(); - } - isReading = false; - forceReadStream = -1; - result = ret; - lastStream = nullptr; - fullStream = nullptr; + readTask = create_task([this]() { ReadDataLoop(); }); + isSleeping = false; } + sleepTimer->pause(); } - bool FFmpegReader::CheckNeedsSleep(bool wasSleeping) { bool force = forceReadStream >= 0; @@ -479,11 +487,6 @@ bool FFmpegReader::CheckNeedsSleep(bool wasSleeping) return sleep; } - -FFmpegReader::~FFmpegReader() -{ -} - // Read the next packet from the stream and push it into the appropriate // sample provider int FFmpegReader::ReadPacket() @@ -497,32 +500,32 @@ int FFmpegReader::ReadPacket() } ret = av_read_frame(avFormatCtx, avPacket); + std::lock_guard lock(mutex); + readResult = ret; + if (ret < 0) { av_packet_free(&avPacket); } else { + if (avPacket->stream_index == forceReadStream) { - std::lock_guard lock(mutex); - - if (avPacket->stream_index == forceReadStream) - { - forceReadStream = -1; - waitStreamEvent.set(); - } - - if (avPacket->stream_index >= (int)sampleProviders->size()) - { - // new stream detected. if this is a subtitle stream, we could create it now. - av_packet_free(&avPacket); - } - - if (avPacket->stream_index < 0) - { - av_packet_free(&avPacket); - } + forceReadStream = -1; + waitStreamEvent.set(); + } + if (avPacket->stream_index >= (int)sampleProviders->size()) + { + // new stream detected. if this is a subtitle stream, we could create it now. + av_packet_free(&avPacket); + } + else if (avPacket->stream_index < 0) + { + av_packet_free(&avPacket); + } + else + { MediaSampleProvider^ provider = sampleProviders->at(avPacket->stream_index); if (provider) { @@ -537,18 +540,21 @@ int FFmpegReader::ReadPacket() } } - std::lock_guard lock(mutex); - result = ret; - return result; + return readResult; } int FFmpegReader::ReadPacketForStream(StreamBuffer^ buffer) { + if (!(buffer->IsEmpty())) + { + return readResult; + } + bool manual; { std::lock_guard lock(mutex); - manual = !isReading; + manual = !isEnabled; } if (manual) @@ -556,18 +562,16 @@ int FFmpegReader::ReadPacketForStream(StreamBuffer^ buffer) // no read-ahead used while (true) { + ReadPacket(); + if (!(buffer->IsEmpty())) { break; } - else if (result < 0) + else if (readResult < 0) { break; } - else - { - ReadPacket(); - } } } else @@ -584,7 +588,7 @@ int FFmpegReader::ReadPacketForStream(StreamBuffer^ buffer) forceReadStream = -1; break; } - else if (result < 0) + else if (readResult < 0) { break; } @@ -593,8 +597,6 @@ int FFmpegReader::ReadPacketForStream(StreamBuffer^ buffer) forceReadStream = buffer->StreamIndex; waitStreamEvent = task_completion_event(); waitStreamTask = create_task(waitStreamEvent); - - sleepEvent.set(); } } @@ -602,6 +604,6 @@ int FFmpegReader::ReadPacketForStream(StreamBuffer^ buffer) } } - return result; + return readResult; } diff --git a/Source/FFmpegReader.h b/Source/FFmpegReader.h index 5d1425765c..82c47257b6 100644 --- a/Source/FFmpegReader.h +++ b/Source/FFmpegReader.h @@ -20,6 +20,7 @@ #include #include +#include #include "MediaSourceConfig.h" #include "MediaSampleProvider.h" @@ -48,6 +49,7 @@ namespace FFmpegInteropX ~FFmpegReader(); bool TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, bool fastSeek, MediaSampleProvider^ videoStream, MediaSampleProvider^ audioStream); HRESULT SeekFast(TimeSpan position, TimeSpan& actualPosition, TimeSpan currentPosition, MediaSampleProvider^ videoStream, MediaSampleProvider^ audioStream); + void OnTimer(int value); void ReadDataLoop(); void FlushCodecs(); bool CheckNeedsSleep(bool wasSleeping); @@ -59,12 +61,14 @@ namespace FFmpegInteropX MediaSampleProvider^ fullStream; std::mutex mutex; - bool isReading; + bool isEnabled; + bool isSleeping; int forceReadStream; - int result; + int readResult; task readTask; - task_completion_event sleepEvent; task_completion_event waitStreamEvent; + call* sleepTimerTarget; + timer* sleepTimer; bool isFirstSeekAfterStreamSwitch; bool isLastSeekForward; From 62603bec5f6b39bd43f1c26143e44d92b6306cd7 Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Mon, 18 Jul 2022 21:58:40 +0200 Subject: [PATCH 09/28] Fix stream buffer for WinRT migration --- Source/FFmpegReader.cpp | 21 +++--- Source/FFmpegReader.h | 4 +- Source/MediaSampleProvider.cpp | 18 ++--- Source/MediaSampleProvider.h | 2 +- Source/MediaSourceConfig.cpp | 2 +- Source/StreamBuffer.h | 3 - Source/SubtitleProvider.h | 131 ++++++++++++++++----------------- Source/pch.h | 4 +- 8 files changed, 89 insertions(+), 96 deletions(-) diff --git a/Source/FFmpegReader.cpp b/Source/FFmpegReader.cpp index 5a3d0eabc7..2daba5c3a8 100644 --- a/Source/FFmpegReader.cpp +++ b/Source/FFmpegReader.cpp @@ -142,7 +142,6 @@ HRESULT FFmpegReader::Seek(TimeSpan position, TimeSpan& actualPosition, TimeSpan { int64_t seekTarget = stream->ConvertPosition(position); - TimeSpan videoSkipPts, audioSkipPts; if (TrySeekBuffered(position, actualPosition, fastSeek, videoStream, audioStream)) { // all good @@ -345,20 +344,20 @@ HRESULT FFmpegReader::SeekFast(TimeSpan position, TimeSpan& actualPosition, Time bool FFmpegReader::TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, bool fastSeek, std::shared_ptr videoStream, std::shared_ptr audioStream) { - bool result = true;; - int vIndex; int aIndex; + bool result = true; + int vIndex = 0; int aIndex = 0; TimeSpan targetPosition = position; if (videoStream) { auto pts = videoStream->ConvertPosition(targetPosition); - vIndex = videoStream->buffer->TryFindPacketIndex(pts, true); + vIndex = videoStream->packetBuffer->TryFindPacketIndex(pts, true); result &= vIndex >= 0; if (result && fastSeek) { - auto targetPacket = videoStream->buffer->PeekPacketIndex(vIndex); + auto targetPacket = videoStream->packetBuffer->PeekPacketIndex(vIndex); if (targetPacket->pts != AV_NOPTS_VALUE) { targetPosition = videoStream->ConvertPosition(targetPacket->pts); @@ -369,7 +368,7 @@ bool FFmpegReader::TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, if (result && audioStream) { auto pts = audioStream->ConvertPosition(targetPosition); - aIndex = audioStream->buffer->TryFindPacketIndex(pts, false); + aIndex = audioStream->packetBuffer->TryFindPacketIndex(pts, false); result &= aIndex >= 0; } @@ -380,7 +379,7 @@ bool FFmpegReader::TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, if (videoStream) { - videoStream->buffer->DropPackets(vIndex); + videoStream->packetBuffer->DropPackets(vIndex); } if (audioStream) @@ -388,18 +387,19 @@ bool FFmpegReader::TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, if (config.FastSeekCleanAudio() && aIndex > 0) { aIndex--; - audioStream->buffer->DropPackets(aIndex); + audioStream->packetBuffer->DropPackets(aIndex); // decode one audio sample to get clean output auto sample = audioStream->GetNextSample(); if (sample) { - actualPosition = sample.Timestamp() + sample.Duration(); + // TODO check if this is a good idea + //actualPosition = sample.Timestamp() + sample.Duration(); } } else { - audioStream->buffer->DropPackets(aIndex); + audioStream->packetBuffer->DropPackets(aIndex); } } @@ -442,6 +442,7 @@ void FFmpegReader::ReadDataLoop() void FFmpegInteropX::FFmpegReader::OnTimer(int value) { + UNREFERENCED_PARAMETER(value); std::lock_guard lock(mutex); if (isEnabled) { diff --git a/Source/FFmpegReader.h b/Source/FFmpegReader.h index c720449c99..00e4187525 100644 --- a/Source/FFmpegReader.h +++ b/Source/FFmpegReader.h @@ -18,8 +18,6 @@ #pragma once -#include -#include #include #include "MediaSourceConfig.h" @@ -35,6 +33,7 @@ namespace FFmpegInteropX { public: FFmpegReader(AVFormatContext* avFormatCtx, std::vector>* sampleProviders, MediaSourceConfig config); + virtual ~FFmpegReader(); int ReadPacket(); int ReadPacketForStream(StreamBuffer* buffer); @@ -45,7 +44,6 @@ namespace FFmpegInteropX private: - ~FFmpegReader(); bool TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, bool fastSeek, std::shared_ptr videoStream, std::shared_ptr audioStream); HRESULT SeekFast(TimeSpan position, TimeSpan& actualPosition, TimeSpan currentPosition, std::shared_ptr videoStream, std::shared_ptr audioStream); void OnTimer(int value); diff --git a/Source/MediaSampleProvider.cpp b/Source/MediaSampleProvider.cpp index 3427c926a8..4eb14d95cd 100644 --- a/Source/MediaSampleProvider.cpp +++ b/Source/MediaSampleProvider.cpp @@ -46,7 +46,7 @@ MediaSampleProvider::MediaSampleProvider( , m_config(config) , m_streamIndex(streamIndex) , hardwareDecoderStatus(hardwareDecoderStatus) - , buffer(new StreamBuffer(streamIndex, config)) + , packetBuffer(new StreamBuffer(streamIndex, config)) { DebugMessage(L"MediaSampleProvider\n"); @@ -264,9 +264,9 @@ HRESULT MediaSampleProvider::GetNextPacket(AVPacket** avPacket, LONGLONG& packet { HRESULT hr = S_OK; - if (buffer->ReadUntilNotEmpty(m_pReader)) + if (packetBuffer->ReadUntilNotEmpty(m_pReader)) { - auto packet = buffer->PopPacket(); + auto packet = packetBuffer->PopPacket(); *avPacket = packet; @@ -303,10 +303,10 @@ HRESULT MediaSampleProvider::GetNextPacketTimestamp(TimeSpan& timestamp, TimeSpa { HRESULT hr = S_FALSE; - if (buffer->ReadUntilNotEmpty(m_pReader)) + if (packetBuffer->ReadUntilNotEmpty(m_pReader)) { // peek next packet and set pts value - auto packet = buffer->PeekPacket(); + auto packet = packetBuffer->PeekPacket(); auto pts = packet->pts != AV_NOPTS_VALUE ? packet->pts : packet->dts; if (pts != AV_NOPTS_VALUE) { @@ -323,7 +323,7 @@ HRESULT MediaSampleProvider::SkipPacketsUntilTimestamp(TimeSpan const& timestamp { HRESULT hr = S_OK; - if (!buffer->SkipUntilTimestamp(m_pReader, ConvertPosition(timestamp))) + if (!packetBuffer->SkipUntilTimestamp(m_pReader, ConvertPosition(timestamp))) { hr = S_FALSE; } @@ -337,7 +337,7 @@ void MediaSampleProvider::QueuePacket(AVPacket* packet) if (m_isEnabled) { - buffer->QueuePacket(packet); + packetBuffer->QueuePacket(packet); } else { @@ -347,7 +347,7 @@ void MediaSampleProvider::QueuePacket(AVPacket* packet) bool MediaSampleProvider::IsBufferFull() { - return buffer->IsFull(this); + return packetBuffer->IsFull(this); } void MediaSampleProvider::Flush(bool flushBuffers) @@ -359,7 +359,7 @@ void MediaSampleProvider::Flush(bool flushBuffers) } if (flushBuffers) { - buffer->Flush(); + packetBuffer->Flush(); } m_isDiscontinuous = true; IsCleanSample = false; diff --git a/Source/MediaSampleProvider.h b/Source/MediaSampleProvider.h index 2a044f8e88..d1c3683967 100644 --- a/Source/MediaSampleProvider.h +++ b/Source/MediaSampleProvider.h @@ -188,7 +188,7 @@ namespace FFmpegInteropX HardwareDecoderStatus hardwareDecoderStatus; public: - std::shared_ptr buffer; + std::shared_ptr packetBuffer; protected: diff --git a/Source/MediaSourceConfig.cpp b/Source/MediaSourceConfig.cpp index ee2228fb35..b061fc8202 100644 --- a/Source/MediaSourceConfig.cpp +++ b/Source/MediaSourceConfig.cpp @@ -105,7 +105,7 @@ namespace winrt::FFmpegInteropX::implementation m_AutoCorrectAnsiSubtitles = true; AnsiSubtitleEncoding(CharacterEncoding::GetSystemDefault()); - m_FastSeek = false; + m_FastSeek = true; m_FastSeekCleanAudio = true; m_FastSeekSmartStreamSwitching = true; diff --git a/Source/StreamBuffer.h b/Source/StreamBuffer.h index 3746690c15..d9bc7d47bd 100644 --- a/Source/StreamBuffer.h +++ b/Source/StreamBuffer.h @@ -18,9 +18,6 @@ #pragma once -#include -#include - #include "FFmpegReader.h" #include "MediaSourceConfig.h" #include "MediaSampleProvider.h" diff --git a/Source/SubtitleProvider.h b/Source/SubtitleProvider.h index 6c9b0abc63..a8d1949c5c 100644 --- a/Source/SubtitleProvider.h +++ b/Source/SubtitleProvider.h @@ -82,97 +82,94 @@ namespace FFmpegInteropX virtual void QueuePacket(AVPacket* packet) override { - if (true)//m_isEnabled) + try { - try - { - TimeSpan position = ConvertPosition(packet->pts); - TimeSpan duration = ConvertDuration(packet->duration); + TimeSpan position = ConvertPosition(packet->pts); + TimeSpan duration = ConvertDuration(packet->duration); - auto cue = CreateCue(packet, &position, &duration); - if (cue && position.count() >= 0) + auto cue = CreateCue(packet, &position, &duration); + if (cue && position.count() >= 0) + { + // apply subtitle delay + position += SubtitleDelay; + if (position.count() < 0) { - // apply subtitle delay - position += SubtitleDelay; - if (position.count() < 0) - { - negativePositionCues.emplace_back(cue, position.count()); - position = std::chrono::seconds(0); - } + negativePositionCues.emplace_back(cue, position.count()); + position = std::chrono::seconds(0); + } - // clip previous extended duration cue, if there is one - if (lastExtendedDurationCue && m_config.PreventModifiedSubtitleDurationOverlap() && - lastExtendedDurationCue.StartTime() + lastExtendedDurationCue.Duration() > position) + // clip previous extended duration cue, if there is one + if (lastExtendedDurationCue && m_config.PreventModifiedSubtitleDurationOverlap() && + lastExtendedDurationCue.StartTime() + lastExtendedDurationCue.Duration() > position) + { + auto diff = position - (lastExtendedDurationCue.StartTime() + lastExtendedDurationCue.Duration()); + auto newDuration = lastExtendedDurationCue.Duration() + diff; + if (newDuration.count() > 0) { - auto diff = position - (lastExtendedDurationCue.StartTime() + lastExtendedDurationCue.Duration()); - auto newDuration = lastExtendedDurationCue.Duration() + diff; - if (newDuration.count() > 0) - { - lastExtendedDurationCue.Duration() = newDuration; - if (!m_config.as().get()->IsExternalSubtitleParser) - { - pendingChangedDurationCues.push_back(lastExtendedDurationCue); - } - } - else + lastExtendedDurationCue.Duration() = newDuration; + if (!m_config.as().get()->IsExternalSubtitleParser) { - // weird subtitle timings, just leave it as is + pendingChangedDurationCues.push_back(lastExtendedDurationCue); } } + else + { + // weird subtitle timings, just leave it as is + } + } lastExtendedDurationCue = nullptr; - if (duration.count() < 0) + if (duration.count() < 0) + { + duration = TimeSpan(InfiniteDuration); + } + else + { + if (m_config.AdditionalSubtitleDuration().count() != 0) { - duration = TimeSpan(InfiniteDuration); + duration += m_config.AdditionalSubtitleDuration(); + lastExtendedDurationCue = cue; } - else + if (duration < m_config.MinimumSubtitleDuration()) { - if (m_config.AdditionalSubtitleDuration().count() != 0) - { - duration += m_config.AdditionalSubtitleDuration(); - lastExtendedDurationCue = cue; - } - if (duration < m_config.MinimumSubtitleDuration()) - { - duration = m_config.MinimumSubtitleDuration(); - lastExtendedDurationCue = cue; - } + duration = m_config.MinimumSubtitleDuration(); + lastExtendedDurationCue = cue; } + } - cue.StartTime(position); - cue.Duration(duration); - AddCue(cue); + cue.StartTime(position); + cue.Duration(duration); + AddCue(cue); - if (!m_config.as()->IsExternalSubtitleParser) + if (!m_config.as()->IsExternalSubtitleParser) + { + isPreviousCueInfiniteDuration = duration.count() >= InfiniteDuration; + } + else + { + // fixup infinite duration cues for external subs + if (isPreviousCueInfiniteDuration) { - isPreviousCueInfiniteDuration = duration.count() >= InfiniteDuration; + infiniteDurationCue.Duration(TimeSpan(cue.StartTime() - infiniteDurationCue.StartTime())); + } + + if (duration.count() >= InfiniteDuration) + { + isPreviousCueInfiniteDuration = true; + infiniteDurationCue = cue; } else { - // fixup infinite duration cues for external subs - if (isPreviousCueInfiniteDuration) - { - infiniteDurationCue.Duration(TimeSpan(cue.StartTime() - infiniteDurationCue.StartTime())); - } - - if (duration.count() >= InfiniteDuration) - { - isPreviousCueInfiniteDuration = true; - infiniteDurationCue = cue; - } - else - { - isPreviousCueInfiniteDuration = false; - infiniteDurationCue = nullptr; - } + isPreviousCueInfiniteDuration = false; + infiniteDurationCue = nullptr; } } } - catch (...) - { - OutputDebugString(L"Failed to create subtitle cue."); - } + } + catch (...) + { + OutputDebugString(L"Failed to create subtitle cue."); } av_packet_free(&packet); } diff --git a/Source/pch.h b/Source/pch.h index f99226bc4b..ff72f61e0f 100644 --- a/Source/pch.h +++ b/Source/pch.h @@ -98,10 +98,10 @@ std::vector inline to_vector(IVectorView input) template std::function inline weak_handler(T* instance, void(T::* instanceMethod)(TSender, TArgs)) { - std::weak_ptr wr = instance->weak_from_this(); + auto wr = instance->weak_from_this(); auto handler = [wr, instanceMethod](TSender sender, TArgs args) { - auto instanceLocked = wr.lock(); + auto instanceLocked = std::dynamic_pointer_cast(wr.lock()); if (instanceLocked) { (instanceLocked.get()->*instanceMethod)(sender, args); From de87a6f82927bf0174c3a28b3c3d03db2a1183fa Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Mon, 18 Jul 2022 22:16:28 +0200 Subject: [PATCH 10/28] Add deprecation attribute --- Source/FFmpegInteropX.idl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Source/FFmpegInteropX.idl b/Source/FFmpegInteropX.idl index c2af24f319..c9c6dbed63 100644 --- a/Source/FFmpegInteropX.idl +++ b/Source/FFmpegInteropX.idl @@ -554,12 +554,12 @@ namespace FFmpegInteropX ///The default BufferTime that gets assigned to the MediaStreamSource for Windows.Storage.Streams.IRandomAccessStream sources. ///Deprecated due to framework bugs and memory consumption. Use ReadAheadBufferSize and ReadAheadBufferDuration instead. - //[deprecated] + [deprecated("Deprecated due to framework bugs and memory consumption. Use ReadAheadBufferSize and ReadAheadBufferDuration instead.", deprecate, 1)] Windows.Foundation.TimeSpan DefaultBufferTime{ get; set; }; ///The default BufferTime that gets assigned to the MediaStreamSource for URI sources. ///Deprecated due to framework bugs and memory consumption. Use ReadAheadBufferSize and ReadAheadBufferDuration instead. - //[deprecated] + [deprecated("Deprecated due to framework bugs and memory consumption. Use ReadAheadBufferSize and ReadAheadBufferDuration instead.", deprecate, 1)] Windows.Foundation.TimeSpan DefaultBufferTimeUri{ get; set; }; ///The maximum number of bytes to buffer ahead per stream. From c0e029d04e30614ad8679c15a4af1def4730a3f7 Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Tue, 19 Jul 2022 23:42:25 +0200 Subject: [PATCH 11/28] Improve stream buffer for fast seek and normal seek --- Samples/MediaPlayerCPP/MainPage.xaml.cpp | 5 ++ Samples/MediaPlayerCS/MainPage.xaml.cs | 5 ++ Source/FFmpegMediaSource.cpp | 2 + Source/FFmpegReader.cpp | 73 +++++++++++++----------- Source/MediaSampleProvider.cpp | 4 +- Source/StreamBuffer.h | 31 ++++++---- 6 files changed, 73 insertions(+), 47 deletions(-) diff --git a/Samples/MediaPlayerCPP/MainPage.xaml.cpp b/Samples/MediaPlayerCPP/MainPage.xaml.cpp index bd13be6f9b..2764a94cc7 100644 --- a/Samples/MediaPlayerCPP/MainPage.xaml.cpp +++ b/Samples/MediaPlayerCPP/MainPage.xaml.cpp @@ -488,6 +488,11 @@ void MediaPlayerCPP::MainPage::EnableVideoEffects_Toggled(Platform::Object^ send void MediaPlayerCPP::MainPage::OnKeyDown(Windows::UI::Core::CoreWindow^ sender, Windows::UI::Core::KeyEventArgs^ args) { + if (args->Handled) + { + return; + } + if (args->VirtualKey == Windows::System::VirtualKey::Enter && (Window::Current->CoreWindow->GetKeyState(Windows::System::VirtualKey::Control) & Windows::UI::Core::CoreVirtualKeyStates::Down) == Windows::UI::Core::CoreVirtualKeyStates::Down && StorageApplicationPermissions::FutureAccessList->Entries->Size == 1) { diff --git a/Samples/MediaPlayerCS/MainPage.xaml.cs b/Samples/MediaPlayerCS/MainPage.xaml.cs index a20f315e3c..4f9113f932 100644 --- a/Samples/MediaPlayerCS/MainPage.xaml.cs +++ b/Samples/MediaPlayerCS/MainPage.xaml.cs @@ -87,6 +87,11 @@ public MainPage() private async void MainPage_KeyDown(CoreWindow sender, KeyEventArgs args) { + if (args.Handled) + { + return; + } + if (args.VirtualKey == VirtualKey.Enter && (Window.Current.CoreWindow.GetKeyState(VirtualKey.Control) & CoreVirtualKeyStates.Down) == CoreVirtualKeyStates.Down && StorageApplicationPermissions.FutureAccessList.Entries.Count == 1) { diff --git a/Source/FFmpegMediaSource.cpp b/Source/FFmpegMediaSource.cpp index e2a85e44c0..40564331fb 100644 --- a/Source/FFmpegMediaSource.cpp +++ b/Source/FFmpegMediaSource.cpp @@ -1952,6 +1952,8 @@ namespace winrt::FFmpegInteropX::implementation HRESULT FFmpegMediaSource::Seek(const TimeSpan& position, TimeSpan& actualPosition, bool allowFastSeek) { + DebugMessage(L"Seek\n"); + auto diffCurrent = position - currentPosition; auto diffLast = position - lastPosition; bool isSeekBeforeStreamSwitch = allowFastSeek && config->FastSeekSmartStreamSwitching() && !isFirstSeekAfterStreamSwitch && diffCurrent.count() > 0 && diffCurrent.count() < 5000000 && diffLast.count() > 0 && diffLast.count() < 10000000; diff --git a/Source/FFmpegReader.cpp b/Source/FFmpegReader.cpp index 2daba5c3a8..e6054cab46 100644 --- a/Source/FFmpegReader.cpp +++ b/Source/FFmpegReader.cpp @@ -135,45 +135,42 @@ HRESULT FFmpegReader::Seek(TimeSpan position, TimeSpan& actualPosition, TimeSpan readResult = 0; } - // Select the first valid stream either from video or audio - auto stream = videoStream ? videoStream : audioStream; + auto isForwardSeek = position > currentPosition; - if (stream) + if (isForwardSeek && TrySeekBuffered(position, actualPosition, fastSeek, videoStream, audioStream)) { + // all good + DebugMessage(L"BufferedSeek!\n"); + } + else if (fastSeek) + { + hr = SeekFast(position, actualPosition, currentPosition, videoStream, audioStream); + } + else + { + DebugMessage(L"NormalSeek\n"); + // Select the first valid stream either from video or audio + auto stream = videoStream ? videoStream : audioStream; int64_t seekTarget = stream->ConvertPosition(position); - - if (TrySeekBuffered(position, actualPosition, fastSeek, videoStream, audioStream)) + if (av_seek_frame(avFormatCtx, stream->StreamIndex(), seekTarget, AVSEEK_FLAG_BACKWARD) < 0) { - // all good - } - else if (fastSeek) - { - hr = SeekFast(position, actualPosition, currentPosition, videoStream, audioStream); + hr = E_FAIL; + DebugMessage(L" - ### Error while seeking\n"); } else { - if (av_seek_frame(avFormatCtx, stream->StreamIndex(), seekTarget, AVSEEK_FLAG_BACKWARD) < 0) - { - hr = E_FAIL; - DebugMessage(L" - ### Error while seeking\n"); - } - else - { - // Flush all active streams with buffers - Flush(); - } + // Flush all active streams with buffers + Flush(); } } - else - { - hr = E_FAIL; - } return hr; } HRESULT FFmpegReader::SeekFast(TimeSpan position, TimeSpan& actualPosition, TimeSpan currentPosition, std::shared_ptr videoStream, std::shared_ptr audioStream) { + DebugMessage(L"SeekFast\n"); + HRESULT hr = S_OK; int64_t seekTarget = videoStream->ConvertPosition(position); bool isUriSource = avFormatCtx->url; @@ -233,11 +230,14 @@ HRESULT FFmpegReader::SeekFast(TimeSpan position, TimeSpan& actualPosition, Time TimeSpan timestampVideo; TimeSpan timestampVideoDuration; hr = videoStream->GetNextPacketTimestamp(timestampVideo, timestampVideoDuration); + bool hasVideoPts = hr == S_OK; - if (hr == S_FALSE && isUriSource) + if (hr == S_FALSE) { - //TODO if (dynamic_cast(videoStream)) + // S_FALSE means that the video packets do not contain timestamps + if (std::dynamic_pointer_cast(videoStream)) { + // If we do not use passthrough, we can decode (and drop) then next sample to get the correct time. auto sample = videoStream->GetNextSample(); if (sample) { @@ -247,13 +247,14 @@ HRESULT FFmpegReader::SeekFast(TimeSpan position, TimeSpan& actualPosition, Time hr = S_OK; } } - //else + else { - //hr = audioStream->GetNextPacketTimestamp(timestampVideo, timestampVideoDuration); + // Otherwise, try with audio stream instead + hr = audioStream->GetNextPacketTimestamp(timestampVideo, timestampVideoDuration); } } - while (hr == S_OK && seekForward && timestampVideo < referenceTime && !isUriSource) + while (hr == S_OK && seekForward && timestampVideo < referenceTime && !isUriSource && hasVideoPts) { // our min position was not respected. try again with higher min and target. min += videoStream->ConvertDuration(TimeSpan{ 50000000 }); @@ -345,14 +346,15 @@ HRESULT FFmpegReader::SeekFast(TimeSpan position, TimeSpan& actualPosition, Time bool FFmpegReader::TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, bool fastSeek, std::shared_ptr videoStream, std::shared_ptr audioStream) { bool result = true; - int vIndex = 0; int aIndex = 0; + int vIndex = -1; + int aIndex = -1; TimeSpan targetPosition = position; if (videoStream) { auto pts = videoStream->ConvertPosition(targetPosition); - vIndex = videoStream->packetBuffer->TryFindPacketIndex(pts, true); + vIndex = videoStream->packetBuffer->TryFindPacketIndex(pts, true, fastSeek); result &= vIndex >= 0; if (result && fastSeek) @@ -368,11 +370,16 @@ bool FFmpegReader::TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, if (result && audioStream) { auto pts = audioStream->ConvertPosition(targetPosition); - aIndex = audioStream->packetBuffer->TryFindPacketIndex(pts, false); + aIndex = audioStream->packetBuffer->TryFindPacketIndex(pts, false, fastSeek); result &= aIndex >= 0; } - if (result) + if (result && vIndex == 0) + { + // We are at correct position already. No flush required. + DebugMessage(L"BufferedSeek: No flush\n"); + } + else if (result) { // Flush all active streams but keep buffers FlushCodecs(); diff --git a/Source/MediaSampleProvider.cpp b/Source/MediaSampleProvider.cpp index 4eb14d95cd..34d9d996af 100644 --- a/Source/MediaSampleProvider.cpp +++ b/Source/MediaSampleProvider.cpp @@ -210,7 +210,7 @@ void FFmpegInteropX::MediaSampleProvider::InitializeStreamInfo() MediaStreamSample MediaSampleProvider::GetNextSample() { - DebugMessage(L"GetNextSample\n"); + //DebugMessage(L"GetNextSample\n"); HRESULT hr = S_OK; @@ -333,7 +333,7 @@ HRESULT MediaSampleProvider::SkipPacketsUntilTimestamp(TimeSpan const& timestamp void MediaSampleProvider::QueuePacket(AVPacket* packet) { - DebugMessage(L" - QueuePacket\n"); + //DebugMessage(L" - QueuePacket\n"); if (m_isEnabled) { diff --git a/Source/StreamBuffer.h b/Source/StreamBuffer.h index d9bc7d47bd..e3e767e709 100644 --- a/Source/StreamBuffer.h +++ b/Source/StreamBuffer.h @@ -161,7 +161,7 @@ namespace FFmpegInteropX return buffer.at(index); } - int TryFindPacketIndex(LONGLONG pts, bool requireKeyFrame) + int TryFindPacketIndex(LONGLONG pts, bool requireKeyFrame, bool fastSeek) { std::lock_guard lock(mutex); @@ -170,28 +170,35 @@ namespace FFmpegInteropX int result = -1; for (auto packet : buffer) { - if (packet->pts <= pts) + if (packet->pts != AV_NOPTS_VALUE) { - if (!requireKeyFrame || packet->flags & AV_PKT_FLAG_KEY) + // with fast seek, we can select a key frame behind seek target + if (packet->pts <= pts || (fastSeek && result == -1)) { - result = index; - if (packet->pts + packet->duration >= pts) + if (!requireKeyFrame || packet->flags & AV_PKT_FLAG_KEY) { - hasEnd = true; - break; + result = index; } } - } - else - { - hasEnd = true; - break; + + if (packet->pts + packet->duration >= pts && (!fastSeek || result > -1)) + { + hasEnd = true; + break; + } } index++; } + if (hasEnd && result == -1 && !fastSeek) + { + // no key frame between current frame and target position. need to decode from here. + result = 0; + } + if (result >= 0 && !hasEnd) { + // key frames found but buffer range not up to target position result = -1; } From 73df120dd40eb35b2bb5b30dbb1bf5801792be9b Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Wed, 20 Jul 2022 20:04:18 +0200 Subject: [PATCH 12/28] Work on stream buffer --- Source/FFmpegMediaSource.cpp | 10 +++++----- Source/FFmpegReader.cpp | 14 ++++++++++++-- Source/StreamBuffer.h | 15 +++++++++++++++ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/Source/FFmpegMediaSource.cpp b/Source/FFmpegMediaSource.cpp index 40564331fb..d42c292888 100644 --- a/Source/FFmpegMediaSource.cpp +++ b/Source/FFmpegMediaSource.cpp @@ -1785,12 +1785,12 @@ namespace winrt::FFmpegInteropX::implementation } } - m_pReader->Start(); + m_pReader->Start(); - isFirstSeek = false; - isFirstSeekAfterStreamSwitch = false; - mutexGuard.unlock(); -} + isFirstSeek = false; + isFirstSeekAfterStreamSwitch = false; + mutexGuard.unlock(); + } void FFmpegMediaSource::OnSampleRequested(winrt::Windows::Media::Core::MediaStreamSource const& sender, winrt::Windows::Media::Core::MediaStreamSourceSampleRequestedEventArgs const& args) { diff --git a/Source/FFmpegReader.cpp b/Source/FFmpegReader.cpp index e6054cab46..dd48daa3f2 100644 --- a/Source/FFmpegReader.cpp +++ b/Source/FFmpegReader.cpp @@ -442,6 +442,12 @@ void FFmpegReader::ReadDataLoop() ret = ReadPacket(); if (ret < 0) { + std::lock_guard lock(mutex); + isEnabled = false; + sleepTimer->stop(); + delete sleepTimer; + delete sleepTimerTarget; + waitStreamEvent.set(); break; } } @@ -556,13 +562,17 @@ int FFmpegReader::ReadPacketForStream(StreamBuffer* buffer) { if (!(buffer->IsEmpty())) { - return readResult; + return 0; } bool manual; { std::lock_guard lock(mutex); manual = !isEnabled; + if (readResult < 0) + { + return readResult; + } } if (manual) @@ -596,7 +606,7 @@ int FFmpegReader::ReadPacketForStream(StreamBuffer* buffer) forceReadStream = -1; break; } - else if (readResult < 0) + else if (readResult < 0 || !isEnabled) { break; } diff --git a/Source/StreamBuffer.h b/Source/StreamBuffer.h index e3e767e709..9441d1967a 100644 --- a/Source/StreamBuffer.h +++ b/Source/StreamBuffer.h @@ -165,6 +165,21 @@ namespace FFmpegInteropX { std::lock_guard lock(mutex); + if (buffer.size() == 0) + { + return -1; + } + + auto firstPacket = buffer.front(); + auto lastPacket = buffer.back(); + auto firstPts = firstPacket->pts != AV_NOPTS_VALUE ? firstPacket->pts : firstPacket->dts; + auto lastPts = lastPacket->pts != AV_NOPTS_VALUE ? lastPacket->pts : lastPacket->dts; + + if (firstPts != AV_NOPTS_VALUE && lastPts != AV_NOPTS_VALUE && (firstPts > pts || lastPts < pts)) + { + return -1; + } + bool hasEnd = false; int index = 0; int result = -1; From fc551259043e7beb78acc1a46fb543be7ff9e757 Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Wed, 20 Jul 2022 22:48:10 +0200 Subject: [PATCH 13/28] Optimize references and seek limit --- Source/FFmpegReader.cpp | 11 ++++++----- Source/SubtitleProviderBitmap.h | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Source/FFmpegReader.cpp b/Source/FFmpegReader.cpp index dd48daa3f2..794270b5e7 100644 --- a/Source/FFmpegReader.cpp +++ b/Source/FFmpegReader.cpp @@ -106,7 +106,7 @@ void FFmpegReader::Stop() void FFmpegReader::FlushCodecs() { std::lock_guard lock(mutex); - for (auto stream : *sampleProviders) + for (auto& stream : *sampleProviders) { if (stream) stream->Flush(false); @@ -116,7 +116,7 @@ void FFmpegReader::FlushCodecs() void FFmpegReader::Flush() { std::lock_guard lock(mutex); - for (auto stream : *sampleProviders) + for (auto& stream : *sampleProviders) { if (stream) stream->Flush(true); @@ -254,7 +254,8 @@ HRESULT FFmpegReader::SeekFast(TimeSpan position, TimeSpan& actualPosition, Time } } - while (hr == S_OK && seekForward && timestampVideo < referenceTime && !isUriSource && hasVideoPts) + int seekCount = 0; + while (hr == S_OK && seekForward && timestampVideo < referenceTime && !isUriSource && hasVideoPts && seekCount++ < 10) { // our min position was not respected. try again with higher min and target. min += videoStream->ConvertDuration(TimeSpan{ 50000000 }); @@ -487,7 +488,7 @@ bool FFmpegReader::CheckNeedsSleep(bool wasSleeping) { sleep = false; fullStream = nullptr; - for (auto stream : *sampleProviders) + for (auto& stream : *sampleProviders) { if (stream && stream->IsBufferFull()) { @@ -540,7 +541,7 @@ int FFmpegInteropX::FFmpegReader::ReadPacket() } else { - auto provider = sampleProviders->at(avPacket->stream_index); + auto& provider = sampleProviders->at(avPacket->stream_index); if (provider) { provider->QueuePacket(avPacket); diff --git a/Source/SubtitleProviderBitmap.h b/Source/SubtitleProviderBitmap.h index e98788236d..406ed2e7b4 100644 --- a/Source/SubtitleProviderBitmap.h +++ b/Source/SubtitleProviderBitmap.h @@ -56,7 +56,7 @@ namespace FFmpegInteropX virtual IMediaCue CreateCue(AVPacket* packet, TimeSpan* position, TimeSpan* duration) override { - AVSubtitle* subtitle = (AVSubtitle*)malloc(sizeof(AVSubtitle)); + AVSubtitle* subtitle = (AVSubtitle*)av_mallocz(sizeof(AVSubtitle)); if (!subtitle) { return nullptr; @@ -107,7 +107,7 @@ namespace FFmpegInteropX else if (result <= 0) { avsubtitle_free(subtitle); - delete subtitle; + av_freep(subtitle); OutputDebugString(L"Failed to decode subtitle."); } return nullptr; From 7a4c7b02c24f069d44b68954eb62a3b3f0675ad0 Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Thu, 21 Jul 2022 00:10:38 +0200 Subject: [PATCH 14/28] Improve config options --- Source/FFmpegInteropX.idl | 11 ++++ Source/FFmpegMediaSource.cpp | 13 ++++- Source/FFmpegMediaSource.h | 4 ++ Source/FFmpegReader.cpp | 101 +++++++++++++++++---------------- Source/FFmpegReader.h | 2 +- Source/MediaSampleProvider.cpp | 2 +- Source/MediaSourceConfig.cpp | 28 +++++++-- Source/MediaSourceConfig.h | 13 ++++- Source/StreamBuffer.h | 27 +++++---- 9 files changed, 130 insertions(+), 71 deletions(-) diff --git a/Source/FFmpegInteropX.idl b/Source/FFmpegInteropX.idl index c9c6dbed63..3915abc39b 100644 --- a/Source/FFmpegInteropX.idl +++ b/Source/FFmpegInteropX.idl @@ -435,6 +435,9 @@ namespace FFmpegInteropX ///The subtitle stream. Windows.Foundation.IAsyncOperation > AddExternalSubtitleAsync(Windows.Storage.Streams.IRandomAccessStream stream); + ///Starts filling the read-ahead buffer, if enabled in the configuration. + void StartBuffering(); + // Properties ///Gets the configuration that has been passed when creating the MSS instance. @@ -546,8 +549,12 @@ namespace FFmpegInteropX Double MaxSupportedPlaybackRate{ get; set; }; ///The buffer size in bytes to use for Windows.Storage.Streams.IRandomAccessStream sources. + [deprecated("Deprecated due to irritating name. Use ReadAheadBufferSize and ReadAheadBufferDuration instead.", deprecate, 1)] UInt32 StreamBufferSize{ get; set; }; + ///The maximum number of bytes to read in one chunk for Windows.Storage.Streams.IRandomAccessStream sources. + UInt32 FileStreamReadSize{ get; set; }; + ///Additional options to use when creating the ffmpeg AVFormatContext. Windows.Foundation.Collections.PropertySet FFmpegOptions{ get; set; }; @@ -562,6 +569,10 @@ namespace FFmpegInteropX [deprecated("Deprecated due to framework bugs and memory consumption. Use ReadAheadBufferSize and ReadAheadBufferDuration instead.", deprecate, 1)] Windows.Foundation.TimeSpan DefaultBufferTimeUri{ get; set; }; + ///Enables or disables the read-ahead buffer. + ///This value can be changed any time during playback. + Int64 ReadAheadBufferEnabled{ get; set; }; + ///The maximum number of bytes to buffer ahead per stream. ///This value can be changed any time during playback. Int64 ReadAheadBufferSize{ get; set; }; diff --git a/Source/FFmpegMediaSource.cpp b/Source/FFmpegMediaSource.cpp index d42c292888..79fe05132d 100644 --- a/Source/FFmpegMediaSource.cpp +++ b/Source/FFmpegMediaSource.cpp @@ -133,7 +133,7 @@ namespace winrt::FFmpegInteropX::implementation { // Setup FFmpeg custom IO to access file as stream. This is necessary when accessing any file outside of app installation directory and appdata folder. // Credit to Philipp Sch http://www.codeproject.com/Tips/489450/Creating-Custom-FFmpeg-IO-Context - fileStreamBuffer = (unsigned char*)av_malloc(config->StreamBufferSize()); + fileStreamBuffer = (unsigned char*)av_malloc(config->FileStreamReadSize()); if (fileStreamBuffer == nullptr) { hr = E_OUTOFMEMORY; @@ -1327,6 +1327,11 @@ namespace winrt::FFmpegInteropX::implementation return AddExternalSubtitleAsync(stream, config->DefaultExternalSubtitleStreamName()); } + void FFmpegMediaSource::StartBuffering() + { + m_pReader->Start(); + } + FFmpegInteropX::MediaSourceConfig FFmpegMediaSource::Configuration() { return config.as(); @@ -1785,8 +1790,6 @@ namespace winrt::FFmpegInteropX::implementation } } - m_pReader->Start(); - isFirstSeek = false; isFirstSeekAfterStreamSwitch = false; mutexGuard.unlock(); @@ -1798,6 +1801,10 @@ namespace winrt::FFmpegInteropX::implementation mutexGuard.lock(); if (mss != nullptr) { + if (config->ReadAheadBufferEnabled()) + { + m_pReader->Start(); + } if (currentAudioStream && args.Request().StreamDescriptor() == currentAudioStream->StreamDescriptor()) { auto sample = currentAudioStream->GetNextSample(); diff --git a/Source/FFmpegMediaSource.h b/Source/FFmpegMediaSource.h index 8102095ee4..e39cda3343 100644 --- a/Source/FFmpegMediaSource.h +++ b/Source/FFmpegMediaSource.h @@ -93,6 +93,10 @@ namespace winrt::FFmpegInteropX::implementation ///The subtitle stream. IAsyncOperation> AddExternalSubtitleAsync(IRandomAccessStream stream); + ///Starts filling the read-ahead buffer, if enabled in the configuration. + ///Let the stream buffer fill before starting playback. + void StartBuffering(); + ///Gets the configuration that has been passed when creating the MSS instance. FFmpegInteropX::MediaSourceConfig Configuration(); diff --git a/Source/FFmpegReader.cpp b/Source/FFmpegReader.cpp index 794270b5e7..d472d84452 100644 --- a/Source/FFmpegReader.cpp +++ b/Source/FFmpegReader.cpp @@ -38,26 +38,24 @@ FFmpegReader::~FFmpegReader() void FFmpegReader::Start() { + std::lock_guard lock(mutex); + if (!isActive && (config.ReadAheadBufferEnabled() || config.ReadAheadBufferSize() > 0 || config.ReadAheadBufferDuration().count() > 0) && !config.as()->IsFrameGrabber) { - std::lock_guard lock(mutex); - if (!isEnabled && (config.ReadAheadBufferSize() > 0 || config.ReadAheadBufferDuration().count() > 0) && !config.as()->IsFrameGrabber) + sleepTimerTarget = new call([this](int value) { OnTimer(value); }); + if (!sleepTimerTarget) { - sleepTimerTarget = new call([this](int value) { OnTimer(value); }); - if (!sleepTimerTarget) - { - return; - } - - sleepTimer = new timer(100u, 0, sleepTimerTarget, true); - if (!sleepTimer) - { - delete sleepTimerTarget; - return; - } + return; + } - readTask = create_task([this]() { this->ReadDataLoop(); }); - isEnabled = true; + sleepTimer = new timer(100u, 0, sleepTimerTarget, true); + if (!sleepTimer) + { + delete sleepTimerTarget; + return; } + + readTask = create_task([this]() { this->ReadDataLoop(); }); + isActive = true; } } @@ -65,10 +63,10 @@ void FFmpegReader::Stop() { bool wait = false; { - std::lock_guard lock(mutex); - if (isEnabled) + std::lock_guard lock(mutex); + if (isActive) { - isEnabled = false; + isActive = false; wait = true; sleepTimer->stop(); @@ -105,7 +103,7 @@ void FFmpegReader::Stop() void FFmpegReader::FlushCodecs() { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); for (auto& stream : *sampleProviders) { if (stream) @@ -115,7 +113,7 @@ void FFmpegReader::FlushCodecs() void FFmpegReader::Flush() { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); for (auto& stream : *sampleProviders) { if (stream) @@ -128,6 +126,8 @@ HRESULT FFmpegReader::Seek(TimeSpan position, TimeSpan& actualPosition, TimeSpan { Stop(); + std::lock_guard lock(mutex); + auto hr = S_OK; if (readResult != 0) { @@ -424,41 +424,45 @@ void FFmpegReader::ReadDataLoop() while (true) { - { - std::lock_guard lock(mutex); - if (!isEnabled) - { - break; - } + // Read next packet + ret = ReadPacket(); - sleep = CheckNeedsSleep(sleep); - if (sleep) - { - isSleeping = true; - sleepTimer->start(); - break; - } + // Lock and check result + std::lock_guard lock(mutex); + if (!isActive) + { + // Stopped externally. No need to clean up. + break; } - - ret = ReadPacket(); - if (ret < 0) + else if (ret < 0 || !config.ReadAheadBufferEnabled()) { - std::lock_guard lock(mutex); - isEnabled = false; + // Self stop. Cleanup. + isActive = false; sleepTimer->stop(); delete sleepTimer; delete sleepTimerTarget; waitStreamEvent.set(); break; } + else + { + // Check if needs sleep + sleep = CheckNeedsSleep(sleep); + if (sleep) + { + isSleeping = true; + sleepTimer->start(); + break; + } + } } } void FFmpegInteropX::FFmpegReader::OnTimer(int value) { UNREFERENCED_PARAMETER(value); - std::lock_guard lock(mutex); - if (isEnabled) + std::lock_guard lock(mutex); + if (isActive) { readTask = create_task([this]() { ReadDataLoop(); }); isSleeping = false; @@ -482,9 +486,8 @@ bool FFmpegReader::CheckNeedsSleep(bool wasSleeping) sleep = true; fullStream = lastStream; } - // check if we can stop sleeping - if (sleep && !fullStream && !fullStream->IsBufferFull()) + else if (sleep && !fullStream && !fullStream->IsBufferFull()) { sleep = false; fullStream = nullptr; @@ -511,11 +514,11 @@ int FFmpegInteropX::FFmpegReader::ReadPacket() if (!avPacket) { - ret = E_OUTOFMEMORY; + return E_OUTOFMEMORY; } ret = av_read_frame(avFormatCtx, avPacket); - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); readResult = ret; if (ret < 0) @@ -568,8 +571,8 @@ int FFmpegReader::ReadPacketForStream(StreamBuffer* buffer) bool manual; { - std::lock_guard lock(mutex); - manual = !isEnabled; + std::lock_guard lock(mutex); + manual = !isActive; if (readResult < 0) { return readResult; @@ -600,14 +603,14 @@ int FFmpegReader::ReadPacketForStream(StreamBuffer* buffer) { task waitStreamTask; { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); if (!(buffer->IsEmpty())) { forceReadStream = -1; break; } - else if (readResult < 0 || !isEnabled) + else if (readResult < 0 || !isActive) { break; } diff --git a/Source/FFmpegReader.h b/Source/FFmpegReader.h index 00e4187525..67813d51dc 100644 --- a/Source/FFmpegReader.h +++ b/Source/FFmpegReader.h @@ -58,7 +58,7 @@ namespace FFmpegInteropX std::shared_ptr fullStream{ nullptr }; std::mutex mutex; - bool isEnabled = false; + bool isActive = false; bool isSleeping = false; int forceReadStream = 0; int readResult = 0; diff --git a/Source/MediaSampleProvider.cpp b/Source/MediaSampleProvider.cpp index 34d9d996af..5976d3ed15 100644 --- a/Source/MediaSampleProvider.cpp +++ b/Source/MediaSampleProvider.cpp @@ -347,7 +347,7 @@ void MediaSampleProvider::QueuePacket(AVPacket* packet) bool MediaSampleProvider::IsBufferFull() { - return packetBuffer->IsFull(this); + return IsEnabled() && packetBuffer->IsFull(this); } void MediaSampleProvider::Flush(bool flushBuffers) diff --git a/Source/MediaSourceConfig.cpp b/Source/MediaSourceConfig.cpp index b061fc8202..a65314ad49 100644 --- a/Source/MediaSourceConfig.cpp +++ b/Source/MediaSourceConfig.cpp @@ -33,14 +33,14 @@ namespace winrt::FFmpegInteropX::implementation m_MaxAudioThreads = 2; m_MaxSupportedPlaybackRate = 4.0; - m_StreamBufferSize = 16384; + m_FileStreamReadSize = 16384; m_FFmpegOptions = PropertySet(); m_DefaultBufferTime = TimeSpan{ 0 }; m_DefaultBufferTimeUri = TimeSpan{ 50000000 }; - + m_ReadAheadBufferEnabled = true; m_ReadAheadBufferSize = 100 * 1024 * 1024; m_ReadAheadBufferDuration = TimeSpan{ 600000000 }; @@ -273,12 +273,22 @@ namespace winrt::FFmpegInteropX::implementation uint32_t MediaSourceConfig::StreamBufferSize() { - return m_StreamBufferSize; + return m_FileStreamReadSize; } void MediaSourceConfig::StreamBufferSize(uint32_t value) { - m_StreamBufferSize = value; + m_FileStreamReadSize = value; + } + + uint32_t MediaSourceConfig::FileStreamReadSize() + { + return m_FileStreamReadSize; + } + + void MediaSourceConfig::FileStreamReadSize(uint32_t value) + { + m_FileStreamReadSize = value; } Windows::Foundation::Collections::PropertySet MediaSourceConfig::FFmpegOptions() @@ -311,6 +321,16 @@ namespace winrt::FFmpegInteropX::implementation m_DefaultBufferTimeUri = value; } + void MediaSourceConfig::ReadAheadBufferEnabled(bool value) + { + m_ReadAheadBufferEnabled = value; + } + + bool MediaSourceConfig::ReadAheadBufferEnabled() + { + return m_ReadAheadBufferEnabled; + } + void MediaSourceConfig::ReadAheadBufferSize(long long value) { m_ReadAheadBufferSize = value; diff --git a/Source/MediaSourceConfig.h b/Source/MediaSourceConfig.h index 680acf67c3..e5338d00b4 100644 --- a/Source/MediaSourceConfig.h +++ b/Source/MediaSourceConfig.h @@ -82,9 +82,14 @@ namespace winrt::FFmpegInteropX::implementation void MaxSupportedPlaybackRate(double value); ///The buffer size in bytes to use for Windows.Storage.Streams.IRandomAccessStream sources. + //[deprecated("Deprecated due to irritating name. Use ReadAheadBufferSize and ReadAheadBufferDuration instead.", deprecate, 1)] uint32_t StreamBufferSize(); void StreamBufferSize(uint32_t value); + ///The maximum number of bytes to read in one chunk for Windows.Storage.Streams.IRandomAccessStream sources. + uint32_t FileStreamReadSize(); + void FileStreamReadSize(uint32_t value); + ///Additional options to use when creating the ffmpeg AVFormatContext. Windows::Foundation::Collections::PropertySet FFmpegOptions(); void FFmpegOptions(Windows::Foundation::Collections::PropertySet const& value); @@ -101,6 +106,11 @@ namespace winrt::FFmpegInteropX::implementation void DefaultBufferTimeUri(winrt::Windows::Foundation::TimeSpan const& value); + ///Enables or disables the read-ahead buffer. + ///This value can be changed any time during playback. + bool ReadAheadBufferEnabled(); + void ReadAheadBufferEnabled(bool value); + ///The maximum number of bytes to buffer ahead per stream. ///This value can be changed any time during playback. long long ReadAheadBufferSize(); @@ -229,10 +239,11 @@ namespace winrt::FFmpegInteropX::implementation unsigned int m_MaxVideoThreads = 0; unsigned int m_MaxAudioThreads = 0; double m_MaxSupportedPlaybackRate = 0.0; - unsigned int m_StreamBufferSize = 0; + unsigned int m_FileStreamReadSize = 0; winrt::Windows::Foundation::Collections::PropertySet m_FFmpegOptions = {}; winrt::Windows::Foundation::TimeSpan m_DefaultBufferTime{}; winrt::Windows::Foundation::TimeSpan m_DefaultBufferTimeUri{}; + bool m_ReadAheadBufferEnabled = false; long long m_ReadAheadBufferSize = 0; winrt::Windows::Foundation::TimeSpan m_ReadAheadBufferDuration{}; bool m_AutoSelectForcedSubtitles = false; diff --git a/Source/StreamBuffer.h b/Source/StreamBuffer.h index 9441d1967a..17d344bce4 100644 --- a/Source/StreamBuffer.h +++ b/Source/StreamBuffer.h @@ -33,14 +33,13 @@ namespace FFmpegInteropX : config(config) { StreamIndex = streamIndex; - } int StreamIndex; void QueuePacket(AVPacket* packet) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); buffer.push_back(packet); bufferSize += packet->size; } @@ -100,13 +99,17 @@ namespace FFmpegInteropX bool IsEmpty() { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); return buffer.empty(); } bool IsFull(MediaSampleProvider* sampleProvider) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); + if (buffer.empty()) + { + return false; + } auto maxSize = config.ReadAheadBufferSize(); auto maxDuration = config.ReadAheadBufferDuration(); @@ -129,7 +132,7 @@ namespace FFmpegInteropX AVPacket* PopPacket() { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); AVPacket* packet = NULL; if (!buffer.empty()) @@ -144,7 +147,7 @@ namespace FFmpegInteropX AVPacket* PeekPacket() { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); AVPacket* packet = NULL; if (!buffer.empty()) @@ -157,13 +160,13 @@ namespace FFmpegInteropX AVPacket* PeekPacketIndex(int index) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); return buffer.at(index); } int TryFindPacketIndex(LONGLONG pts, bool requireKeyFrame, bool fastSeek) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); if (buffer.size() == 0) { @@ -222,7 +225,7 @@ namespace FFmpegInteropX void Flush() { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); while (!buffer.empty()) { auto packet = buffer.front(); @@ -235,7 +238,7 @@ namespace FFmpegInteropX void DropPackets(int count) { - std::lock_guard lock(mutex); + std::lock_guard lock(mutex); for (int i = 0; i < count; i++) { auto packet = buffer.front(); @@ -249,7 +252,7 @@ namespace FFmpegInteropX private: std::deque buffer; std::mutex mutex; - size_t bufferSize; - winrt::FFmpegInteropX::MediaSourceConfig config; + size_t bufferSize = 0; + winrt::FFmpegInteropX::MediaSourceConfig config{nullptr}; }; } From 4e833a0554341a21b9f13812992cd22805b93e8a Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Mon, 25 Jul 2022 23:12:45 +0200 Subject: [PATCH 15/28] Improve buffer and fix possible deadlock after mutex change --- Source/FFmpegMediaSource.cpp | 190 ++++++++++++++++++++--------------- Source/FFmpegReader.cpp | 30 +++--- Source/FFmpegReader.h | 5 +- Source/StreamBuffer.h | 118 ++++++++++++++++++---- 4 files changed, 223 insertions(+), 120 deletions(-) diff --git a/Source/FFmpegMediaSource.cpp b/Source/FFmpegMediaSource.cpp index 6bb090a5cb..087ac7045f 100644 --- a/Source/FFmpegMediaSource.cpp +++ b/Source/FFmpegMediaSource.cpp @@ -1285,6 +1285,7 @@ namespace winrt::FFmpegInteropX::implementation void FFmpegMediaSource::StartBuffering() { + std::lock_guard lock(mutex); m_pReader->Start(); } @@ -1690,58 +1691,65 @@ namespace winrt::FFmpegInteropX::implementation std::lock_guard lock(mutex); MediaStreamSourceStartingRequest request = args.Request(); - if (isFirstSeek && avHardwareContext) + try { - HRESULT hr = DirectXInteropHelper::GetDeviceManagerFromStreamSource(sender, &deviceManager); - if (SUCCEEDED(hr)) hr = D3D11VideoSampleProvider::InitializeHardwareDeviceContext(sender, avHardwareContext, &device, &deviceContext, deviceManager, &deviceHandle); - - if (SUCCEEDED(hr)) + if (isFirstSeek && avHardwareContext) { - // assign device and context - for (auto &stream : videoStreams) - { - // set device pointers to stream - hr = stream->SetHardwareDevice(device, deviceContext, avHardwareContext); + HRESULT hr = DirectXInteropHelper::GetDeviceManagerFromStreamSource(sender, &deviceManager); + if (SUCCEEDED(hr)) hr = D3D11VideoSampleProvider::InitializeHardwareDeviceContext(sender, avHardwareContext, &device, &deviceContext, deviceManager, &deviceHandle); - if (!SUCCEEDED(hr)) + if (SUCCEEDED(hr)) + { + // assign device and context + for (auto& stream : videoStreams) { - break; + // set device pointers to stream + hr = stream->SetHardwareDevice(device, deviceContext, avHardwareContext); + + if (!SUCCEEDED(hr)) + { + break; + } } } - } - else - { - // unref all hw device contexts - for (auto &stream : videoStreams) + else { - stream->FreeHardwareDevice(); + // unref all hw device contexts + for (auto& stream : videoStreams) + { + stream->FreeHardwareDevice(); + } + av_buffer_unref(&avHardwareContext); + SAFE_RELEASE(device); + SAFE_RELEASE(deviceContext); } - av_buffer_unref(&avHardwareContext); - SAFE_RELEASE(device); - SAFE_RELEASE(deviceContext); } - } - // Perform seek operation when MediaStreamSource received seek event from MediaElement - if (request.StartPosition() && request.StartPosition().Value().count() <= mediaDuration.count() && (!isFirstSeek || request.StartPosition().Value().count() > 0)) - { - if (currentVideoStream && !currentVideoStream->IsEnabled()) + // Perform seek operation when MediaStreamSource received seek event from MediaElement + if (request.StartPosition() && request.StartPosition().Value().count() <= mediaDuration.count() && (!isFirstSeek || request.StartPosition().Value().count() > 0)) { - currentVideoStream->EnableStream(); - } + if (currentVideoStream && !currentVideoStream->IsEnabled()) + { + currentVideoStream->EnableStream(); + } - if (currentAudioStream && !currentAudioStream->IsEnabled()) - { - currentAudioStream->EnableStream(); - } + if (currentAudioStream && !currentAudioStream->IsEnabled()) + { + currentAudioStream->EnableStream(); + } - TimeSpan actualPosition = request.StartPosition().Value(); - auto hr = Seek(request.StartPosition().Value(), actualPosition, true); - if (SUCCEEDED(hr)) - { - request.SetActualStartPosition(actualPosition); + TimeSpan actualPosition = request.StartPosition().Value(); + auto hr = Seek(request.StartPosition().Value(), actualPosition, true); + if (SUCCEEDED(hr)) + { + request.SetActualStartPosition(actualPosition); + } } } + catch (...) + { + DebugMessage(L"Exception in OnStarting()!"); + } isFirstSeek = false; isFirstSeekAfterStreamSwitch = false; @@ -1751,28 +1759,36 @@ namespace winrt::FFmpegInteropX::implementation { UNREFERENCED_PARAMETER(sender); std::lock_guard lock(mutex); - if (mss != nullptr) + + try { - if (config->ReadAheadBufferEnabled()) + if (mss != nullptr) { - m_pReader->Start(); - } - if (currentAudioStream && args.Request().StreamDescriptor() == currentAudioStream->StreamDescriptor()) - { - auto sample = currentAudioStream->GetNextSample(); - args.Request().Sample(sample); - } - else if (currentVideoStream && args.Request().StreamDescriptor() == currentVideoStream->StreamDescriptor()) - { - CheckVideoDeviceChanged(); - auto sample = currentVideoStream->GetNextSample(); - args.Request().Sample(sample); - } - else - { - args.Request().Sample(nullptr); + if (config->ReadAheadBufferEnabled()) + { + m_pReader->Start(); + } + if (currentAudioStream && args.Request().StreamDescriptor() == currentAudioStream->StreamDescriptor()) + { + auto sample = currentAudioStream->GetNextSample(); + args.Request().Sample(sample); + } + else if (currentVideoStream && args.Request().StreamDescriptor() == currentVideoStream->StreamDescriptor()) + { + CheckVideoDeviceChanged(); + auto sample = currentVideoStream->GetNextSample(); + args.Request().Sample(sample); + } + else + { + args.Request().Sample(nullptr); + } } } + catch (...) + { + DebugMessage(L"Exception in OnSampleRequested()!"); + } } @@ -1864,46 +1880,54 @@ namespace winrt::FFmpegInteropX::implementation { UNREFERENCED_PARAMETER(sender); std::lock_guard lock(mutex); - m_pReader->Stop(); - m_pReader->Flush(); - if (currentAudioStream && args.Request().OldStreamDescriptor() == currentAudioStream->StreamDescriptor()) + try { - if (!currentAudioEffects.empty()) + m_pReader->Stop(); + m_pReader->Flush(); + + if (currentAudioStream && args.Request().OldStreamDescriptor() == currentAudioStream->StreamDescriptor()) { - currentAudioStream->DisableFilters(); + if (!currentAudioEffects.empty()) + { + currentAudioStream->DisableFilters(); + } + currentAudioStream->DisableStream(); + currentAudioStream = nullptr; + } + if (currentVideoStream && args.Request().OldStreamDescriptor() == currentVideoStream->StreamDescriptor()) + { + currentVideoStream->DisableStream(); + currentVideoStream = nullptr; } - currentAudioStream->DisableStream(); - currentAudioStream = nullptr; - } - if (currentVideoStream && args.Request().OldStreamDescriptor() == currentVideoStream->StreamDescriptor()) - { - currentVideoStream->DisableStream(); - currentVideoStream = nullptr; - } - for (auto &stream : audioStreams) - { - if (stream->StreamDescriptor() == args.Request().NewStreamDescriptor()) + for (auto& stream : audioStreams) { - currentAudioStream = stream; - currentAudioStream->EnableStream(); - if (!currentAudioEffects.empty()) + if (stream->StreamDescriptor() == args.Request().NewStreamDescriptor()) { - currentAudioStream->SetFilters(currentAudioEffects); + currentAudioStream = stream; + currentAudioStream->EnableStream(); + if (!currentAudioEffects.empty()) + { + currentAudioStream->SetFilters(currentAudioEffects); + } } } - } - for (auto &stream : videoStreams) - { - if (stream->StreamDescriptor() == args.Request().NewStreamDescriptor()) + for (auto& stream : videoStreams) { - currentVideoStream = stream; - currentVideoStream->EnableStream(); + if (stream->StreamDescriptor() == args.Request().NewStreamDescriptor()) + { + currentVideoStream = stream; + currentVideoStream->EnableStream(); + } } - } - isFirstSeekAfterStreamSwitch = config->FastSeekSmartStreamSwitching(); + isFirstSeekAfterStreamSwitch = config->FastSeekSmartStreamSwitching(); + } + catch (...) + { + DebugMessage(L"Exception in OnSwitchStreamsRequested()!"); + } } HRESULT FFmpegMediaSource::Seek(const TimeSpan& position, TimeSpan& actualPosition, bool allowFastSeek) diff --git a/Source/FFmpegReader.cpp b/Source/FFmpegReader.cpp index d472d84452..04c4f9fc17 100644 --- a/Source/FFmpegReader.cpp +++ b/Source/FFmpegReader.cpp @@ -70,8 +70,6 @@ void FFmpegReader::Stop() wait = true; sleepTimer->stop(); - delete sleepTimer; - delete sleepTimerTarget; } } @@ -90,6 +88,8 @@ void FFmpegReader::Stop() Sleep(1); } } + delete sleepTimer; + delete sleepTimerTarget; } if (forceReadStream != -1) @@ -101,9 +101,14 @@ void FFmpegReader::Stop() fullStream = nullptr; } -void FFmpegReader::FlushCodecs() +void FFmpegReader::Flush() { std::lock_guard lock(mutex); + FlushCodecsAndBuffers(); +} + +void FFmpegReader::FlushCodecs() +{ for (auto& stream : *sampleProviders) { if (stream) @@ -111,9 +116,8 @@ void FFmpegReader::FlushCodecs() } } -void FFmpegReader::Flush() +void FFmpegReader::FlushCodecsAndBuffers() { - std::lock_guard lock(mutex); for (auto& stream : *sampleProviders) { if (stream) @@ -137,7 +141,7 @@ HRESULT FFmpegReader::Seek(TimeSpan position, TimeSpan& actualPosition, TimeSpan auto isForwardSeek = position > currentPosition; - if (isForwardSeek && TrySeekBuffered(position, actualPosition, fastSeek, videoStream, audioStream)) + if (isForwardSeek && TrySeekBuffered(position, actualPosition, fastSeek, isForwardSeek, videoStream, audioStream)) { // all good DebugMessage(L"BufferedSeek!\n"); @@ -160,7 +164,7 @@ HRESULT FFmpegReader::Seek(TimeSpan position, TimeSpan& actualPosition, TimeSpan else { // Flush all active streams with buffers - Flush(); + FlushCodecsAndBuffers(); } } @@ -224,7 +228,7 @@ HRESULT FFmpegReader::SeekFast(TimeSpan position, TimeSpan& actualPosition, Time else { // Flush all active streams with buffers - Flush(); + FlushCodecsAndBuffers(); // get and apply keyframe position for fast seeking TimeSpan timestampVideo; @@ -269,7 +273,7 @@ HRESULT FFmpegReader::SeekFast(TimeSpan position, TimeSpan& actualPosition, Time else { // Flush all active streams with buffers - Flush(); + FlushCodecsAndBuffers(); // get updated timestamp hr = videoStream->GetNextPacketTimestamp(timestampVideo, timestampVideoDuration); @@ -307,7 +311,7 @@ HRESULT FFmpegReader::SeekFast(TimeSpan position, TimeSpan& actualPosition, Time else { // Flush all active streams with buffers - Flush(); + FlushCodecsAndBuffers(); // Now drop all packets until desired keyframe position videoStream->SkipPacketsUntilTimestamp(timestampVideo); @@ -344,7 +348,7 @@ HRESULT FFmpegReader::SeekFast(TimeSpan position, TimeSpan& actualPosition, Time return hr; } -bool FFmpegReader::TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, bool fastSeek, std::shared_ptr videoStream, std::shared_ptr audioStream) +bool FFmpegReader::TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, bool fastSeek, bool isForwardSeek, std::shared_ptr videoStream, std::shared_ptr audioStream) { bool result = true; int vIndex = -1; @@ -355,7 +359,7 @@ bool FFmpegReader::TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, if (videoStream) { auto pts = videoStream->ConvertPosition(targetPosition); - vIndex = videoStream->packetBuffer->TryFindPacketIndex(pts, true, fastSeek); + vIndex = videoStream->packetBuffer->TryFindPacketIndex(pts, true, fastSeek, isForwardSeek); result &= vIndex >= 0; if (result && fastSeek) @@ -371,7 +375,7 @@ bool FFmpegReader::TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, if (result && audioStream) { auto pts = audioStream->ConvertPosition(targetPosition); - aIndex = audioStream->packetBuffer->TryFindPacketIndex(pts, false, fastSeek); + aIndex = audioStream->packetBuffer->TryFindPacketIndex(pts, false, fastSeek, isForwardSeek); result &= aIndex >= 0; } diff --git a/Source/FFmpegReader.h b/Source/FFmpegReader.h index 67813d51dc..f443fa58c9 100644 --- a/Source/FFmpegReader.h +++ b/Source/FFmpegReader.h @@ -44,11 +44,12 @@ namespace FFmpegInteropX private: - bool TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, bool fastSeek, std::shared_ptr videoStream, std::shared_ptr audioStream); + bool TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, bool fastSeek, bool isForwardSeek, std::shared_ptr videoStream, std::shared_ptr audioStream); HRESULT SeekFast(TimeSpan position, TimeSpan& actualPosition, TimeSpan currentPosition, std::shared_ptr videoStream, std::shared_ptr audioStream); void OnTimer(int value); void ReadDataLoop(); void FlushCodecs(); + void FlushCodecsAndBuffers(); bool CheckNeedsSleep(bool wasSleeping); AVFormatContext* avFormatCtx; @@ -57,7 +58,7 @@ namespace FFmpegInteropX std::shared_ptr lastStream{ nullptr }; std::shared_ptr fullStream{ nullptr }; - std::mutex mutex; + std::recursive_mutex mutex; bool isActive = false; bool isSleeping = false; int forceReadStream = 0; diff --git a/Source/StreamBuffer.h b/Source/StreamBuffer.h index 17d344bce4..4463c057aa 100644 --- a/Source/StreamBuffer.h +++ b/Source/StreamBuffer.h @@ -164,7 +164,7 @@ namespace FFmpegInteropX return buffer.at(index); } - int TryFindPacketIndex(LONGLONG pts, bool requireKeyFrame, bool fastSeek) + int TryFindPacketIndex(LONGLONG pts, bool requireKeyFrame, bool fastSeek, bool isForwardSeek) { std::lock_guard lock(mutex); @@ -175,52 +175,126 @@ namespace FFmpegInteropX auto firstPacket = buffer.front(); auto lastPacket = buffer.back(); - auto firstPts = firstPacket->pts != AV_NOPTS_VALUE ? firstPacket->pts : firstPacket->dts; - auto lastPts = lastPacket->pts != AV_NOPTS_VALUE ? lastPacket->pts : lastPacket->dts; + auto firstPts = GetTimestamp(firstPacket); + auto lastPts = GetTimestamp(lastPacket); if (firstPts != AV_NOPTS_VALUE && lastPts != AV_NOPTS_VALUE && (firstPts > pts || lastPts < pts)) { return -1; } - bool hasEnd = false; + if (requireKeyFrame && fastSeek) + { + return TryFindClosestKeyframe(pts, isForwardSeek); + } + else + { + return TryFindClosestPacket(pts); + } + } + + int TryFindClosestPacket(long long target) + { int index = 0; int result = -1; for (auto packet : buffer) { - if (packet->pts != AV_NOPTS_VALUE) + auto pts = GetTimestamp(packet); + if (pts != AV_NOPTS_VALUE) { - // with fast seek, we can select a key frame behind seek target - if (packet->pts <= pts || (fastSeek && result == -1)) + if (pts + packet->duration >= target) { - if (!requireKeyFrame || packet->flags & AV_PKT_FLAG_KEY) - { - result = index; - } + result = index; + break; } + } + index++; + } + return result; + } - if (packet->pts + packet->duration >= pts && (!fastSeek || result > -1)) + int TryFindClosestKeyframe(long long target, bool isForwardSeek) + { + bool hasTarget = false; + int index = 0; + int packetBeforeIndex = -1; + int packetAfterIndex = -1; + long long packetBeforePts = -1; + long long packetAfterPts = -1; + for (auto packet : buffer) + { + auto pts = GetTimestamp(packet); + if (pts != AV_NOPTS_VALUE) + { + if (pts <= target && packet->flags & AV_PKT_FLAG_KEY) { - hasEnd = true; - break; + packetBeforeIndex = index; + packetBeforePts = pts; + } + + if (pts + packet->duration >= target) + { + hasTarget = true; + if (packet->flags & AV_PKT_FLAG_KEY) + { + packetAfterIndex = index; + packetAfterPts = pts; + break; + } } } index++; } - if (hasEnd && result == -1 && !fastSeek) + if (hasTarget) { - // no key frame between current frame and target position. need to decode from here. - result = 0; - } + if (packetBeforeIndex >= 0 && packetAfterIndex >= 0) + { + // keyframes before and after found. select closest. + auto diffBefore = target - packetBeforePts; + auto diffAfter = packetAfterPts - target; + if (diffBefore <= diffAfter) + { + return packetBeforeIndex; + } + else + { + return packetAfterIndex; + } + } + else if (packetBeforeIndex >= 0) + { + // only keyframe before position found. return it. + return packetBeforeIndex; + } + else + { + // only keyframe after position found. use it or continue from current position. + auto diffCurrent = target - GetTimestamp(buffer[0]); + auto diffAfter = packetAfterPts - target; + if (diffCurrent < diffAfter && !isForwardSeek) + { + return 0; + } + else + { + return packetAfterIndex; + } + } - if (result >= 0 && !hasEnd) + // no key frame between current frame and target position + return -1; + } + else { - // key frames found but buffer range not up to target position - result = -1; + // target not found in buffer range + return -1; } + } - return result; + long long GetTimestamp(AVPacket* packet) + { + return packet->pts != AV_NOPTS_VALUE ? packet->pts : packet->dts; } void Flush() From 5d41768fe982887ea05b024d5a7cea9017105def Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Sat, 30 Jul 2022 00:07:42 +0200 Subject: [PATCH 16/28] Improve DASH fast seeking --- Source/FFmpegReader.cpp | 12 +++---- Source/StreamBuffer.h | 75 ++++++++++++++++++++++++++++------------- 2 files changed, 56 insertions(+), 31 deletions(-) diff --git a/Source/FFmpegReader.cpp b/Source/FFmpegReader.cpp index fa931d2307..5f727f0e62 100644 --- a/Source/FFmpegReader.cpp +++ b/Source/FFmpegReader.cpp @@ -360,23 +360,21 @@ bool FFmpegReader::TrySeekBuffered(TimeSpan position, TimeSpan& actualPosition, if (videoStream) { auto pts = videoStream->ConvertPosition(targetPosition); - vIndex = videoStream->packetBuffer->TryFindPacketIndex(pts, true, fastSeek, isForwardSeek); + LONGLONG resultPts = pts; + vIndex = videoStream->packetBuffer->TryFindPacketIndex(pts, resultPts, true, fastSeek, isForwardSeek); result &= vIndex >= 0; if (result && fastSeek) { - auto targetPacket = videoStream->packetBuffer->PeekPacketIndex(vIndex); - if (targetPacket->pts != AV_NOPTS_VALUE) - { - targetPosition = videoStream->ConvertPosition(targetPacket->pts); - } + targetPosition = videoStream->ConvertPosition(resultPts); } } if (result && audioStream) { auto pts = audioStream->ConvertPosition(targetPosition); - aIndex = audioStream->packetBuffer->TryFindPacketIndex(pts, false, fastSeek, isForwardSeek); + LONGLONG resultPts = pts; + aIndex = audioStream->packetBuffer->TryFindPacketIndex(pts, resultPts, false, fastSeek, isForwardSeek); result &= aIndex >= 0; } diff --git a/Source/StreamBuffer.h b/Source/StreamBuffer.h index 4463c057aa..f4db894cc1 100644 --- a/Source/StreamBuffer.h +++ b/Source/StreamBuffer.h @@ -164,7 +164,7 @@ namespace FFmpegInteropX return buffer.at(index); } - int TryFindPacketIndex(LONGLONG pts, bool requireKeyFrame, bool fastSeek, bool isForwardSeek) + int TryFindPacketIndex(LONGLONG pts, LONGLONG& resultPts, bool requireKeyFrame, bool fastSeek, bool isForwardSeek) { std::lock_guard lock(mutex); @@ -183,9 +183,9 @@ namespace FFmpegInteropX return -1; } - if (requireKeyFrame && fastSeek) + if (requireKeyFrame) { - return TryFindClosestKeyframe(pts, isForwardSeek); + return TryFindClosestKeyframe(pts, isForwardSeek, fastSeek, resultPts); } else { @@ -213,7 +213,7 @@ namespace FFmpegInteropX return result; } - int TryFindClosestKeyframe(long long target, bool isForwardSeek) + int TryFindClosestKeyframe(long long target, bool isForwardSeek, bool fastSeek, LONGLONG& resultPts) { bool hasTarget = false; int index = 0; @@ -221,9 +221,15 @@ namespace FFmpegInteropX int packetAfterIndex = -1; long long packetBeforePts = -1; long long packetAfterPts = -1; + long long lastPacketPts = AV_NOPTS_VALUE; for (auto packet : buffer) { auto pts = GetTimestamp(packet); + if (pts == AV_NOPTS_VALUE && packet->flags & AV_PKT_FLAG_KEY) + { + // in some streams, key frames do not have pts/dts. use previous packet value instead. + pts = lastPacketPts; + } if (pts != AV_NOPTS_VALUE) { if (pts <= target && packet->flags & AV_PKT_FLAG_KEY) @@ -243,47 +249,68 @@ namespace FFmpegInteropX } } } + lastPacketPts = pts; index++; } if (hasTarget) { - if (packetBeforeIndex >= 0 && packetAfterIndex >= 0) + if (!fastSeek) { - // keyframes before and after found. select closest. - auto diffBefore = target - packetBeforePts; - auto diffAfter = packetAfterPts - target; - if (diffBefore <= diffAfter) + // no fast seek: use packet before or decode from current position + if (packetBeforeIndex >= 0) { return packetBeforeIndex; } + else if (hasTarget) + { + return 0; + } else { - return packetAfterIndex; + return -1; } } - else if (packetBeforeIndex >= 0) - { - // only keyframe before position found. return it. - return packetBeforeIndex; - } else { - // only keyframe after position found. use it or continue from current position. - auto diffCurrent = target - GetTimestamp(buffer[0]); - auto diffAfter = packetAfterPts - target; - if (diffCurrent < diffAfter && !isForwardSeek) + if (packetBeforeIndex >= 0 && packetAfterIndex >= 0) { - return 0; + // keyframes before and after found. select closest. + auto diffBefore = target - packetBeforePts; + auto diffAfter = packetAfterPts - target; + if (diffBefore <= diffAfter) + { + resultPts = packetBeforePts; + return packetBeforeIndex; + } + else + { + resultPts = packetAfterPts; + return packetAfterIndex; + } + } + else if (packetBeforeIndex >= 0) + { + // only keyframe before position found. return it. + resultPts = packetBeforePts; + return packetBeforeIndex; } else { - return packetAfterIndex; + // only keyframe after position found. use it or continue from current position. + auto diffCurrent = target - GetTimestamp(buffer[0]); + auto diffAfter = packetAfterPts - target; + if (diffCurrent < diffAfter && !isForwardSeek) + { + return 0; + } + else + { + resultPts = packetAfterPts; + return packetAfterIndex; + } } } - - // no key frame between current frame and target position - return -1; } else { From efd81c9f21e572172a522577441f0fac4e94b45c Mon Sep 17 00:00:00 2001 From: Lukas Fellechner Date: Thu, 18 Aug 2022 00:07:09 +0200 Subject: [PATCH 17/28] Make stream buffer configurable in samples --- Samples/MediaPlayerCPP/MainPage.xaml | 6 +++++- Samples/MediaPlayerCPP/MainPage.xaml.cpp | 10 ++++++++++ Samples/MediaPlayerCPP/MainPage.xaml.h | 4 ++++ Samples/MediaPlayerCS/MainPage.xaml | 4 ++++ Samples/MediaPlayerCS/MainPage.xaml.cs | 25 +++++++++++++++++------- Source/FFmpegInteropX.idl | 2 +- 6 files changed, 42 insertions(+), 9 deletions(-) diff --git a/Samples/MediaPlayerCPP/MainPage.xaml b/Samples/MediaPlayerCPP/MainPage.xaml index 231ad02cf8..48fcdd0226 100644 --- a/Samples/MediaPlayerCPP/MainPage.xaml +++ b/Samples/MediaPlayerCPP/MainPage.xaml @@ -1,4 +1,4 @@ -