Stream buffer implementation (#270)

Add async read-ahead buffer for streaming scenarios
ffmpeginteropx · Oct 20, 2022 · 71b1a76 · 71b1a76
1 parent ffe72fa
commit 71b1a76
Show file tree

Hide file tree

Showing 33 changed files with 1,847 additions and 1,387 deletions.
diff --git a/Samples/MediaPlayerCPP/MainPage.xaml b/Samples/MediaPlayerCPP/MainPage.xaml
@@ -60,6 +60,10 @@
 
                         <ToggleSwitch Header="Fast Seeking (keyframe seeking requires PlaybackSession property to be assigned)" OffContent="OFF" OnContent="ON" IsOn="{x:Bind Mode=TwoWay, Path=Config.FastSeek}" Margin="0,20,0,0" />
 
+                        <ToggleSwitch Header="Read-ahead buffering" OffContent="OFF" OnContent="ON" IsOn="{x:Bind Mode=TwoWay, Path=Config.ReadAheadBufferEnabled}" Margin="0,20,0,0" />
+                        <Slider Minimum="1" Maximum="100" SnapsTo="StepValues" TickFrequency="10" StepFrequency="1" Header="Read-ahead buffer duration (s)" Value="{x:Bind Path=Config.ReadAheadBufferDuration, Mode=TwoWay, Converter={StaticResource TimeSpanToDoubleConverter}}" />
+                        <Slider Minimum="1" Maximum="100" SnapsTo="StepValues" TickFrequency="10" StepFrequency="1" Header="Read-ahead buffer size (MB)" Value="{x:Bind GetBufferSizeMB(), BindBack=SetBufferSizeMB, Mode=TwoWay}" />
+
                         <Button Content="Extract Frame" Click="ExtractFrame" Margin="0,20,10,10" />
                         <ToggleSwitch x:Name="grabFrameExactSeek" Header="Exact Seek for Extract Frame" OffContent="OFF" OnContent="ON" IsOn="True"/>
 

diff --git a/Samples/MediaPlayerCPP/MainPage.xaml.cpp b/Samples/MediaPlayerCPP/MainPage.xaml.cpp
@@ -97,6 +97,20 @@ task<void> MainPage::TryOpenLastFile()
     }
 }
 
+task<void> MainPage::TryOpenLastUri()
+{
+    try
+    {
+        //Try open last uri
+        auto uri = (String^)ApplicationData::Current->LocalSettings->Values->Lookup("LastUri");
+        co_await OpenUriStream(uri);
+    }
+    catch (Exception^ ex)
+    {
+        DisplayErrorMessage(ex->Message);
+    }
+}
+
 void MainPage::OpenLocalFile(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e)
 {
     OpenLocalFile();
@@ -133,16 +147,15 @@ task<void> MainPage::OpenLocalFile(StorageFile^ file)
     // Open StorageFile as IRandomAccessStream to be passed to FFmpegMediaSource
     try
     {
+        StorageApplicationPermissions::FutureAccessList->Clear();
+        StorageApplicationPermissions::FutureAccessList->Add(file);
+
         auto stream = co_await file->OpenAsync(FileAccessMode::Read);
         // Instantiate FFmpegMediaSource using the opened local file stream
         FFmpegMSS = co_await FFmpegMediaSource::CreateFromStreamAsync(stream, Config);
 
-        StorageApplicationPermissions::FutureAccessList->Clear();
-        StorageApplicationPermissions::FutureAccessList->Add(file);
-
         playbackItem = FFmpegMSS->CreateMediaPlaybackItem();
 
-
         // Pass MediaPlaybackItem to Media Element
         mediaPlayer->Source = playbackItem;
 
@@ -181,8 +194,7 @@ task<void> MainPage::OpenUriStream(Platform::String^ uri)
     // https://www.ffmpeg.org/ffmpeg-formats.html
     // 
     // If format cannot be detected, try to increase probesize, max_probe_packets and analyzeduration!
-
-    // Below are some sample options that you can set to configure RTSP streaming
+
     //Config->FFmpegOptions->Insert("rtsp_flags", "prefer_tcp");
     Config->FFmpegOptions->Insert("stimeout", 1000000);
     Config->FFmpegOptions->Insert("timeout", 1000000);
@@ -191,6 +203,8 @@ task<void> MainPage::OpenUriStream(Platform::String^ uri)
     mediaPlayer->Source = nullptr;
     try
     {
+        ApplicationData::Current->LocalSettings->Values->Insert("LastUri", uri);
+
         FFmpegMSS = co_await FFmpegMediaSource::CreateFromUriAsync(uri, Config);
         playbackItem = FFmpegMSS->CreateMediaPlaybackItem();
 
@@ -512,14 +526,38 @@ void MediaPlayerCPP::MainPage::OnKeyDown(Windows::UI::Core::CoreWindow^ sender,
         TryOpenLastFile();
     }
 
+    if (args->VirtualKey == Windows::System::VirtualKey::Enter && (Window::Current->CoreWindow->GetKeyState(Windows::System::VirtualKey::Shift) & Windows::UI::Core::CoreVirtualKeyStates::Down)
+        == Windows::UI::Core::CoreVirtualKeyStates::Down && ApplicationData::Current->LocalSettings->Values->HasKey("LastUri"))
+    {
+        TryOpenLastUri();
+    }
+
+    if (args->Handled)
+    {
+        return;
+    }
+
     if (args->VirtualKey == Windows::System::VirtualKey::V)
     {
         if (playbackItem && playbackItem->VideoTracks->Size > 1)
         {
-            playbackItem->VideoTracks->SelectedIndex =
+            bool reverse = (Window::Current->CoreWindow->GetKeyState(Windows::System::VirtualKey::Shift) & Windows::UI::Core::CoreVirtualKeyStates::Down) == Windows::UI::Core::CoreVirtualKeyStates::Down;
+            int index = reverse ?
+                (playbackItem->VideoTracks->SelectedIndex - 1) % playbackItem->VideoTracks->Size :
                 (playbackItem->VideoTracks->SelectedIndex + 1) % playbackItem->VideoTracks->Size;
+            playbackItem->VideoTracks->SelectedIndex = index;
         }
     }
+
+    if (args->VirtualKey == Windows::System::VirtualKey::Right && FFmpegMSS && mediaPlayer->PlaybackSession->CanSeek)
+    {
+        mediaPlayer->PlaybackSession->Position = TimeSpan{ mediaPlayer->PlaybackSession->Position.Duration + 50000000 };
+    }
+
+    if (args->VirtualKey == Windows::System::VirtualKey::Left && FFmpegMSS && mediaPlayer->PlaybackSession->CanSeek)
+    {
+        mediaPlayer->PlaybackSession->Position = TimeSpan{ mediaPlayer->PlaybackSession->Position.Duration - 50000000 };
+    }
 }
 
 
@@ -572,3 +610,13 @@ void MediaPlayerCPP::MainPage::ffmpegAudioFilters_KeyDown(Platform::Object^ send
         }
     }
 }
+
+double MediaPlayerCPP::MainPage::GetBufferSizeMB()
+{
+    return (double)Config->ReadAheadBufferSize / (1024*1024);
+}
+
+void MediaPlayerCPP::MainPage::SetBufferSizeMB(double value)
+{
+    Config->ReadAheadBufferSize = (long long)(value * (1024 * 1024));
+}
diff --git a/Samples/MediaPlayerCPP/MainPage.xaml.h b/Samples/MediaPlayerCPP/MainPage.xaml.h
@@ -1,4 +1,4 @@
-//*****************************************************************************
+//*****************************************************************************
 //
 //	Copyright 2015 Microsoft Corporation
 //
@@ -39,11 +39,15 @@ namespace MediaPlayerCPP
         property FFmpegInteropX::MediaSourceConfig^ Config;
         property FFmpegInteropX::VideoEffectConfiguration^ VideoEffectConfiguration;
 
+        double GetBufferSizeMB();
+        void SetBufferSizeMB(double value);
+
     private:
         void OpenLocalFile(Platform::Object^ sender, Windows::UI::Xaml::RoutedEventArgs^ e);
         task<void> OpenLocalFile();
         task<void> OpenLocalFile(Windows::Storage::StorageFile^ file);
         task<void> TryOpenLastFile();
+        task<void> TryOpenLastUri();
         void URIBoxKeyUp(Platform::Object^ sender, Windows::UI::Xaml::Input::KeyRoutedEventArgs^ e);
         task<void> OpenUriStream(Platform::String^ uri);
         void MediaFailed(Platform::Object^ sender, Windows::UI::Xaml::ExceptionRoutedEventArgs^ e);
@@ -81,5 +85,6 @@ namespace MediaPlayerCPP
         void ffmpegAudioFilters_KeyDown(Platform::Object^ sender, Windows::UI::Xaml::Input::KeyRoutedEventArgs^ e);
         void OnMediaOpened(Windows::Media::Playback::MediaPlayer^ sender, Platform::Object^ args);
         void OnMediaFailed(Windows::Media::Playback::MediaPlayer^ sender, Windows::Media::Playback::MediaPlayerFailedEventArgs^ args);
+
     };
 }
diff --git a/Samples/MediaPlayerCS/MainPage.xaml b/Samples/MediaPlayerCS/MainPage.xaml
@@ -62,6 +62,10 @@
 
                         <ToggleSwitch Header="Fast Seeking (keyframe seeking requires PlaybackSession property to be assigned)" OffContent="OFF" OnContent="ON" IsOn="{x:Bind Mode=TwoWay, Path=Config.FastSeek}" Margin="0,20,0,0" />
 
+                        <ToggleSwitch Header="Read-ahead buffering" OffContent="OFF" OnContent="ON" IsOn="{x:Bind Mode=TwoWay, Path=Config.ReadAheadBufferEnabled}" Margin="0,20,0,0" />
+                        <Slider Minimum="1" Maximum="100" SnapsTo="StepValues" TickFrequency="10" StepFrequency="1" Header="Read-ahead buffer duration (s)" Value="{x:Bind Path=Config.ReadAheadBufferDuration, Mode=TwoWay, Converter={StaticResource TimeSpanToDoubleConverter}}" />
+                        <Slider Minimum="1" Maximum="100" SnapsTo="StepValues" TickFrequency="10" StepFrequency="1" Header="Read-ahead buffer size (MB)" Value="{x:Bind GetBufferSizeMB(), BindBack=SetBufferSizeMB, Mode=TwoWay}" />
+
                         <Button Content="Extract Frame" Click="ExtractFrame"  Margin="0,20,10,10" />
                         <ToggleSwitch x:Name="grabFrameExactSeek" Header="Exact Seek for Extract Frame" OffContent="OFF" OnContent="ON" IsOn="True"/>
 

diff --git a/Samples/MediaPlayerCS/MainPage.xaml.cs b/Samples/MediaPlayerCS/MainPage.xaml.cs
@@ -93,14 +93,38 @@ private async void MainPage_KeyDown(CoreWindow sender, KeyEventArgs args)
             {
                 await TryOpenLastFile();
             }
+            if (args.VirtualKey == VirtualKey.Enter && (Window.Current.CoreWindow.GetKeyState(VirtualKey.Shift) & CoreVirtualKeyStates.Down)
+               == CoreVirtualKeyStates.Down && ApplicationData.Current.LocalSettings.Values.ContainsKey("LastUri"))
+            {
+                await TryOpenLastUri();
+            }
+
+            if (args.Handled)
+            {
+                return;
+            }
+
             if (args.VirtualKey == VirtualKey.V)
             {
                 if (playbackItem != null && playbackItem.VideoTracks.Count > 1)
                 {
-                    playbackItem.VideoTracks.SelectedIndex =
+                    bool reverse = (Window.Current.CoreWindow.GetKeyState(VirtualKey.Control) & CoreVirtualKeyStates.Down) == CoreVirtualKeyStates.Down;
+                    int index = reverse ?
+                        (playbackItem.VideoTracks.SelectedIndex - 1) % playbackItem.VideoTracks.Count :
                         (playbackItem.VideoTracks.SelectedIndex + 1) % playbackItem.VideoTracks.Count;
+                    playbackItem.VideoTracks.SelectedIndex = index;
                 }
             }
+
+            if (args.VirtualKey == VirtualKey.Right && FFmpegMSS != null && mediaPlayer.PlaybackSession.CanSeek)
+            {
+                mediaPlayer.PlaybackSession.Position += TimeSpan.FromSeconds(5);
+            }
+
+            if (args.VirtualKey == VirtualKey.Left && FFmpegMSS != null && mediaPlayer.PlaybackSession.CanSeek)
+            {
+                mediaPlayer.PlaybackSession.Position -= TimeSpan.FromSeconds(5);
+            }
         }
 
         private async void CodecChecker_CodecRequired(object sender, CodecRequiredEventArgs args)
@@ -152,6 +176,18 @@ private async Task TryOpenLastFile()
             {
             }
         }
+        private async Task TryOpenLastUri()
+        {
+            try
+            {
+                //Try open last uri
+                var uri = (string)ApplicationData.Current.LocalSettings.Values["LastUri"];
+                await OpenStreamUri(uri);
+            }
+            catch (Exception)
+            {
+            }
+        }
 
         public MediaSourceConfig Config { get; set; }
 
@@ -227,36 +263,43 @@ private async void URIBoxKeyUp(object sender, KeyRoutedEventArgs e)
                 // Mark event as handled to prevent duplicate event to re-triggered
                 e.Handled = true;
 
-                try
-                {
-                    // Set FFmpeg specific options:
-                    // https://www.ffmpeg.org/ffmpeg-protocols.html
-                    // https://www.ffmpeg.org/ffmpeg-formats.html
+                await OpenStreamUri(uri);
+            }
+        }
 
-                    // If format cannot be detected, try to increase probesize, max_probe_packets and analyzeduration!
+        private async Task OpenStreamUri(string uri)
+        {
+            try
+            {
+                ApplicationData.Current.LocalSettings.Values["LastUri"] = uri;
 
-                    // Below are some sample options that you can set to configure RTSP streaming
-                    // Config.FFmpegOptions.Add("rtsp_flags", "prefer_tcp");
-                    Config.FFmpegOptions.Add("stimeout", 1000000);
-                    Config.FFmpegOptions.Add("timeout", 1000000);
+                // Set FFmpeg specific options:
+                // https://www.ffmpeg.org/ffmpeg-protocols.html
+                // https://www.ffmpeg.org/ffmpeg-formats.html
 
-                    // Instantiate FFmpegMediaSource using the URI
-                    mediaPlayer.Source = null;
-                    FFmpegMSS = await FFmpegMediaSource.CreateFromUriAsync(uri, Config);
+                // If format cannot be detected, try to increase probesize, max_probe_packets and analyzeduration!
 
-                    var source = FFmpegMSS.CreateMediaPlaybackItem();
+                // Config.FFmpegOptions.Add("rtsp_flags", "prefer_tcp");
+                Config.FFmpegOptions.Add("stimeout", 1000000);
+                Config.FFmpegOptions.Add("timeout", 1000000);
 
-                    // Pass MediaStreamSource to Media Element
-                    mediaPlayer.Source = source;
+                // Instantiate FFmpegMediaSource using the URI
+                mediaPlayer.Source = null;
+                FFmpegMSS = await FFmpegMediaSource.CreateFromUriAsync(uri, Config);
 
-                    // Close control panel after opening media
-                    Splitter.IsPaneOpen = false;
+                if (AutoCreatePlaybackItem)
+                {
+                    CreatePlaybackItemAndStartPlaybackInternal();
                 }
-                catch (Exception ex)
+                else
                 {
-                    await DisplayErrorMessage(ex.Message);
+                    playbackItem = null;
                 }
             }
+            catch (Exception ex)
+            {
+                await DisplayErrorMessage(ex.Message);
+            }
         }
 
         private async void ExtractFrame(object sender, RoutedEventArgs e)
@@ -600,5 +643,15 @@ private void ffmpegAudioFilters_LostFocus(object sender, RoutedEventArgs e)
             Config.FFmpegAudioFilters = ffmpegAudioFilters.Text;
             FFmpegMSS?.SetFFmpegAudioFilters(ffmpegAudioFilters.Text);
         }
+
+        private double GetBufferSizeMB()
+        {
+            return Config.ReadAheadBufferSize / (1024 * 1024);
+        }
+
+        private long SetBufferSizeMB(double value)
+        {
+            return Config.ReadAheadBufferSize = (long)(value * (1024 * 1024));
+        }
     }
 }
diff --git a/Samples/MediaPlayerWinUI/App.xaml.cs b/Samples/MediaPlayerWinUI/App.xaml.cs
@@ -1,20 +1,4 @@
-using Microsoft.UI.Xaml;
-using Microsoft.UI.Xaml.Controls;
-using Microsoft.UI.Xaml.Controls.Primitives;
-using Microsoft.UI.Xaml.Data;
-using Microsoft.UI.Xaml.Input;
-using Microsoft.UI.Xaml.Media;
-using Microsoft.UI.Xaml.Navigation;
-using Microsoft.UI.Xaml.Shapes;
-using System;
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Runtime.InteropServices.WindowsRuntime;
-using Windows.ApplicationModel;
-using Windows.ApplicationModel.Activation;
-using Windows.Foundation;
-using Windows.Foundation.Collections;
+using Microsoft.UI.Xaml;
 
 // To learn more about WinUI, the WinUI project structure,
 // and more about our project templates, see: http://aka.ms/winui-project-info.

diff --git a/Samples/MediaPlayerWinUI/MainPage.xaml.cs b/Samples/MediaPlayerWinUI/MainPage.xaml.cs
@@ -1,13 +1,9 @@
 using Microsoft.UI.Xaml;
 using Microsoft.UI.Xaml.Controls;
-using Microsoft.UI.Xaml.Controls.Primitives;
 using Microsoft.UI.Xaml.Data;
 using Microsoft.UI.Xaml.Input;
-using Microsoft.UI.Xaml.Media;
-using Microsoft.UI.Xaml.Navigation;
 using System;
 using System.Collections.Generic;
-using System.IO;
 using System.Linq;
 using System.Runtime.InteropServices.WindowsRuntime;
 using System.Threading.Tasks;
@@ -89,14 +85,39 @@ private async void MainPage_KeyDown(object sender, KeyRoutedEventArgs args)
             {
                 await TryOpenLastFile();
             }
+
+            if (args.Key == VirtualKey.Enter && (Microsoft.UI.Input.InputKeyboardSource.GetKeyStateForCurrentThread(VirtualKey.Shift) & CoreVirtualKeyStates.Down)
+               == CoreVirtualKeyStates.Down && StorageApplicationPermissions.FutureAccessList.Entries.Count == 1)
+            {
+                await TryOpenLastFile();
+            }
+
+            if (args.Handled)
+            {
+                return;
+            }
+
             if (args.Key == VirtualKey.V && !args.Handled)
             {
                 if (playbackItem != null && playbackItem.VideoTracks.Count > 1)
                 {
-                    playbackItem.VideoTracks.SelectedIndex =
+                    bool reverse = (Microsoft.UI.Input.InputKeyboardSource.GetKeyStateForCurrentThread(VirtualKey.Control) & CoreVirtualKeyStates.Down) == CoreVirtualKeyStates.Down;
+                    int index = reverse ?
+                        (playbackItem.VideoTracks.SelectedIndex - 1) % playbackItem.VideoTracks.Count :
                         (playbackItem.VideoTracks.SelectedIndex + 1) % playbackItem.VideoTracks.Count;
+                    playbackItem.VideoTracks.SelectedIndex = index;
                 }
             }
+
+            if (args.Key == VirtualKey.Right && FFmpegMSS != null && mediaPlayer.PlaybackSession.CanSeek)
+            {
+                mediaPlayer.PlaybackSession.Position += TimeSpan.FromSeconds(5);
+            }
+
+            if (args.Key == VirtualKey.Left && FFmpegMSS != null && mediaPlayer.PlaybackSession.CanSeek)
+            {
+                mediaPlayer.PlaybackSession.Position -= TimeSpan.FromSeconds(5);
+            }
         }
 
         private void CodecChecker_CodecRequired(object sender, CodecRequiredEventArgs args)