Merge pull request #6088 from peppy/wasapi-low-latency

Add low latency initialisation support on windows by directly interacting with WASAPI

osu.Framework.Tests/Audio/BassTestComponents.cs

@@ -57,7 +57,7 @@ public void Add(params AudioComponent[] component)
 
         internal BassAudioMixer CreateMixer()
         {
-            var mixer = new BassAudioMixer(Mixer, "Test mixer");
+            var mixer = new BassAudioMixer(null, Mixer, "Test mixer");
             mixerComponents.AddItem(mixer);
             return mixer;
         }

osu.Framework/Audio/AudioManager.cs

@@ -108,6 +108,30 @@ public class AudioManager : AudioCollectionManager<AudioComponent>
             MaxValue = 1
         };
 
+        /// <summary>
+        /// Whether a global mixer is being used for audio routing.
+        /// For now, this is only the case on Windows when using shared mode WASAPI initialisation.
+        /// </summary>
+        public IBindable<bool> UsingGlobalMixer => usingGlobalMixer;
+
+        private readonly Bindable<bool> usingGlobalMixer = new BindableBool();
+
+        /// <summary>
+        /// If a global mixer is being used, this will be the BASS handle for it.
+        /// If non-null, all game mixers should be added to this mixer.
+        /// </summary>
+        /// <remarks>
+        /// When this is non-null, all mixers created via <see cref="CreateAudioMixer"/>
+        /// will themselves be added to the global mixer, which will handle playback itself.
+        ///
+        /// In this mode of operation, nested mixers will be created with the <see cref="BassFlags.Decode"/>
+        /// flag, meaning they no longer handle playback directly.
+        ///
+        /// An eventual goal would be to use a global mixer across all platforms as it can result
+        /// in more control and better playback performance.
+        /// </remarks>
+        internal readonly IBindable<int?> GlobalMixerHandle = new Bindable<int?>();
+
         public override bool IsLoaded => base.IsLoaded &&
                                          // bass default device is a null device (-1), not the actual system default.
                                          Bass.CurrentDevice != Bass.DefaultDevice;
@@ -146,7 +170,12 @@ public AudioManager(AudioThread audioThread, ResourceStore<byte[]> trackStore, R
 
             thread.RegisterManager(this);
 
-            AudioDevice.ValueChanged += onDeviceChanged;
+            AudioDevice.ValueChanged += _ => onDeviceChanged();
+            GlobalMixerHandle.ValueChanged += handle =>
+            {
+                onDeviceChanged();
+                usingGlobalMixer.Value = handle.NewValue.HasValue;
+            };
 
             AddItem(TrackMixer = createAudioMixer(null, nameof(TrackMixer)));
             AddItem(SampleMixer = createAudioMixer(null, nameof(SampleMixer)));
@@ -205,9 +234,9 @@ protected override void Dispose(bool disposing)
             base.Dispose(disposing);
         }
 
-        private void onDeviceChanged(ValueChangedEvent<string> args)
+        private void onDeviceChanged()
         {
-            scheduler.Add(() => setAudioDevice(args.NewValue));
+            scheduler.Add(() => setAudioDevice(AudioDevice.Value));
         }
 
         private void onDevicesChanged()
@@ -236,7 +265,7 @@ public AudioMixer CreateAudioMixer(string identifier = default) =>
 
         private AudioMixer createAudioMixer(AudioMixer fallbackMixer, string identifier)
         {
-            var mixer = new BassAudioMixer(fallbackMixer, identifier);
+            var mixer = new BassAudioMixer(this, fallbackMixer, identifier);
             AddItem(mixer);
             return mixer;
         }
@@ -312,7 +341,7 @@ private bool setAudioDevice(string deviceName = null)
             if (setAudioDevice(Bass.NoSoundDevice))
                 return true;
 
-            //we're fucked. even "No sound" device won't initialise.
+            // we're boned. even "No sound" device won't initialise.
             return false;
         }
 
@@ -365,7 +394,7 @@ protected virtual bool InitBass(int device)
                 return true;
 
             // this likely doesn't help us but also doesn't seem to cause any issues or any cpu increase.
-            Bass.UpdatePeriod = 5;
+            Bass.UpdatePeriod = 1;
 
             // reduce latency to a known sane minimum.
             Bass.DeviceBufferLength = 10;
@@ -390,7 +419,10 @@ protected virtual bool InitBass(int device)
             // See https://www.un4seen.com/forum/?topic=19601 for more information.
             Bass.Configure((ManagedBass.Configuration)70, false);
 
-            return AudioThread.InitDevice(device);
+            if (!thread.InitDevice(device))
+                return false;
+
+            return true;
         }
 
         private void syncAudioDevices()

osu.Framework/Audio/Mixing/Bass/BassAudioMixer.cs

@@ -22,6 +22,8 @@ namespace osu.Framework.Audio.Mixing.Bass
     /// </summary>
     internal class BassAudioMixer : AudioMixer, IBassAudio
     {
+        private readonly AudioManager? manager;
+
         /// <summary>
         /// The handle for this mixer.
         /// </summary>
@@ -42,11 +44,13 @@ internal class BassAudioMixer : AudioMixer, IBassAudio
         /// <summary>
         /// Creates a new <see cref="BassAudioMixer"/>.
         /// </summary>
+        /// <param name="manager">The game's audio manager.</param>
         /// <param name="fallbackMixer"><inheritdoc /></param>
         /// <param name="identifier">An identifier displayed on the audio mixer visualiser.</param>
-        public BassAudioMixer(AudioMixer? fallbackMixer, string identifier)
+        public BassAudioMixer(AudioManager? manager, AudioMixer? fallbackMixer, string identifier)
             : base(fallbackMixer, identifier)
         {
+            this.manager = manager;
             EnqueueAction(createMixer);
         }
 
@@ -248,7 +252,12 @@ public void UpdateDevice(int deviceIndex)
             if (Handle == 0)
                 createMixer();
             else
+            {
                 ManagedBass.Bass.ChannelSetDevice(Handle, deviceIndex);
+
+                if (manager?.GlobalMixerHandle.Value != null)
+                    BassMix.MixerAddChannel(manager.GlobalMixerHandle.Value.Value, Handle, BassFlags.MixerChanBuffer | BassFlags.MixerChanNoRampin);
+            }
         }
 
         protected override void UpdateState()
@@ -277,7 +286,9 @@ private void createMixer()
             if (!ManagedBass.Bass.GetDeviceInfo(ManagedBass.Bass.CurrentDevice, out var deviceInfo) || !deviceInfo.IsInitialized)
                 return;
 
-            Handle = BassMix.CreateMixerStream(frequency, 2, BassFlags.MixerNonStop);
+            Handle = manager?.GlobalMixerHandle.Value != null
+                ? BassMix.CreateMixerStream(frequency, 2, BassFlags.MixerNonStop | BassFlags.Decode)
+                : BassMix.CreateMixerStream(frequency, 2, BassFlags.MixerNonStop);
 
             if (Handle == 0)
                 return;
@@ -293,6 +304,9 @@ private void createMixer()
 
             Effects.BindCollectionChanged(onEffectsChanged, true);
 
+            if (manager?.GlobalMixerHandle.Value != null)
+                BassMix.MixerAddChannel(manager.GlobalMixerHandle.Value.Value, Handle, BassFlags.MixerChanBuffer | BassFlags.MixerChanNoRampin);
+
             ManagedBass.Bass.ChannelPlay(Handle);
         }
 

osu.Framework/Graphics/Visualisation/Audio/AudioChannelDisplay.cs

@@ -4,7 +4,9 @@
 using System;
 using ManagedBass;
 using ManagedBass.Mix;
+using osu.Framework.Allocation;
 using osu.Framework.Audio;
+using osu.Framework.Bindables;
 using osu.Framework.Graphics.Containers;
 using osu.Framework.Graphics.Shapes;
 using osu.Framework.Graphics.Sprites;
@@ -28,6 +30,7 @@ public partial class AudioChannelDisplay : CompositeDrawable
         private float maxPeak = float.MinValue;
         private double lastMaxPeakTime;
         private readonly bool isOutputChannel;
+        private IBindable<bool> usingGlobalMixer = null!;
 
         public AudioChannelDisplay(int channelHandle, bool isOutputChannel = false)
         {
@@ -100,13 +103,19 @@ public AudioChannelDisplay(int channelHandle, bool isOutputChannel = false)
             };
         }
 
+        [BackgroundDependencyLoader]
+        private void load(AudioManager audioManager)
+        {
+            usingGlobalMixer = audioManager.UsingGlobalMixer.GetBoundCopy();
+        }
+
         protected override void Update()
         {
             base.Update();
 
             float[] levels = new float[2];
 
-            if (isOutputChannel)
+            if (isOutputChannel && !usingGlobalMixer.Value)
                 Bass.ChannelGetLevel(ChannelHandle, levels, 1 / 1000f * sample_window, LevelRetrievalFlags.Stereo);
             else
                 BassMix.ChannelGetLevel(ChannelHandle, levels, 1 / 1000f * sample_window, LevelRetrievalFlags.Stereo);

osu.Framework/Threading/AudioThread.cs

@@ -7,7 +7,10 @@
 using System.Diagnostics;
 using System.Linq;
 using ManagedBass;
+using ManagedBass.Mix;
+using ManagedBass.Wasapi;
 using osu.Framework.Audio;
+using osu.Framework.Bindables;
 using osu.Framework.Development;
 using osu.Framework.Platform.Linux.Native;
 
@@ -73,12 +76,16 @@ internal void RegisterManager(AudioManager manager)
 
                 managers.Add(manager);
             }
+
+            manager.GlobalMixerHandle.BindTo(globalMixerHandle);
         }
 
         internal void UnregisterManager(AudioManager manager)
         {
             lock (managers)
                 managers.Remove(manager);
+
+            manager.GlobalMixerHandle.UnbindFrom(globalMixerHandle);
         }
 
         protected override void OnExit()
@@ -109,22 +116,35 @@ protected override void OnExit()
                 FreeDevice(d);
         }
 
-        internal static bool InitDevice(int deviceId)
+        #region BASS Initialisation
+
+        // TODO: All this bass init stuff should probably not be in this class.
+
+        private WasapiProcedure? wasapiProcedure;
+        private WasapiNotifyProcedure? wasapiNotifyProcedure;
+
+        /// <summary>
+        /// If a global mixer is being used, this will be the BASS handle for it.
+        /// If non-null, all game mixers should be added to this mixer.
+        /// </summary>
+        private readonly Bindable<int?> globalMixerHandle = new Bindable<int?>();
+
+        internal bool InitDevice(int deviceId)
         {
             Debug.Assert(ThreadSafety.IsAudioThread);
             Trace.Assert(deviceId != -1); // The real device ID should always be used, as the -1 device has special cases which are hard to work with.
 
             // Try to initialise the device, or request a re-initialise.
-            if (Bass.Init(deviceId, Flags: (DeviceInitFlags)128)) // 128 == BASS_DEVICE_REINIT
-            {
-                initialised_devices.Add(deviceId);
-                return true;
-            }
+            if (!Bass.Init(deviceId, Flags: (DeviceInitFlags)128)) // 128 == BASS_DEVICE_REINIT
+                return false;
+
+            attemptWasapiInitialisation();
 
-            return false;
+            initialised_devices.Add(deviceId);
+            return true;
         }
 
-        internal static void FreeDevice(int deviceId)
+        internal void FreeDevice(int deviceId)
         {
             Debug.Assert(ThreadSafety.IsAudioThread);
 
@@ -136,6 +156,8 @@ internal static void FreeDevice(int deviceId)
                 Bass.Free();
             }
 
+            freeWasapi();
+
             if (selectedDevice != deviceId && canSelectDevice(selectedDevice))
                 Bass.CurrentDevice = selectedDevice;
 
@@ -155,5 +177,88 @@ internal static void PreloadBass()
                 Library.Load("libbass.so", Library.LoadFlags.RTLD_LAZY | Library.LoadFlags.RTLD_GLOBAL);
             }
         }
+
+        private void attemptWasapiInitialisation()
+        {
+            if (RuntimeInfo.OS != RuntimeInfo.Platform.Windows)
+                return;
+
+            int wasapiDevice = -1;
+
+            // WASAPI device indices don't match normal BASS devices.
+            // Each device is listed multiple times with each supported channel/frequency pair.
+            //
+            // Working backwards to find the correct device is how bass does things internally (see BassWasapi.GetBassDevice).
+            if (Bass.CurrentDevice > 0)
+            {
+                string driver = Bass.GetDeviceInfo(Bass.CurrentDevice).Driver;
+
+                if (!string.IsNullOrEmpty(driver))
+                {
+                    // In the normal execution case, BassWasapi.GetDeviceInfo will return false as soon as we reach the end of devices.
+                    // This while condition is just a safety to avoid looping forever.
+                    // It's intentionally quite high because if a user has many audio devices, this list can get long.
+                    //
+                    // Retrieving device info here isn't free. In the future we may want to investigate a better method.
+                    while (wasapiDevice < 16384)
+                    {
+                        if (!BassWasapi.GetDeviceInfo(++wasapiDevice, out WasapiDeviceInfo info))
+                            break;
+
+                        if (info.ID == driver)
+                            break;
+                    }
+                }
+            }
+
+            // To keep things in a sane state let's only keep one device initialised via wasapi.
+            freeWasapi();
+            initWasapi(wasapiDevice);
+        }
+
+        private void initWasapi(int wasapiDevice)
+        {
+            // This is intentionally initialised inline and stored to a field.
+            // If we don't do this, it gets GC'd away.
+            wasapiProcedure = (buffer, length, _) =>
+            {
+                if (globalMixerHandle.Value == null)
+                    return 0;
+
+                return Bass.ChannelGetData(globalMixerHandle.Value!.Value, buffer, length);
+            };
+            wasapiNotifyProcedure = (notify, device, _) => Scheduler.Add(() =>
+            {
+                if (notify == WasapiNotificationType.DefaultOutput)
+                {
+                    freeWasapi();
+                    initWasapi(device);
+                }
+            });
+
+            bool initialised = BassWasapi.Init(wasapiDevice, Procedure: wasapiProcedure, Buffer: 0.001f, Period: 0.001f);
+
+            if (!initialised)
+                return;
+
+            BassWasapi.GetInfo(out var wasapiInfo);
+            globalMixerHandle.Value = BassMix.CreateMixerStream(wasapiInfo.Frequency, wasapiInfo.Channels, BassFlags.MixerNonStop | BassFlags.Decode | BassFlags.Float);
+            BassWasapi.Start();
+
+            BassWasapi.SetNotify(wasapiNotifyProcedure);
+        }
+
+        private void freeWasapi()
+        {
+            if (globalMixerHandle.Value == null) return;
+
+            // The mixer probably doesn't need to be recycled. Just keeping things sane for now.
+            Bass.StreamFree(globalMixerHandle.Value.Value);
+            BassWasapi.Stop();
+            BassWasapi.Free();
+            globalMixerHandle.Value = null;
+        }
+
+        #endregion
     }
 }

osu.Framework/osu.Framework.csproj

@@ -27,6 +27,7 @@
     <PackageReference Include="ppy.ManagedBass" Version="2022.1216.0" />
     <PackageReference Include="ppy.ManagedBass.Fx" Version="2022.1216.0" />
     <PackageReference Include="ppy.ManagedBass.Mix" Version="2022.1216.0" />
+    <PackageReference Include="ppy.ManagedBass.Wasapi" Version="2022.1216.0" />
     <PackageReference Include="ppy.Veldrid" Version="4.9.3-g91ce5a6cda" />
     <PackageReference Include="ppy.Veldrid.SPIRV" Version="1.0.15-gca6cec7843" />
     <PackageReference Include="SharpFNT" Version="2.0.0" />
@@ -43,7 +44,7 @@
 
     <!-- DO NOT use ProjectReference for native packaging project.
          See https://github.com/NuGet/Home/issues/4514 and https://github.com/dotnet/sdk/issues/765 . -->
-    <PackageReference Include="ppy.osu.Framework.NativeLibs" Version="2023.1205.0-nativelibs" />
+    <PackageReference Include="ppy.osu.Framework.NativeLibs" Version="2023.1225.0-nativelibs" />
 
     <!-- Any version ahead of this will cause AOT issues with iOS
          See https://github.com/mono/mono/issues/21188 -->