Temporarily hardcoded runtime detection (just for fun)

SightKeeper.Application/Extensions/YoloCLIExtensions.cs

@@ -73,8 +73,9 @@ private static async Task SetRunsDirectory(string directory, ILogger logger)
 
     public static async Task<string> ExportToONNX(string modelPath, ushort imagesSize, ILogger logger)
     {
-        modelPath = ReplaceBackSlashes(modelPath);
-        var outputStream = CLIExtensions.RunCLICommand($"yolo export model=\'{modelPath}\' format=onnx imgsz={imagesSize} opset=15", logger);
+        var fullModelPath = Path.GetFullPath(modelPath);
+        var fullModelPathWithReplacedSlashes = ReplaceBackSlashes(fullModelPath);
+        var outputStream = CLIExtensions.RunCLICommand($"yolo export model=\'{fullModelPathWithReplacedSlashes}\' format=onnx imgsz={imagesSize} opset=15", logger);
         await outputStream.WhereNotNull().Where(content => content.Contains("export success")).FirstAsync();
         var onnxModelPath = modelPath.Replace(".pt", ".onnx");
         Guard.IsTrue(File.Exists(onnxModelPath));

SightKeeper.Application/Scoring/StreamDetector.cs

@@ -0,0 +1,101 @@
+using System.Collections.Immutable;
+using System.Reactive.Disposables;
+using System.Reactive.Linq;
+using System.Reactive.Subjects;
+using CommunityToolkit.Diagnostics;
+using Serilog;
+using SerilogTimings;
+using SightKeeper.Domain.Model;
+
+namespace SightKeeper.Application.Scoring;
+
+public sealed class StreamDetector
+{
+    public IObservable<ImmutableList<DetectionItem>> ObservableDetection => _detection;
+
+    public bool IsEnabled
+    {
+        get => _d != null;
+        set
+        {
+            var isEnabled = IsEnabled;
+            if (value == isEnabled)
+                return;
+            if (value)
+            {
+                var sleepTime = TimeSpan.FromMilliseconds(10);
+                Guard.IsNotNull(Weights);
+                _screenCapture.Resolution = Weights.Library.DataSet.Resolution;
+                Guard.IsNull(_d);
+                var previousImage = Array.Empty<byte>();
+                _d = Observable.Create<ImmutableList<DetectionItem>>(async observer =>
+                {
+                    var work = true;
+                    while (work)
+                    {
+                        using var operation = Operation.Begin("Detecting in loop");
+                        var image = await _screenCapture.CaptureAsync();
+                        using var imagesComparisonOperation = Operation.Begin("Comparing images");
+                        var imagesAreEqual = image.SequenceEqual(previousImage);
+                        imagesComparisonOperation.Complete(nameof(imagesAreEqual), imagesAreEqual);
+                        if (imagesAreEqual)
+                        {
+                            Log.Debug("Images are equal, skipping...");
+                            continue;
+                        }
+                        previousImage = image;
+                        var result = await _detector.Detect(image, CancellationToken.None);
+                        operation.Complete();
+                        observer.OnNext(result.ToImmutableList());
+                        BurnTime(sleepTime);
+                    }
+                    return Disposable.Create(() =>
+                    {
+                        work = false;
+                    });
+                }).Subscribe(_detection);
+            }
+            else
+            {
+                Guard.IsNotNull(_d);
+                _d.Dispose();
+            }
+        }
+    }
+
+    private static void BurnTime(TimeSpan time)
+    {
+        var start = DateTime.UtcNow;
+        var stop = start + time;
+        while (DateTime.UtcNow < stop)
+        {
+        }
+    }
+
+    public Weights? Weights
+    {
+        get => _detector.Weights;
+        set => _detector.Weights = value;
+    }
+    public float ProbabilityThreshold
+    {
+        get => _detector.ProbabilityThreshold;
+        set => _detector.ProbabilityThreshold = value;
+    }
+    public float IoU
+    {
+        get => _detector.IoU;
+        set => _detector.IoU = value;
+    }
+
+    public StreamDetector(Detector detector, ScreenCapture screenCapture)
+    {
+        _detector = detector;
+        _screenCapture = screenCapture;
+    }
+
+    private readonly Detector _detector;
+    private readonly ScreenCapture _screenCapture;
+    private IDisposable? _d;
+    private readonly Subject<ImmutableList<DetectionItem>> _detection = new();
+}

SightKeeper.Avalonia/AppBootstrapper.cs

@@ -114,6 +114,8 @@ private static void SetupServices(ContainerBuilder builder)
 		builder.RegisterType<NewProfileDataValidator>().As<IValidator<NewProfileData>>();
 		builder.RegisterType<ProfilesObservableRepository>().InstancePerMainViewModel();
 		builder.RegisterType<SightKeeper.Application.ProfileEditor>().InstancePerMainViewModel();
+		builder.RegisterType<StreamDetector>();
+		builder.RegisterType<WindowsMouseMover>().As<MouseMover>();
 
 		SimpleReactiveGlobalHook hook = new();
 		builder.RegisterInstance(hook).As<IReactiveGlobalHook>();

SightKeeper.Avalonia/ViewModels/Tabs/Profiles/Tab/FakeProfilesViewModel.cs

@@ -12,6 +12,8 @@ public sealed class FakeProfilesViewModel : IProfilesViewModel
     public IReadOnlyCollection<ProfileViewModel> Profiles { get; }
     public ICommand CreateProfileCommand => FakeViewModel.CommandSubstitute;
     public ICommand EditProfileCommand => FakeViewModel.CommandSubstitute;
+    public ICommand LaunchProfileCommand => FakeViewModel.CommandSubstitute;
+    public ICommand StopProfileCommand => FakeViewModel.CommandSubstitute;
 
     public FakeProfilesViewModel()
     {

SightKeeper.Avalonia/ViewModels/Tabs/Profiles/Tab/IProfilesViewModel.cs

@@ -8,4 +8,6 @@ public interface IProfilesViewModel
     IReadOnlyCollection<ProfileViewModel> Profiles { get; }
     ICommand CreateProfileCommand { get; }
     ICommand EditProfileCommand { get; }
+    ICommand LaunchProfileCommand { get; }
+    ICommand StopProfileCommand { get; }
 }

SightKeeper.Avalonia/ViewModels/Tabs/Profiles/Tab/ProfilesViewModel.cs

@@ -1,10 +1,16 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
+using System.Drawing;
+using System.Linq;
+using System.Numerics;
 using System.Threading.Tasks;
 using System.Windows.Input;
 using Autofac;
 using CommunityToolkit.Diagnostics;
 using CommunityToolkit.Mvvm.Input;
+using Serilog;
 using SightKeeper.Application;
+using SightKeeper.Application.Scoring;
 using SightKeeper.Avalonia.Extensions;
 using SightKeeper.Avalonia.ViewModels.Tabs.Profiles.Editor;
 
@@ -14,15 +20,19 @@ public sealed partial class ProfilesViewModel : ViewModel, IProfilesViewModel
 {
     public IReadOnlyCollection<ProfileViewModel> Profiles { get; }
 
-    public ProfilesViewModel(ProfilesListViewModel profilesList, ILifetimeScope scope, ProfileCreator profileCreator)
+    public ProfilesViewModel(ProfilesListViewModel profilesList, ILifetimeScope scope, ProfileCreator profileCreator, StreamDetector streamDetector, MouseMover mouseMover)
     {
         _scope = scope;
         _profileCreator = profileCreator;
+        _streamDetector = streamDetector;
+        _mouseMover = mouseMover;
         Profiles = profilesList.ProfileViewModels;
     }
 
     private readonly ILifetimeScope _scope;
     private readonly ProfileCreator _profileCreator;
+    private readonly StreamDetector _streamDetector;
+    private readonly MouseMover _mouseMover;
 
     ICommand IProfilesViewModel.CreateProfileCommand => CreateProfileCommand;
     [RelayCommand]
@@ -39,9 +49,70 @@ private async Task CreateProfile()
     }
 
     ICommand IProfilesViewModel.EditProfileCommand => EditProfileCommand;
+
     [RelayCommand]
     private async Task EditProfile(ProfileViewModel profile)
     {
 
     }
+
+    ICommand IProfilesViewModel.LaunchProfileCommand => LaunchProfileCommand;
+    [RelayCommand(CanExecute = nameof(CanLaunchProfile))]
+    private async Task LaunchProfile(ProfileViewModel profileViewModel)
+    {
+        if (_runningProfile != null)
+        {
+        }
+        _streamDetector.Weights = profileViewModel.Profile.Weights;
+        _streamDetector.IsEnabled = true;
+        var distanceThreshold = 3;
+        var distanceThresholdSquared = distanceThreshold * distanceThreshold;
+        _streamDetector.ObservableDetection.Subscribe(detection =>
+        {
+            if (!detection.Any())
+                return;
+            Log.Debug("Detected: {Detection}", detection);
+            var closest = detection.MinBy(item => GetDistance(item.Bounding));
+            var move = GetMove(closest.Bounding) * profileViewModel.Profile.Weights.Library.DataSet.Resolution / 2;
+            if (move.LengthSquared() < distanceThresholdSquared)
+                return;
+            _mouseMover.Move((short)move.X, (short)move.Y);
+        });
+        _runningProfile = profileViewModel;
+        LaunchProfileCommand.NotifyCanExecuteChanged();
+        StopProfileCommand.NotifyCanExecuteChanged();
+    }
+    private bool CanLaunchProfile() => _runningProfile == null;
+
+    private float GetDistance(RectangleF rect)
+    {
+        var center = new Vector2(0.5f, 0.5f);
+        var pos = new Vector2((rect.Left + rect.Right) / 2, (rect.Top + rect.Bottom) / 2);
+        var difference = pos - center;
+        return difference.LengthSquared();
+    }
+
+    private Vector2 GetMove(RectangleF rect)
+    {
+        var center = new Vector2(0.5f, 0.5f);
+        var pos = new Vector2((rect.Left + rect.Right) / 2, (rect.Top + rect.Bottom) / 2);
+        var difference = pos - center;
+        return difference;
+    }
+
+    ICommand IProfilesViewModel.StopProfileCommand => StopProfileCommand;
+    [RelayCommand(CanExecute = nameof(CanStopProfile))]
+    private async Task StopProfile(ProfileViewModel profile)
+    {
+        Guard.IsNotNull(_runningProfile);
+        _streamDetector.IsEnabled = false;
+        _streamDetector.Weights = null;
+        _runningProfile = null;
+        LaunchProfileCommand.NotifyCanExecuteChanged();
+        StopProfileCommand.NotifyCanExecuteChanged();
+    }
+
+    private bool CanStopProfile(ProfileViewModel profile) => profile == _runningProfile;
+
+    private ProfileViewModel? _runningProfile;
 }

SightKeeper.Avalonia/Views/Profiles/ProfilesTab.axaml

@@ -38,7 +38,6 @@
                             <Grid RowDefinitions="* *"
                                   ColumnDefinitions="* Auto">
                                 <TextBlock Grid.Row="0" Grid.Column="0"
-                                           Grid.ColumnSpan="2"
                                            Text="{Binding Name}"
                                            FontSize="18"
                                            VerticalAlignment="Center"
@@ -48,12 +47,27 @@
                                            FontSize="14"
                                            VerticalAlignment="Center"
                                            Margin="0 0 5 0"/>
+                                <Button Grid.Row="0" Grid.Column="1"
+                                        Content="{icons:MaterialIconExt Play}"
+                                        Padding="5"
+                                        Command="{Binding #Root.((profilesViewModels:IProfilesViewModel)DataContext).LaunchProfileCommand}"
+                                        CommandParameter="{Binding}"
+                                        ToolTip.Tip="Launch"
+                                        IsVisible="{Binding $self.IsEffectivelyEnabled}"/>
+                                <Button Grid.Row="0" Grid.Column="1"
+                                        Content="{icons:MaterialIconExt Stop}"
+                                        Padding="5"
+                                        Command="{Binding #Root.((profilesViewModels:IProfilesViewModel)DataContext).StopProfileCommand}"
+                                        CommandParameter="{Binding}"
+                                        ToolTip.Tip="Stop"
+                                        Name="StopButton"
+                                        IsVisible="{Binding $self.IsEffectivelyEnabled}"/>
                                 <Button Grid.Row="1" Grid.Column="1"
                                         Content="{icons:MaterialIconExt Cog}"
                                         Padding="5"
                                         Command="{Binding #Root.((profilesViewModels:IProfilesViewModel)DataContext).EditProfileCommand}"
                                         CommandParameter="{Binding}"
-                                        ToolTip.Tip="Edit profile"/>
+                                        ToolTip.Tip="Edit"/>
                             </Grid>
                         </Border>
                     </DataTemplate>

SightKeeper.Data/Services/DataSet/DbDataSetsDataAccess.cs

@@ -21,7 +21,7 @@ public DbDataSetsDataAccess(AppDbContext dbContext)
     {
         lock (_dbContext)
             return _dbContext.DataSets
-                .Include(dataSet => dataSet.ItemClasses)
+                .Include(dataSet => dataSet.ItemClasses.OrderBy(itemClass => EF.Property<int>(itemClass, "Id")))
                 .Include(dataSet => dataSet.Game)
                 .Include(dataSet => dataSet.ScreenshotsLibrary)
                 .ToList();

SightKeeper.Services/Scoring/ONNXDetector.cs

@@ -6,13 +6,22 @@
 using SightKeeper.Application.Scoring;
 using SightKeeper.Domain.Model;
 using SightKeeper.Domain.Model.Common;
+using SightKeeper.Domain.Services;
 using SixLabors.ImageSharp;
 using RectangleF = System.Drawing.RectangleF;
 
 namespace SightKeeper.Services.Scoring;
 
 public sealed class ONNXDetector : Detector
 {
+    private readonly WeightsDataAccess _weightsDataAccess;
+
+
+    private async void SetWeights(Weights weights)
+    {
+        var weightsData = await _weightsDataAccess.LoadWeightsData(weights, WeightsFormat.ONNX);
+        _predictor = new YoloV8(new ModelSelector(weightsData.Content), CreateMetadata(weights.Library.DataSet));
+    }
     public Weights? Weights
     {
         get => _weights;
@@ -24,7 +33,7 @@ public Weights? Weights
             _itemClasses = null;
             if (value == null)
                 return;
-            _predictor = new YoloV8(new ModelSelector(value.ONNXData.Content), CreateMetadata(value.Library.DataSet));
+            SetWeights(value);
             _itemClasses = value.Library.DataSet.ItemClasses
                 .Select((itemClass, itemClassIndex) => (itemClass, itemClassIndex))
                 .ToDictionary(t => t.itemClassIndex, t => t.itemClass);
@@ -53,6 +62,11 @@ public float IoU
         }
     }
 
+    public ONNXDetector(WeightsDataAccess weightsDataAccess)
+    {
+        _weightsDataAccess = weightsDataAccess;
+    }
+
     public async Task<IReadOnlyCollection<DetectionItem>> Detect(byte[] image, CancellationToken cancellationToken)
     {
         using var operation = Operation.Begin("Detecting on image");