Add scroll script and fix other Windows extensions

Add the `windows: scroll` extension script for compatibility with appium-windows-driver. Document the other Windows extension scripts and throw on unsupported options. Also fix the calculation of the mouse position by element ID and x and y coordinates.
FlaUI · Nov 5, 2024 · 654a4c4 · 654a4c4
1 parent 4408741
commit 654a4c4
Show file tree

Hide file tree

Showing 7 changed files with 110 additions and 42 deletions.
diff --git a/README.md b/README.md
@@ -179,6 +179,10 @@ Using the WebdriverIO JavaScript client:
 const result = driver.executeScript("powerShell", [{ command: `1+1` }]);
 ```
 
+## Windows extensions
+
+To enable easy switching from appium-windows-driver, there is a rudimentary implementation of `windows: click`, `windows: hover`, `windows: scroll` and `windows: keys`.
+
 ## Supported WebDriver Commands
 
 | Method | URI Template                                                   | Command                        | Implemented                        |

diff --git a/src/FlaUI.WebDriver/Controllers/ExecuteController.cs b/src/FlaUI.WebDriver/Controllers/ExecuteController.cs
@@ -35,8 +35,10 @@ public async Task<ActionResult> ExecuteScript([FromRoute] string sessionId, [Fro
                     return await ExecuteWindowsClickScript(session, executeScriptRequest);
                 case "windows: hover":
                     return await ExecuteWindowsHoverScript(session, executeScriptRequest);
+                case "windows: scroll":
+                    return await ExecuteWindowsScrollScript(session, executeScriptRequest);
                 default:
-                    throw WebDriverResponseException.UnsupportedOperation("Only 'powerShell' scripts are supported");
+                    throw WebDriverResponseException.UnsupportedOperation("Only 'powerShell', 'windows: keys', 'windows: click', 'windows: hover' scripts are supported");
             }
         }
 
@@ -103,6 +105,21 @@ private async Task<ActionResult> ExecuteWindowsClickScript(Session session, Exec
             return WebDriverResult.Success();
         }
 
+        private async Task<ActionResult> ExecuteWindowsScrollScript(Session session, ExecuteScriptRequest executeScriptRequest)
+        {
+            if (executeScriptRequest.Args.Count != 1)
+            {
+                throw WebDriverResponseException.InvalidArgument($"Expected an array of exactly 1 arguments for the windows: click script, but got {executeScriptRequest.Args.Count} arguments");
+            }
+            var action = JsonSerializer.Deserialize<WindowsScrollScript>(executeScriptRequest.Args[0], new JsonSerializerOptions() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase });
+            if (action == null)
+            {
+                throw WebDriverResponseException.InvalidArgument("Action cannot be null");
+            }
+            await _windowsExtensionService.ExecuteScrollScript(session, action);
+            return WebDriverResult.Success();
+        }
+
         private async Task<ActionResult> ExecuteWindowsHoverScript(Session session, ExecuteScriptRequest executeScriptRequest)
         {
             if (executeScriptRequest.Args.Count != 1)

diff --git a/src/FlaUI.WebDriver/Models/WindowsClickScript.cs b/src/FlaUI.WebDriver/Models/WindowsClickScript.cs
@@ -6,5 +6,9 @@ public class WindowsClickScript
         public int? X { get; set; }
         public int? Y { get; set; }
         public string? Button { get; set; }
+        public string[]? ModifierKeys { get; set; }
+        public int? DurationMs { get; set; }
+        public int? Times { get; set; }
+        public int? InterClickDelayMs { get; set; }
     }
 }
diff --git a/src/FlaUI.WebDriver/Models/WindowsHoverScript.cs b/src/FlaUI.WebDriver/Models/WindowsHoverScript.cs
@@ -9,5 +9,6 @@ public class WindowsHoverScript
         public int? EndY { get; set; }
         public string? EndElementId { get; set; }
         public int? DurationMs { get; set; }
+        public string[]? ModifierKeys { get; set; }
     }
 }
diff --git a/src/FlaUI.WebDriver/Models/WindowsScrollScript.cs b/src/FlaUI.WebDriver/Models/WindowsScrollScript.cs
@@ -0,0 +1,12 @@
+namespace FlaUI.WebDriver.Models
+{
+    public class WindowsScrollScript
+    {
+        public string? ElementId { get; set; }
+        public int? X { get; set; }
+        public int? Y { get; set; }
+        public int? DeltaX { get; set; }
+        public int? DeltaY { get; set; }
+        public string[]? ModifierKeys { get; set; }
+    }
+}
diff --git a/src/FlaUI.WebDriver/Services/IWindowsExtensionService.cs b/src/FlaUI.WebDriver/Services/IWindowsExtensionService.cs
@@ -5,6 +5,7 @@ namespace FlaUI.WebDriver.Services
     public interface IWindowsExtensionService
     {
         Task ExecuteClickScript(Session session, WindowsClickScript action);
+        Task ExecuteScrollScript(Session session, WindowsScrollScript action);
         Task ExecuteHoverScript(Session session, WindowsHoverScript action);
         Task ExecuteKeyScript(Session session, WindowsKeyScript action);
     }

diff --git a/src/FlaUI.WebDriver/Services/WindowsExtensionService.cs b/src/FlaUI.WebDriver/Services/WindowsExtensionService.cs
@@ -1,4 +1,5 @@
 using FlaUI.Core.Input;
+using FlaUI.Core.Tools;
 using FlaUI.Core.WindowsAPI;
 using FlaUI.WebDriver.Models;
 using System.Drawing;
@@ -16,76 +17,104 @@ public WindowsExtensionService(ILogger<WindowsExtensionService> logger)
 
         public async Task ExecuteClickScript(Session session, WindowsClickScript action)
         {
-            var mouseButton = action.Button != null ? Enum.Parse<MouseButton>(action.Button, true) : MouseButton.Left;
-            if (action.ElementId != null)
+            if (action.DurationMs.HasValue)
             {
-                var element = session.FindKnownElementById(action.ElementId);
-                if (element == null)
-                {
-                    throw WebDriverResponseException.ElementNotFound(action.ElementId);
-                }
-                _logger.LogDebug("Clicking element {ElementId} with mouse button {MouseButton}", action.ElementId, mouseButton);
-                Mouse.Click(element.BoundingRectangle.Location, mouseButton);
+                throw WebDriverResponseException.UnsupportedOperation("Duration is not yet supported");
             }
-            else if (action.X.HasValue && action.Y.HasValue)
+            if (action.Times.HasValue)
             {
-                _logger.LogDebug("Clicking point ({X}, {Y}) with mouse button {MouseButton}", action.X.Value, action.Y.Value, mouseButton);
-                Mouse.Click(new Point { X = action.X.Value, Y = action.Y.Value }, mouseButton);
+                throw WebDriverResponseException.UnsupportedOperation("Times is not yet supported");
             }
-            else
+            if (action.ModifierKeys != null)
             {
-                throw WebDriverResponseException.InvalidArgument("Either \"elementId\" or \"x\" and \"y\" must be provided");
+                throw WebDriverResponseException.UnsupportedOperation("Modifier keys are not yet supported");
             }
+            var point = GetPoint(action.ElementId, action.X, action.Y, session);
+            var mouseButton = action.Button != null ? Enum.Parse<MouseButton>(action.Button, true) : MouseButton.Left;
+            _logger.LogDebug("Clicking point ({X}, {Y}) with mouse button {MouseButton}", point.X, point.Y, mouseButton);
+            Mouse.Click(point, mouseButton);
             await Task.Yield();
         }
 
-        public async Task ExecuteHoverScript(Session session, WindowsHoverScript action)
+        public async Task ExecuteScrollScript(Session session, WindowsScrollScript action)
         {
-            if (action.StartX.HasValue && action.StartY.HasValue)
+            if (action.ModifierKeys != null)
             {
-                _logger.LogDebug("Moving mouse to ({X}, {Y})", action.StartX.Value, action.StartY.Value);
-                Mouse.MoveTo(action.StartX.Value, action.StartY.Value);
+                throw WebDriverResponseException.UnsupportedOperation("Modifier keys are not yet supported");
             }
-            else if (action.StartElementId != null)
+            var point = GetPoint(action.ElementId, action.X, action.Y, session);
+            _logger.LogDebug("Scrolling at point ({X}, {Y})", point.X, point.Y);
+            Mouse.Position = point;
+            if (action.DeltaY.HasValue && action.DeltaY.Value != 0)
             {
-                var element = session.FindKnownElementById(action.StartElementId);
+                Mouse.Scroll(action.DeltaY.Value);
+            }
+            if (action.DeltaX.HasValue && action.DeltaX.Value != 0)
+            {
+                Mouse.HorizontalScroll(-action.DeltaX.Value);
+            }
+            await Task.Yield();
+        }
+
+        private Point GetPoint(string? elementId, int? x, int? y, Session session)
+        {
+            if (elementId != null)
+            {
+                var element = session.FindKnownElementById(elementId);
                 if (element == null)
                 {
-                    throw WebDriverResponseException.ElementNotFound(action.StartElementId);
+                    throw WebDriverResponseException.ElementNotFound(elementId);
                 }
-                _logger.LogDebug("Moving mouse to element {ElementId}", action.StartElementId);
-                Mouse.MoveTo(element.BoundingRectangle.Location);
+
+                if (x.HasValue && y.HasValue)
+                {
+                    return new Point
+                    {
+                        X = element.BoundingRectangle.Left + x.Value,
+                        Y = element.BoundingRectangle.Top + y.Value
+                    };
+                }
+
+                return element.BoundingRectangle.Center();
             }
-            else
+
+            if (x.HasValue && y.HasValue)
             {
-                throw WebDriverResponseException.InvalidArgument("Either \"startElementId\" or \"startX\" and \"startY\" must be provided");
+                return new Point { X = x.Value, Y = y.Value };
             }
+
+            throw WebDriverResponseException.InvalidArgument("Either element ID or x and y must be provided");
+        }
 
-            if (action.DurationMs.HasValue)
+        public async Task ExecuteHoverScript(Session session, WindowsHoverScript action)
+        {
+            if (action.ModifierKeys != null)
             {
-                _logger.LogDebug("Waiting for {DurationMs}ms", action.DurationMs.Value);
-                await Task.Delay(action.DurationMs.Value);
+                throw WebDriverResponseException.UnsupportedOperation("Modifier keys are not yet supported");
             }
+            var startingPoint = GetPoint(action.StartElementId, action.StartX, action.StartY, session);
+            var endPoint = GetPoint(action.EndElementId, action.EndX, action.EndY, session);
+
+            _logger.LogDebug("Moving mouse to starting point ({X}, {Y})", startingPoint.X, startingPoint.Y);
+            Mouse.Position = startingPoint;
 
-            if (action.EndX.HasValue && action.EndY.HasValue)
+            if (endPoint == startingPoint)
             {
-                _logger.LogDebug("Moving mouse to ({X}, {Y})", action.EndX.Value, action.EndY.Value);
-                Mouse.MoveTo(action.EndX.Value, action.EndY.Value);
+                // Hover for specified time
+                await Task.Delay(action.DurationMs ?? 10);
+                return;
             }
-            else if (action.EndElementId != null)
+
+            _logger.LogDebug("Moving mouse to end point ({X}, {Y})", endPoint.X, endPoint.Y);
+            if (action.DurationMs.HasValue)
             {
-                var element = session.FindKnownElementById(action.EndElementId);
-                if (element == null)
+                if (action.DurationMs.Value <= 0)
                 {
-                    throw WebDriverResponseException.ElementNotFound(action.EndElementId);
+                    throw WebDriverResponseException.UnsupportedOperation("Duration less than or equal to zero is not supported");
                 }
-                _logger.LogDebug("Moving mouse to element {ElementId}", action.EndElementId);
-                Mouse.MoveTo(element.BoundingRectangle.Location);
-            }
-            else
-            {
-                throw WebDriverResponseException.InvalidArgument("Either \"endElementId\" or \"endX\" and \"endY\" must be provided");
+                Mouse.MovePixelsPerMillisecond = endPoint.Distance(startingPoint) / action.DurationMs.Value;
             }
+            Mouse.MoveTo(endPoint);
         }
 
         public async Task ExecuteKeyScript(Session session, WindowsKeyScript action)