From f41a0038c75761023fb42cc2b391ea0d739ca8bb Mon Sep 17 00:00:00 2001 From: kyle-ignis Date: Tue, 4 Feb 2025 16:12:53 -0700 Subject: [PATCH] add shadow dom support --- .../src/main/java/maestro/UiElement.kt | 17 ++++++ .../main/java/maestro/drivers/WebDriver.kt | 47 +++++++++++++--- .../src/main/resources/maestro-web.js | 56 +++++++++++++------ .../kotlin/maestro/test/IntegrationTest.kt | 28 ++++++++++ 4 files changed, 121 insertions(+), 27 deletions(-) diff --git a/maestro-client/src/main/java/maestro/UiElement.kt b/maestro-client/src/main/java/maestro/UiElement.kt index 36b148d62b..1cb3ad2821 100644 --- a/maestro-client/src/main/java/maestro/UiElement.kt +++ b/maestro-client/src/main/java/maestro/UiElement.kt @@ -22,6 +22,10 @@ package maestro data class UiElement( val treeNode: TreeNode, val bounds: Bounds, + val attributes: Map, + val children: MutableList, + val nativeElement: Any? = null, + val hasShadowRoot: Boolean = attributes["has-shadow-root"] == "true" ) { fun distanceTo(other: UiElement): Float { @@ -79,6 +83,17 @@ data class UiElement( return isXWithinBounds && isYWithinBounds } + fun findInShadowDom(predicate: (UiElement) -> Boolean): UiElement? { + if (predicate(this)) return this + + return children.asSequence() + .map { it.toUiElement() } + .filterNotNull() + .firstOrNull { + it.findInShadowDom(predicate) != null + } + } + companion object { fun TreeNode.toUiElement(): UiElement { @@ -106,6 +121,8 @@ data class UiElement( width = boundsArr[2] - boundsArr[0], height = boundsArr[3] - boundsArr[1] ), + attributes, + mutableListOf(), ) } } diff --git a/maestro-client/src/main/java/maestro/drivers/WebDriver.kt b/maestro-client/src/main/java/maestro/drivers/WebDriver.kt index c5f37d77bd..d41c728d37 100644 --- a/maestro-client/src/main/java/maestro/drivers/WebDriver.kt +++ b/maestro-client/src/main/java/maestro/drivers/WebDriver.kt @@ -33,6 +33,7 @@ import org.slf4j.LoggerFactory import java.io.File import java.time.Duration import java.util.* +import maestro.UiElement class WebDriver( @@ -83,7 +84,7 @@ class WebDriver( return seleniumDriver ?: error("Driver is not open") } - private fun executeJS(js: String): Any? { + private fun executeJS(js: String, vararg args: Any?): Any? { val executor = seleniumDriver as JavascriptExecutor try { @@ -94,10 +95,10 @@ class WebDriver( } Thread.sleep(100) - return executor.executeScript(js) + return executor.executeScript(js, *args) } catch (e: Exception) { if (e.message?.contains("getContentDescription") == true) { - return executeJS(js) + return executeJS(js, *args) } return null } @@ -190,9 +191,6 @@ class WebDriver( detectWindowChange() - // retrieve view hierarchy from DOM - // There are edge cases where executeJS returns null, and we cannot get the hierarchy. In this situation - // we retry multiple times until throwing an error eventually. (See issue #1936) var contentDesc: Any? = null var retry = 0 while (contentDesc == null) { @@ -205,10 +203,10 @@ class WebDriver( } } - // parse into TreeNodes + // Parse into TreeNodes with shadow DOM support fun parse(domRepresentation: Map): TreeNode { val attrs = domRepresentation["attributes"] as Map - + val attributes = mutableMapOf( "text" to attrs["text"] as String, "bounds" to attrs["bounds"] as String, @@ -216,8 +214,11 @@ class WebDriver( if (attrs.containsKey("resource-id") && attrs["resource-id"] != null) { attributes["resource-id"] = attrs["resource-id"] as String } + if (attrs.containsKey("has-shadow-root")) { + attributes["has-shadow-root"] = attrs["has-shadow-root"].toString() + } + val children = domRepresentation["children"] as List> - return TreeNode(attributes = attributes, children = children.map { parse(it) }) } @@ -474,6 +475,34 @@ class WebDriver( // Do nothing } + // Add method to handle shadow DOM elements + private fun handleShadowDOM(element: UiElement): UiElement { + val driver = ensureOpen() + + // If element has shadow root, get its content + if (element.attributes["has-shadow-root"] == "true") { + val shadowRoot = executeJS( + "return arguments[0].shadowRoot", + element.nativeElement + ) + + // Update element with shadow DOM content + if (shadowRoot != null) { + // Merge shadow DOM content with regular content + val shadowContent = executeJS( + "return window.maestro.getContentDescription(arguments[0])", + shadowRoot + ) + if (shadowContent != null) { + // Merge the shadow DOM content with the element's existing content + element.children.addAll((shadowContent as Map)["children"] as List) + } + } + } + + return element + } + companion object { private const val SCREENSHOT_DIFF_THRESHOLD = 0.005 private const val RETRY_FETCHING_CONTENT_DESCRIPTION = 10 diff --git a/maestro-client/src/main/resources/maestro-web.js b/maestro-client/src/main/resources/maestro-web.js index 29b76a04a2..04d2f6a18e 100644 --- a/maestro-client/src/main/resources/maestro-web.js +++ b/maestro-client/src/main/resources/maestro-web.js @@ -9,10 +9,21 @@ switch (node.tagName.toLowerCase()) { case 'input': return node.value || node.placeholder || node.ariaLabel || '' - default: + let text = '' const childNodes = [...(node.childNodes || [])].filter(node => node.nodeType === Node.TEXT_NODE) - return childNodes.map(node => node.textContent.replace('\n', '').replace('\t', '')).join('') + text = childNodes.map(node => node.textContent.trim()).join(' ') + + if (node.shadowRoot) { + const shadowText = [...node.shadowRoot.childNodes] + .filter(node => node.nodeType === Node.TEXT_NODE) + .map(node => node.textContent.trim()) + .join(' ') + if (shadowText) { + text = text ? `${text} ${shadowText}` : shadowText + } + } + return text } } @@ -25,26 +36,35 @@ const isDocumentLoading = () => document.readyState !== 'complete' const traverse = (node) => { - if (!node || isInvalidTag(node)) return null + if (!node || isInvalidTag(node)) return null + + let children = [] + if (node.shadowRoot) { + children = [...(node.shadowRoot.children || [])].map(child => traverse(child)).filter(el => !!el) + } + children = children.concat([...node.children || []].map(child => traverse(child)).filter(el => !!el)) + + const attributes = { + text: getNodeText(node), + bounds: getNodeBounds(node), + } - const children = [...node.children || []].map(child => traverse(child)).filter(el => !!el) - const attributes = { - text: getNodeText(node), - bounds: getNodeBounds(node), - } + if (!!node.id || !!node.ariaLabel || !!node.name || !!node.title || !!node.htmlFor || !!node.attributes['data-testid']) { + attributes['resource-id'] = node.id || node.ariaLabel || node.name || node.title || node.htmlFor || node.attributes['data-testid']?.value + } - if (!!node.id || !!node.ariaLabel || !!node.name || !!node.title || !!node.htmlFor || !!node.attributes['data-testid']) { - attributes['resource-id'] = node.id || node.ariaLabel || node.name || node.title || node.htmlFor || node.attributes['data-testid']?.value - } + if (node.shadowRoot) { + attributes['has-shadow-root'] = 'true' + } - if (node.tagName.toLowerCase() === 'body') { - attributes['is-loading'] = isDocumentLoading() - } + if (node.tagName.toLowerCase() === 'body') { + attributes['is-loading'] = isDocumentLoading() + } - return { - attributes, - children, - } + return { + attributes, + children, + } } // -------------- Public API -------------- diff --git a/maestro-test/src/test/kotlin/maestro/test/IntegrationTest.kt b/maestro-test/src/test/kotlin/maestro/test/IntegrationTest.kt index 1c7d67dc84..830bd15ccb 100644 --- a/maestro-test/src/test/kotlin/maestro/test/IntegrationTest.kt +++ b/maestro-test/src/test/kotlin/maestro/test/IntegrationTest.kt @@ -3218,6 +3218,34 @@ class IntegrationTest { ) } + @Test + fun `Case - Shadow DOM support`() { + // Given + val commands = readCommands("shadow_dom_test") + + val driver = driver { + element { + id = "shadow-host" + bounds = Bounds(0, 0, 200, 200) + + // Simulate shadow DOM content + element { + text = "Shadow DOM Content" + bounds = Bounds(10, 10, 190, 190) + } + } + } + + // When + Maestro(driver).use { + orchestra(it).runFlow(commands) + } + + // Then + // No test failure + driver.assertNoInteraction() + } + private fun orchestra( maestro: Maestro, ) = Orchestra(