From aab145dab6e608911f7f4903e1eb9096c8451f6c Mon Sep 17 00:00:00 2001
From: Barret Schloerke <barret@posit.co>
Date: Fri, 30 Aug 2024 16:32:25 -0400
Subject: [PATCH] CI(debug): Print full stdout/stderr when command fails
 (#1660)

Co-authored-by: Joe Cheng <joe@posit.co>
---
 Makefile                               | 21 +++++++++++++--------
 tests/playwright/utils/deploy_utils.py | 22 +++++++++++++++++++++-
 2 files changed, 34 insertions(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index 7921200cc..ddc39a91b 100644
--- a/Makefile
+++ b/Makefile
@@ -149,14 +149,21 @@ clean-js: FORCE
 SUB_FILE:=
 PYTEST_BROWSERS:= --browser webkit --browser firefox --browser chromium
 PYTEST_DEPLOYS_BROWSERS:= --browser chromium
+
+# Full test path to playwright tests
+TEST_FILE:=tests/playwright/$(SUB_FILE)
+# Default `make` values that shouldn't be directly used; (Use `TEST_FILE` instead!)
+DEPLOYS_TEST_FILE:=tests/playwright/deploys$(SUB_FILE)
+SHINY_TEST_FILE:=tests/playwright/shiny/$(SUB_FILE)
+EXAMPLES_TEST_FILE:=tests/playwright/examples/$(SUB_FILE)
+
 install-playwright: FORCE
 	playwright install --with-deps
 
 install-rsconnect: FORCE
 	pip install git+https://github.com/rstudio/rsconnect-python.git#egg=rsconnect-python
 
-# Full test path to playwright tests
-TEST_FILE:="tests/playwright/$(SUB_FILE)"
+
 # All end-to-end tests with playwright
 playwright: install-playwright ## All end-to-end tests with playwright; (TEST_FILE="" from root of repo)
 	pytest $(TEST_FILE) $(PYTEST_BROWSERS)
@@ -169,20 +176,18 @@ playwright-show-trace: ## Show trace of failed tests
 
 # end-to-end tests with playwright; (SUB_FILE="" within tests/playwright/shiny/)
 playwright-shiny: FORCE
-	$(MAKE) playwright TEST_FILE="tests/playwright/shiny/$(SUB_FILE)"
+	$(MAKE) playwright TEST_FILE="$(SHINY_TEST_FILE)"
 
 # end-to-end tests on deployed apps with playwright; (SUB_FILE="" within tests/playwright/deploys/)
 playwright-deploys: FORCE
-	$(MAKE) playwright PYTEST_BROWSERS="$(PYTEST_DEPLOYS_BROWSERS)" TEST_FILE="$(TEST_FILE)"
-playwright-deploys-legacy: FORCE
-	$(MAKE) playwright TEST_FILE="tests/playwright/deploys/$(SUB_FILE)" PYTEST_BROWSERS="$(PYTEST_DEPLOYS_BROWSERS)"
+	$(MAKE) playwright PYTEST_BROWSERS="$(PYTEST_DEPLOYS_BROWSERS)" TEST_FILE="$(DEPLOYS_TEST_FILE)"
 
 # end-to-end tests on all py-shiny examples with playwright; (SUB_FILE="" within tests/playwright/examples/)
 playwright-examples: FORCE
-	$(MAKE) playwright TEST_FILE="tests/playwright/examples/$(SUB_FILE)"
+	$(MAKE) playwright TEST_FILE="$(EXAMPLES_TEST_FILE)"
 
 coverage: FORCE ## check combined code coverage (must run e2e last)
-	pytest --cov-report term-missing --cov=shiny tests/pytest/ tests/playwright/shiny/$(SUB_FILE) $(PYTEST_BROWSERS)
+	pytest --cov-report term-missing --cov=shiny tests/pytest/ $(SHINY_TEST_FILE) $(PYTEST_BROWSERS)
 	coverage html
 	$(BROWSER) htmlcov/index.html
 
diff --git a/tests/playwright/utils/deploy_utils.py b/tests/playwright/utils/deploy_utils.py
index 53501624b..4a0b9d512 100644
--- a/tests/playwright/utils/deploy_utils.py
+++ b/tests/playwright/utils/deploy_utils.py
@@ -2,6 +2,7 @@
 
 import json
 import os
+import re
 import shutil
 import subprocess
 import sys
@@ -70,14 +71,33 @@ def skip_on_webkit(fn: CallableT) -> CallableT:
 def run_command(cmd: str) -> str:
     output = subprocess.run(
         cmd,
-        check=True,
+        check=False,
         capture_output=True,
         text=True,
         shell=True,
     )
+    if output.returncode != 0:
+        print(
+            "Failed to run command!",
+            "\nstdout:",
+            output.stdout,
+            "\nstderr:",
+            output.stderr,
+            file=sys.stderr,
+            sep="\n",
+        )
+        raise RuntimeError(f"Failed to run command: {redact_api_key(cmd)}")
     return output.stdout
 
 
+def redact_api_key(cmd: str) -> str:
+    # Redact the value of the `--api-key` CLI argument, replace it with `***`
+    # Create a regex that replaces the argument following `--api-key`
+    # with `***` (e.g. `--api-key my-api-key` -> `--api-key ***`)
+
+    return re.sub(r"(--api-key\s+)(\S+)", r"\1***", cmd)
+
+
 def deploy_to_connect(app_name: str, app_dir: str) -> str:
     if not api_key:
         raise RuntimeError("No api key found. Cannot deploy.")