fix: don't use chromium for browser-based tests (#478)

.github/workflows/build.yml

@@ -32,6 +32,13 @@ jobs:
       -
         name: API tests
         run: make go-tfe-tests
+      -
+        # chromium breaks browser-based tests:
+        # https://github.com/chromedp/chromedp/issues/1325#issuecomment-1606114185
+        name: Remove chromium
+        run: |
+          sudo rm /usr/bin/chromium
+          sudo rm /usr/bin/chromium-browser
       -
         name: Tests
         env:

internal/integration/agent_token_ui_test.go

@@ -23,21 +23,21 @@ func TestAgentTokenUI(t *testing.T) {
 			chromedp.Navigate(organizationURL(svc.Hostname(), org.Name)),
 			screenshot(t),
 			// go to list of agent tokens
-			chromedp.Click("#agent_tokens > a", chromedp.NodeVisible),
+			chromedp.Click("#agent_tokens > a", chromedp.NodeVisible, chromedp.ByQuery),
 			screenshot(t),
 			// go to new agent token page
 			chromedp.Click(`//button[text()='New Agent Token']`, chromedp.NodeVisible),
 			screenshot(t),
 			// enter description for new agent token
-			chromedp.Focus("input#description", chromedp.NodeVisible),
+			chromedp.Focus("input#description", chromedp.NodeVisible, chromedp.ByQuery),
 			input.InsertText("my-new-agent-token"),
 			screenshot(t),
 			// submit form
 			chromedp.Click(`//button[text()='Create token']`, chromedp.NodeVisible),
 			screenshot(t),
 			matchRegex(t, ".flash-success", `Created token:\s+[\w-]+\.[\w-]+\.[\w-]+`),
 			// click clipboard icon to copy token into clipboard
-			chromedp.Click(`//div[@class='flash flash-success']//img[@class='clipboard-icon']`, chromedp.BySearch),
+			chromedp.Click(`//div[@class='flash flash-success']//img[@class='clipboard-icon']`),
 			chromedp.Evaluate(`window.navigator.clipboard.readText()`, &clipboardContent, func(p *runtime.EvaluateParams) *runtime.EvaluateParams {
 				return p.WithAwaitPromise(true)
 			}),

internal/integration/github_login_test.go

@@ -46,7 +46,7 @@ func TestGithubLogin(t *testing.T) {
 		chromedp.Navigate("https://" + svc.Hostname() + "/login"),
 		screenshot(t, "github_login_button"),
 		// login
-		chromedp.Click("a.login-button-github", chromedp.NodeVisible),
+		chromedp.Click("a.login-button-github", chromedp.NodeVisible, chromedp.ByQuery),
 		screenshot(t),
 		// check login confirmation message
 		matchText(t, ".content > p", "You are logged in as bobby"),

internal/integration/main_test.go

@@ -5,6 +5,7 @@ import (
 	"crypto/rand"
 	"fmt"
 	"os"
+	"os/exec"
 	"path"
 	"path/filepath"
 	"testing"
@@ -38,6 +39,12 @@ func TestMain(m *testing.M) {
 }
 
 func doMain(m *testing.M) (int, error) {
+	for _, chromium := range []string{"chromium", "chromium-browser"} {
+		if _, err := exec.LookPath(chromium); err == nil {
+			return 0, fmt.Errorf("found %s executable in path; chromium has a bug that breaks the browser-based tests; see https://github.com/chromedp/chromedp/issues/1325", chromium)
+		}
+	}
+
 	// The otfd daemon spawned in an integration test uses a self-signed cert.
 	// The following environment variable instructs any Go program spawned in a
 	// test, e.g. the terraform CLI, the otf agent, etc, to trust the

internal/integration/module_e2e_test.go

@@ -36,7 +36,7 @@ func TestModuleE2E(t *testing.T) {
 			chromedp.Navigate(organizationURL(svc.Hostname(), org.Name)),
 			screenshot(t),
 			// go to modules
-			chromedp.Click("#modules > a", chromedp.NodeVisible),
+			chromedp.Click("#modules > a", chromedp.NodeVisible, chromedp.ByQuery),
 			screenshot(t, "modules_list"),
 			// click publish button
 			chromedp.Click(`//button[text()='Publish']`, chromedp.NodeVisible),

internal/integration/plan_permission_e2e_test.go

@@ -60,16 +60,16 @@ func TestIntegration_PlanPermission(t *testing.T) {
 		chromedp.Navigate(workspaceURL(svc.Hostname(), org.Name, "my-test-workspace")),
 		screenshot(t),
 		// select operation for run
-		chromedp.SetValue(`//select[@id="start-run-operation"]`, "plan-only", chromedp.BySearch),
+		chromedp.SetValue(`//select[@id="start-run-operation"]`, "plan-only"),
 		screenshot(t),
 		// confirm plan begins and ends
 		chromedp.WaitReady(`body`),
-		chromedp.WaitReady(`//*[@id='tailed-plan-logs']//text()[contains(.,'Initializing the backend')]`, chromedp.BySearch),
+		chromedp.WaitReady(`//*[@id='tailed-plan-logs']//text()[contains(.,'Initializing the backend')]`),
 		screenshot(t),
 		chromedp.WaitReady(`#plan-status.phase-status-finished`),
 		screenshot(t),
 		// wait for run to enter planned-and-finished state
-		chromedp.WaitReady(`//*[@class='status status-planned_and_finished']`, chromedp.BySearch),
+		chromedp.WaitReady(`//*[@class='status status-planned_and_finished']`),
 		screenshot(t),
 		// run widget should show plan summary
 		matchRegex(t, `//div[@class='item']//div[@class='resource-summary']`, `\+[0-9]+ \~[0-9]+ \-[0-9]+`),

internal/integration/retry_run_ui_test.go

@@ -35,18 +35,18 @@ func TestIntegration_RetryRunUI(t *testing.T) {
 	browser.Run(t, ctx, chromedp.Tasks{
 		chromedp.Navigate(runURL(daemon.Hostname(), r.ID)),
 		// run should be in planned and finished state
-		chromedp.WaitReady(`//*[@class='status status-planned_and_finished']`, chromedp.BySearch),
+		chromedp.WaitReady(`//*[@class='status status-planned_and_finished']`),
 		screenshot(t, "run_page_planned_and_finished_state"),
 		// click retry button
-		chromedp.Click(`//button[text()='retry run']`, chromedp.NodeVisible, chromedp.BySearch),
+		chromedp.Click(`//button[text()='retry run']`, chromedp.NodeVisible),
 		screenshot(t),
 		// confirm plan begins and ends
 		chromedp.WaitReady(`body`),
-		chromedp.WaitReady(`//*[@id='tailed-plan-logs']//text()[contains(.,'Initializing the backend')]`, chromedp.BySearch),
+		chromedp.WaitReady(`//*[@id='tailed-plan-logs']//text()[contains(.,'Initializing the backend')]`),
 		screenshot(t),
-		chromedp.WaitReady(`#plan-status.phase-status-finished`),
+		chromedp.WaitReady(`#plan-status.phase-status-finished`, chromedp.ByQuery),
 		// confirm retry button re-appears
-		chromedp.WaitReady(`//button[text()='retry run']`, chromedp.BySearch),
+		chromedp.WaitReady(`//button[text()='retry run']`),
 		screenshot(t),
 	})
 }

internal/integration/run_list_ui_test.go

@@ -34,9 +34,9 @@ func TestIntegration_RunListUI(t *testing.T) {
 			return nil
 		}),
 		// should be one run listed
-		chromedp.Nodes(`//div[@id='content-list']//*[@class='item']`, &runListingAfter, chromedp.BySearch),
+		chromedp.Nodes(`//div[@id='content-list']//*[@class='item']`, &runListingAfter),
 		// and its status should be 'planned and finished'
-		chromedp.WaitVisible(`//*[@class='item']//*[@class='status status-planned_and_finished']`, chromedp.BySearch),
+		chromedp.WaitVisible(`//*[@class='item']//*[@class='status status-planned_and_finished']`),
 	})
 	assert.Equal(t, 1, len(runListingAfter))
 }

internal/integration/site_admin_ui_test.go

@@ -27,33 +27,33 @@ func TestSiteAdminUI(t *testing.T) {
 		screenshot(t, "no_authenticators_site_admin_login"),
 		// use the link in the bottom right corner
 		matchText(t, ".footer-site-login", "site admin"),
-		chromedp.Click(".footer-site-login > a", chromedp.NodeVisible),
+		chromedp.Click(".footer-site-login > a", chromedp.NodeVisible, chromedp.ByQuery),
 		screenshot(t),
 		// enter token
-		chromedp.Focus("input#token", chromedp.NodeVisible),
+		chromedp.Focus("input#token", chromedp.NodeVisible, chromedp.ByQuery),
 		input.InsertText("abc123"),
 		screenshot(t, "site_admin_login_enter_token"),
-		chromedp.Submit("input#token"),
+		chromedp.Submit("input#token", chromedp.ByQuery),
 		screenshot(t, "site_admin_profile"),
 		matchText(t, ".content > p", "You are logged in as site-admin"),
 		// now go to the list of organizations
 		chromedp.Navigate("https://" + daemon.Hostname() + "/app/organizations"),
 		// add an org
-		chromedp.Click("#new-organization-button", chromedp.NodeVisible),
+		chromedp.Click("#new-organization-button", chromedp.NodeVisible, chromedp.ByQuery),
 		screenshot(t),
-		chromedp.Focus("input#name", chromedp.NodeVisible),
+		chromedp.Focus("input#name", chromedp.NodeVisible, chromedp.ByQuery),
 		input.InsertText("my-new-org"),
 		screenshot(t, "new_org_enter_name"),
-		chromedp.Submit("input#name"),
+		chromedp.Submit("input#name", chromedp.ByQuery),
 		screenshot(t, "new_org_created"),
 		chromedp.Location(&orgLocation),
 		matchText(t, ".flash-success", "created organization: my-new-org"),
 		// go to organization settings
-		chromedp.Click("#settings > a", chromedp.NodeVisible),
+		chromedp.Click("#settings > a", chromedp.NodeVisible, chromedp.ByQuery),
 		screenshot(t),
 		// change organization name
-		chromedp.Focus("input#name", chromedp.NodeVisible),
-		chromedp.Clear("input#name"),
+		chromedp.Focus("input#name", chromedp.NodeVisible, chromedp.ByQuery),
+		chromedp.Clear("input#name", chromedp.ByQuery),
 		input.InsertText("newly-named-org"),
 		screenshot(t),
 		chromedp.Click(`//button[text()='Update organization name']`, chromedp.NodeVisible),

internal/integration/start_run_ui_test.go

@@ -30,31 +30,31 @@ func TestStartRunUI(t *testing.T) {
 		chromedp.Click(`//a[text()='settings']`, chromedp.NodeVisible),
 		screenshot(t),
 		// click 'queue destroy plan' button
-		chromedp.Click(`//button[@id='queue-destroy-plan-button']`, chromedp.BySearch),
+		chromedp.Click(`//button[@id='queue-destroy-plan-button']`),
 		screenshot(t),
 		// confirm plan begins and ends
 		chromedp.WaitReady(`body`),
-		chromedp.WaitReady(`//*[@id='tailed-plan-logs']//text()[contains(.,'Initializing the backend')]`, chromedp.BySearch),
+		chromedp.WaitReady(`//*[@id='tailed-plan-logs']//text()[contains(.,'Initializing the backend')]`),
 		screenshot(t),
-		chromedp.WaitReady(`#plan-status.phase-status-finished`),
+		chromedp.WaitReady(`#plan-status.phase-status-finished`, chromedp.ByQuery),
 		screenshot(t),
 		// wait for run to enter planned state
-		chromedp.WaitReady(`//*[@class='status status-planned']`, chromedp.BySearch),
+		chromedp.WaitReady(`//*[@class='status status-planned']`),
 		screenshot(t),
 		// run widget should show plan summary
 		matchRegex(t, `//div[@class='item']//div[@class='resource-summary']`, `\+[0-9]+ \~[0-9]+ \-[0-9]+`),
 		screenshot(t),
 		// run widget should show discard button
-		chromedp.WaitReady(`//button[@id='run-discard-button']`, chromedp.BySearch),
+		chromedp.WaitReady(`//button[@id='run-discard-button']`),
 		screenshot(t),
 		// click 'confirm & apply' button once it becomes visible
-		chromedp.Click(`//button[text()='apply']`, chromedp.NodeVisible, chromedp.BySearch),
+		chromedp.Click(`//button[text()='apply']`, chromedp.NodeVisible),
 		screenshot(t),
 		// confirm apply begins and ends
-		chromedp.WaitReady(`//*[@id='tailed-apply-logs']//text()[contains(.,'Initializing the backend')]`, chromedp.BySearch),
-		chromedp.WaitReady(`#apply-status.phase-status-finished`),
+		chromedp.WaitReady(`//*[@id='tailed-apply-logs']//text()[contains(.,'Initializing the backend')]`),
+		chromedp.WaitReady(`#apply-status.phase-status-finished`, chromedp.ByQuery),
 		// confirm run ends in applied state
-		chromedp.WaitReady(`//*[@class='status status-applied']`, chromedp.BySearch),
+		chromedp.WaitReady(`//*[@class='status status-applied']`),
 		// run widget should show apply summary
 		matchRegex(t, `//div[@class='item']//div[@class='resource-summary']`, `\+[0-9]+ \~[0-9]+ \-[0-9]+`),
 		screenshot(t),

internal/integration/tag_e2e_test.go

@@ -84,7 +84,7 @@ resource "null_resource" "tags_e2e" {}
 		screenshot(t),
 		matchText(t, ".flash-success", "removed tag: bar"),
 		// add new tag
-		chromedp.Focus("input#new-tag-name", chromedp.NodeVisible),
+		chromedp.Focus("input#new-tag-name", chromedp.NodeVisible, chromedp.ByQuery),
 		input.InsertText("baz"),
 		chromedp.Click(`//button[text()='Add new tag']`, chromedp.NodeVisible),
 		screenshot(t),

internal/integration/team_ui_test.go

@@ -27,7 +27,7 @@ func TestIntegration_TeamUI(t *testing.T) {
 			chromedp.Click(`//div[@class='content-list']//a[text()='owners']`, chromedp.NodeVisible),
 			screenshot(t, "owners_team_page"),
 			// select newbie as new team member
-			chromedp.SetValue(`//select[@id="select-add-member"]`, newbie.Username, chromedp.BySearch),
+			chromedp.SetValue(`//select[@id="select-add-member"]`, newbie.Username),
 			screenshot(t),
 			// submit
 			chromedp.Click(`//button[text()='Add member']`, chromedp.NodeVisible),

internal/integration/ui_helpers_test.go

@@ -30,13 +30,13 @@ func createWorkspace(t *testing.T, hostname, org, name string) chromedp.Tasks {
 	return chromedp.Tasks{
 		chromedp.Navigate(organizationURL(hostname, org)),
 		chromedp.WaitReady(`body`),
-		chromedp.Click("#menu-item-workspaces > a"),
+		chromedp.Click("#menu-item-workspaces > a", chromedp.ByQuery),
 		chromedp.WaitReady(`body`),
 		chromedp.Click("#new-workspace-button", chromedp.NodeVisible, chromedp.ByQuery),
 		chromedp.WaitReady(`body`),
-		chromedp.Focus("input#name", chromedp.NodeVisible),
+		chromedp.Focus("input#name", chromedp.NodeVisible, chromedp.ByQuery),
 		input.InsertText(name),
-		chromedp.Click("#create-workspace-button", chromedp.NodeVisible),
+		chromedp.Click("#create-workspace-button", chromedp.NodeVisible, chromedp.ByQuery),
 		chromedp.WaitReady(`body`),
 		matchText(t, ".flash-success", "created workspace: "+name),
 	}
@@ -155,8 +155,8 @@ func addWorkspacePermission(t *testing.T, hostname, org, workspaceName, team, ro
 		matchText(t, "#permissions-owners td:first-child", "owners"),
 		matchText(t, "#permissions-owners td:last-child", "admin"),
 		// assign role to team
-		chromedp.SetValue(`//select[@id="permissions-add-select-role"]`, role, chromedp.BySearch),
-		chromedp.SetValue(`//select[@id="permissions-add-select-team"]`, team, chromedp.BySearch),
+		chromedp.SetValue(`//select[@id="permissions-add-select-role"]`, role),
+		chromedp.SetValue(`//select[@id="permissions-add-select-team"]`, team),
 		// scroll to bottom so that permissions are visible in screenshot
 		chromedp.ActionFunc(func(ctx context.Context) error {
 			_, exp, err := runtime.Evaluate(`window.scrollTo(0,document.body.scrollHeight);`).Do(ctx)
@@ -169,7 +169,7 @@ func addWorkspacePermission(t *testing.T, hostname, org, workspaceName, team, ro
 			return nil
 		}),
 		screenshot(t, "workspace_permissions"),
-		chromedp.Click("#permissions-add-button", chromedp.NodeVisible),
+		chromedp.Click("#permissions-add-button", chromedp.NodeVisible, chromedp.ByQuery),
 		screenshot(t),
 		matchText(t, ".flash-success", "updated workspace permissions"),
 	}
@@ -181,19 +181,19 @@ func createGithubVCSProviderTasks(t *testing.T, hostname, org, name string) chro
 		chromedp.Navigate(organizationURL(hostname, org)),
 		screenshot(t, "organization_main_menu"),
 		// go to vcs providers
-		chromedp.Click("#vcs_providers > a", chromedp.NodeVisible),
+		chromedp.Click("#vcs_providers > a", chromedp.NodeVisible, chromedp.ByQuery),
 		screenshot(t, "vcs_providers_list"),
 		// click 'New Github VCS Provider' button
 		chromedp.Click(`//button[text()='New Github VCS Provider']`, chromedp.NodeVisible),
 		screenshot(t, "new_github_vcs_provider_form"),
 		// enter fake github token and name
-		chromedp.Focus("input#token", chromedp.NodeVisible),
+		chromedp.Focus("input#token", chromedp.NodeVisible, chromedp.ByQuery),
 		input.InsertText("fake-github-personal-token"),
-		chromedp.Focus("input#name"),
+		chromedp.Focus("input#name", chromedp.ByQuery),
 		input.InsertText(name),
 		screenshot(t),
 		// submit form to create provider
-		chromedp.Submit("input#token"),
+		chromedp.Submit("input#token", chromedp.ByQuery),
 		screenshot(t),
 		matchText(t, ".flash-success", "created provider: github"),
 	}
@@ -206,31 +206,31 @@ func startRunTasks(t *testing.T, hostname, organization, workspaceName string, o
 		chromedp.Navigate(workspaceURL(hostname, organization, workspaceName)),
 		screenshot(t, "connected_workspace_main_page"),
 		// select operation for run
-		chromedp.SetValue(`//select[@id="start-run-operation"]`, string(op), chromedp.BySearch),
+		chromedp.SetValue(`//select[@id="start-run-operation"]`, string(op)),
 		screenshot(t, "run_page_started"),
 		// confirm plan begins and ends
 		chromedp.WaitReady(`body`),
-		chromedp.WaitReady(`//*[@id='tailed-plan-logs']//text()[contains(.,'Initializing the backend')]`, chromedp.BySearch),
+		chromedp.WaitReady(`//*[@id='tailed-plan-logs']//text()[contains(.,'Initializing the backend')]`),
 		screenshot(t),
 		chromedp.WaitReady(`#plan-status.phase-status-finished`),
 		screenshot(t),
 		// wait for run to enter planned state
-		chromedp.WaitReady(`//*[@class='status status-planned']`, chromedp.BySearch),
+		chromedp.WaitReady(`//*[@class='status status-planned']`),
 		screenshot(t),
 		// run widget should show plan summary
 		matchRegex(t, `//div[@class='item']//div[@class='resource-summary']`, `\+[0-9]+ \~[0-9]+ \-[0-9]+`),
 		screenshot(t, "run_page_planned_state"),
 		// run widget should show discard button
-		chromedp.WaitReady(`//button[@id='run-discard-button']`, chromedp.BySearch),
+		chromedp.WaitReady(`//button[@id='run-discard-button']`),
 		screenshot(t),
 		// click 'apply' button once it becomes visible
-		chromedp.Click(`//button[text()='apply']`, chromedp.NodeVisible, chromedp.BySearch),
+		chromedp.Click(`//button[text()='apply']`, chromedp.NodeVisible),
 		screenshot(t),
 		// confirm apply begins and ends
-		chromedp.WaitReady(`//*[@id='tailed-apply-logs']//text()[contains(.,'Initializing the backend')]`, chromedp.BySearch),
-		chromedp.WaitReady(`#apply-status.phase-status-finished`),
+		chromedp.WaitReady(`//*[@id='tailed-apply-logs']//text()[contains(.,'Initializing the backend')]`),
+		chromedp.WaitReady(`#apply-status.phase-status-finished`, chromedp.ByQuery),
 		// confirm run ends in applied state
-		chromedp.WaitReady(`//*[@class='status status-applied']`, chromedp.BySearch),
+		chromedp.WaitReady(`//*[@class='status status-applied']`),
 		// run widget should show apply summary
 		matchRegex(t, `//div[@class='item']//div[@class='resource-summary']`, `\+[0-9]+ \~[0-9]+ \-[0-9]+`),
 		screenshot(t),

internal/integration/user_token_ui_test.go

@@ -23,7 +23,7 @@ func TestIntegration_UserTokenUI(t *testing.T) {
 		chromedp.Click(`//button[@id='new-user-token-button']`, chromedp.NodeVisible),
 		chromedp.WaitReady(`body`),
 		// enter description for new token and submit
-		chromedp.Focus("input#description", chromedp.NodeVisible),
+		chromedp.Focus("input#description", chromedp.NodeVisible, chromedp.ByQuery),
 		input.InsertText("my new token"),
 		screenshot(t, "user_token_enter_description"),
 		chromedp.Click(`//button[text()='Create token']`, chromedp.NodeVisible),

internal/integration/variable_e2e_test.go

@@ -31,15 +31,15 @@ func TestVariableE2E(t *testing.T) {
 			chromedp.Click(`//button[text()='Add variable']`, chromedp.NodeVisible),
 			screenshot(t),
 			// enter key
-			chromedp.Focus("input#key", chromedp.NodeVisible),
+			chromedp.Focus("input#key", chromedp.NodeVisible, chromedp.ByQuery),
 			input.InsertText("foo"),
 			screenshot(t),
 			// enter value
-			chromedp.Focus("textarea#value", chromedp.NodeVisible),
+			chromedp.Focus("textarea#value", chromedp.NodeVisible, chromedp.ByQuery),
 			input.InsertText("bar"),
 			screenshot(t),
 			// select terraform variable category
-			chromedp.Click("input#terraform", chromedp.NodeVisible),
+			chromedp.Click("input#terraform", chromedp.NodeVisible, chromedp.ByQuery),
 			screenshot(t),
 			// submit form
 			chromedp.Click(`//button[@id='save-variable-button']`, chromedp.NodeVisible),
@@ -98,7 +98,7 @@ output "foo" {
 			chromedp.Click(`//a[text()='foo']`, chromedp.NodeVisible),
 			screenshot(t),
 			// update value
-			chromedp.Focus("textarea#value", chromedp.NodeVisible),
+			chromedp.Focus("textarea#value", chromedp.NodeVisible, chromedp.ByQuery),
 			input.InsertText("topsecret"),
 			screenshot(t, "variables_entering_top_secret"),
 			// submit form

internal/integration/web_test.go

@@ -57,7 +57,7 @@ func TestWeb(t *testing.T) {
 			chromedp.Navigate(organizationURL(daemon.Hostname(), org.Name)),
 			screenshot(t),
 			// list users
-			chromedp.Click("#users > a", chromedp.NodeVisible),
+			chromedp.Click("#users > a", chromedp.NodeVisible, chromedp.ByQuery),
 			screenshot(t),
 			matchText(t, fmt.Sprintf("#item-user-%s .status", user.Username), user.Username),
 		},
@@ -67,9 +67,9 @@ func TestWeb(t *testing.T) {
 			chromedp.Navigate(organizationURL(daemon.Hostname(), org.Name)),
 			screenshot(t),
 			// list teams
-			chromedp.Click("#teams > a", chromedp.NodeVisible),
+			chromedp.Click("#teams > a", chromedp.NodeVisible, chromedp.ByQuery),
 			// select owners team
-			chromedp.Click("#item-team-owners a", chromedp.NodeVisible),
+			chromedp.Click("#item-team-owners a", chromedp.NodeVisible, chromedp.ByQuery),
 			screenshot(t),
 			matchText(t, fmt.Sprintf("#item-user-%s .status", user.Username), user.Username),
 		},