refactor: use docker compose for integration tests (#483)

.github/workflows/build.yml

@@ -43,8 +43,6 @@ jobs:
         name: Tests
         env:
           GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS }}
-          OTF_TEST_DATABASE_URL: postgres://postgres:postgres@localhost:5433/postgres?sslmode=disable
-          PUBSUB_EMULATOR_HOST: "localhost:8085"
         run: make test
       -
         name: Archive browser screenshots

Makefile

@@ -75,11 +75,6 @@ compose-rm:
 postgres:
 	docker compose up -d postgres
 
-# Run squid via docker compose
-.PHONY: squid
-squid:
-	docker compose up -d squid
-
 # Run staticcheck metalinter recursively against code
 .PHONY: lint
 lint:

docker-compose.yml

@@ -2,19 +2,36 @@ services:
   postgres:
     image: postgres:14-alpine
     ports:
-      - "5433:5432"
+      - 5432
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U postgres"]
       interval: 5s
       timeout: 5s
       retries: 3
-    command: -c fsync=off
+    # * setting high max_connections number avoids the error when running large number of
+    #   integration tests in parallel:
+    #
+    #   FATAL: sorry, too many clients already
+    #
+    # * disabling fsync dramatically improves integration test speed
+    #
+    # NOTE: neither setting is appropriate for a production deployment
+    command: -c fsync=off -c max_connections=999
     environment:
      - POSTGRES_PASSWORD=postgres
+  # Terraform-based tests spawn `terraform`. These tests retrieve providers from
+  # the internet which can consume quite a lot of bandwidth and slow down the tests
+  # significantly. To cache these providers we use the squid caching
+  # proxy (http://www.squid-cache.org/).
+  #
+  # It is configured to use
+  # [SSL-bumping](https://wiki.squid-cache.org/Features/SslBump), which permits
+  # caching content transported via SSL (`terraform` retrieves providers only via
+  # SSL).
   squid:
     image: leg100/squid
     ports:
-      - "3128:3128"
+      - 3128
     healthcheck:
       test: ["CMD-SHELL", "nc -zw1 localhost 3128"]
       interval: 5s
@@ -27,7 +44,7 @@ services:
   pubsub:
     image: google/cloud-sdk:emulators
     ports:
-      - "8085:8085"
+      - 8085
     stop_signal: SIGINT
     command: gcloud beta emulators pubsub start --project=abc123 --host-port=0.0.0.0:8085
   otfd:
@@ -38,14 +55,14 @@ services:
       squid:
         condition: service_healthy
     ports:
-      - "8833:8080"
+      - "8080:8080"
     healthcheck:
       test: ["CMD", "curl", "http://localhost:8080/healthz"]
       interval: 5s
       timeout: 5s
       retries: 3
     environment:
-      - OTF_DATABASE=postgres://postgres:postgres@postgres:5432/postgres?sslmode=disable
+      - OTF_DATABASE=postgres://postgres:postgres@postgres/postgres?sslmode=disable
       - OTF_SECRET=6b07b57377755b07cf61709780ee7484
       - OTF_SITE_TOKEN=site-token
       - OTF_SSL=true

docs/testing.md

@@ -5,83 +5,28 @@
 Change into the repo directory and run unit tests:
 
 ```
-go test ./...
+go test -short ./...
 ```
 
 ### Integration tests
 
 Integration tests require:
 
-* PostgreSQL
-* Terraform >= 1.2.0
+* docker compose
+* terraform >= 1.2.0
 * Chrome
 
-Set the environment variable `OTF_TEST_DATABASE_URL` to a valid connection string. For example, if you have installed postgres on your local machine with the default database `postgres`:
+The following runs integration tests:
 
 ```
-export OTF_TEST_DATABASE_URL=postgres:///postgres
+go test ./internal/integration...
 ```
 
-Then run the both unit and integration tests:
+The tests first stands up external services using docker compose:
 
-```
-go test ./...
-```
-
-!!! note
-	Tests check for the presence of `OTF_TEST_DATABASE_URL`. If it absent then only unit tests are run; otherwise both unit and integration tests are run.
-
-#### Postgres too many connections error
-
-When running integration tests you may run into this error:
-
-```
-FATAL: sorry, too many clients already
-```
-
-The tests make potentially hundreds of connections to postgres at a time and you'll need to increase the maximum number of connections in postgres. Increase the `max_connections` parameter on your installation:
-
-```
-psql postgres:///otf
-alter system set max_connections = 999;
-```
-
-Then reload/restart postgres. On Ubuntu, for example:
-
-```
-sudo systemctl reload postgres
-```
-
-!!! note
-    This is only recommended for a postgres installation dedicated to development and testing purposes. Too many connections in production most likely indicates a problem with the client application.
-
-
-#### Postgres performance optimisation
-
-To further improve the speed of your integration tests turn off `fsync`:
-
-```
-psql postgres:///otf
-alter system set fsync = off;
-```
-
-Then reload/restart postgres. On Ubuntu, for example:
-
-```
-sudo systemctl reload postgres
-```
-
-!!! note
-    This is only recommended for a postgres installation dedicated to development and testing purposes. Disabling `fsync` can lead to data loss.
-
-#### Database cleanup
-
-A dedicated logical database is created in postgres for each individual
-integration test; as a result the above command creates 100s of databases. Upon
-test completion the databases are removed. In certain situtations upon a test
-failure the test may fail to remove a database, in which case you will have to
-manually remove the database (although this is often not a concern other than
-consuming a small amount of disk space).
+* postgres (integration tests require a real database)
+* squid cache (speeds up tests by caching terraform providers)
+* GCP pub/sub emulator (necessary for the GCP pub/sub integration test)
 
 #### Disable headless mode
 
@@ -98,32 +43,6 @@ export OTF_E2E_HEADLESS=false
 <figcaption>Integration tests with headless mode disabled</figcaption>
 </figure>
 
-#### Cache provider requests
-
-Terraform-based tests spawn `terraform`. These tests retrieve providers from
-the internet which can consume quite a lot of bandwidth and slow down the tests
-significantly. To cache these providers it is recommended to use a caching
-proxy. The following make task runs [squid](http://www.squid-cache.org/) in a
-docker container:
-
-```
-make squid
-```
-
-It is configured to use
-[SSL-bumping](https://wiki.squid-cache.org/Features/SslBump), which permits
-caching content transported via SSL (`terraform` retrieves providers only via
-SSL).
-
-You then need to instruct the tests to use the proxy:
-
-```
-export HTTPS_PROXY=localhost:3128
-```
-
-You should now find the tests consume a lot less bandwidth and run several times
-faster.
-
 ### API tests
 
 Tests from the [go-tfe](https://github.com/hashicorp/go-tfe) project are routinely run to ensure OTF correctly implements the documented Terraform Cloud API. However, OTF only implements a subset of the API endpoints, and there are some differences (e.g. an OTF organization has no email address where as a TFC organization does). Therefore a [fork](https://github.com/leg100/go-tfe) of the go-tfe repo is maintained.
@@ -136,9 +55,9 @@ make go-tfe-tests
 
 performs the following steps:
 
-* Starts a docker compose stack of `otfd`, postgres, and [squid](#cache-provider-requests)
+* Starts a docker compose stack of `otfd`, postgres, and squid
 * Runs a subset of `go-tfe` tests using the **forked** repo
 * Runs a subset of `go-tfe` tests using the **upstream** repo
 
 !!! note
-    You can instead manually invoke API tests using the scripts in `./hack`. The tests first require `otfd` to be running at `https://localhost:8833`, with a [site token](../config/flags/#-site-token) set to `site-token`. These settings can be overridden with the environment variables `TFE_ADDRESS` and `TFE_TOKEN`.
+    You can instead manually invoke API tests using the scripts in `./hack`. The tests first require `otfd` to be running at `https://localhost:8080`, with a [site token](../config/flags/#-site-token) set to `site-token`. These settings can be overridden with the environment variables `TFE_ADDRESS` and `TFE_TOKEN`.

hack/go-tfe-tests.bash

@@ -12,7 +12,7 @@ set -e
 GO_TFE_REPO="${GO_TFE_REPO:-github.com/leg100/go-tfe@otf}"
 TESTS="${@:-Test(Variables|Workspaces(Create|List|Update|Delete|Lock|Unlock|ForceUnlock|Read\$|ReadByID)|Organizations(Create|List|Read|Update)|StateVersion|Runs|Plans|Applies(Read|Logs)|ConfigurationVersions)}"
 
-export TFE_ADDRESS="${TFE_ADDRESS:-https://localhost:8833}"
+export TFE_ADDRESS="${TFE_ADDRESS:-https://localhost:8080}"
 # go-tfe tests perform privileged operations (e.g. creating organizations), so
 # we use a site admin token.
 #

internal/integration/agent_token_ui_test.go

@@ -11,7 +11,7 @@ import (
 
 // TestAgentTokenUI demonstrates managing agent tokens via the UI.
 func TestAgentTokenUI(t *testing.T) {
-	t.Parallel()
+	integrationTest(t)
 
 	svc, org, ctx := setup(t, nil)
 

internal/integration/auto_apply_ui_test.go

@@ -10,7 +10,7 @@ import (
 // TestAutoApply tests auto-apply functionality, using the UI to enable
 // auto-apply on a workspace first before invoking 'terraform apply'.
 func TestAutoApply(t *testing.T) {
-	t.Parallel()
+	integrationTest(t)
 
 	svc, org, ctx := setup(t, nil)
 

internal/integration/broker_test.go

@@ -11,7 +11,7 @@ import (
 
 // TestBroker demonstrates publishing and subscribing of events via postgres.
 func TestBroker(t *testing.T) {
-	t.Parallel()
+	integrationTest(t)
 
 	// simulate a cluster of two otfd nodes sharing a database
 	cfg := config{

internal/integration/cloud_block_ui_test.go

@@ -13,7 +13,7 @@ import (
 //
 // https://developer.hashicorp.com/terraform/cli/cloud/settings#the-cloud-block
 func TestCloudBlock(t *testing.T) {
-	t.Parallel()
+	integrationTest(t)
 
 	svc, org, ctx := setup(t, nil)
 

internal/integration/cluster_test.go

@@ -21,7 +21,7 @@ import (
 // processes successfully, e.g. relaying of logs from the agent through to the
 // TF CLI
 func TestCluster(t *testing.T) {
-	t.Parallel()
+	integrationTest(t)
 
 	// start two daemons, one for user, one for agent, both sharing a db
 	connstr := sql.NewTestDB(t)

internal/integration/configuration_version_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func TestConfigurationVersion(t *testing.T) {
-	t.Parallel()
+	integrationTest(t)
 
 	t.Run("create", func(t *testing.T) {
 		svc, _, ctx := setup(t, nil)

internal/integration/connect_repo_e2e_test.go

@@ -14,7 +14,7 @@ import (
 // TestConnectRepoE2E demonstrates connecting a workspace to a VCS repository, pushing a
 // git commit which triggers a run on the workspace.
 func TestConnectRepoE2E(t *testing.T) {
-	t.Parallel()
+	integrationTest(t)
 
 	// create an otf daemon with a fake github backend, serve up a repo and its
 	// contents via tarball. And register a callback to test receipt of commit

internal/integration/db_test.go

@@ -0,0 +1,33 @@
+package integration
+
+import (
+	"context"
+	"testing"
+
+	"github.com/leg100/otf/internal/logr"
+	"github.com/leg100/otf/internal/sql"
+	"github.com/stretchr/testify/require"
+)
+
+// TestWaitAndLock tests acquiring a connection from a pool, obtaining a session
+// lock and then releasing lock and the connection, and it does this several
+// times, to demonstrate that it is returning resources and not running into
+// limits.
+func TestWaitAndLock(t *testing.T) {
+	integrationTest(t)
+
+	ctx := context.Background()
+	db, err := sql.New(ctx, sql.Options{
+		Logger:     logr.Discard(),
+		ConnString: sql.NewTestDB(t),
+	})
+	require.NoError(t, err)
+	t.Cleanup(db.Close)
+
+	for i := 0; i < 100; i++ {
+		func() {
+			err := db.WaitAndLock(ctx, 123, func() error { return nil })
+			require.NoError(t, err)
+		}()
+	}
+}

internal/integration/events_test.go

@@ -11,7 +11,7 @@ import (
 // TestIntegration_Events demonstrates events are triggered and successfully
 // received by a subscriber.
 func TestIntegration_Events(t *testing.T) {
-	t.Parallel()
+	integrationTest(t)
 
 	// disable the scheduler so that the run below doesn't get scheduled and
 	// change state before we test for equality with the received event.

internal/integration/github_login_test.go

@@ -13,7 +13,7 @@ import (
 
 // TestGithubLogin demonstrates logging into the UI via Github OAuth.
 func TestGithubLogin(t *testing.T) {
-	t.Parallel()
+	integrationTest(t)
 
 	// Start daemon with a stub github server populated with a user.
 	cfg := config{

internal/integration/github_pr_test.go

@@ -14,7 +14,7 @@ import (
 // TestIntegration_GithubPR demonstrates the spawning of runs in response to
 // opening and updating a pull-request on github.
 func TestIntegration_GithubPR(t *testing.T) {
-	t.Parallel()
+	integrationTest(t)
 
 	// create an otf daemon with a fake github backend, serve up a repo and its
 	// contents via tarball.

internal/integration/helpers_test.go

@@ -11,6 +11,17 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func integrationTest(t *testing.T) {
+	// An integration test can take a while to run so it be run in parallel to
+	// other integration tests
+	t.Parallel()
+
+	// Skip long-running integration tests if user has passed -short flag
+	if testing.Short() {
+		t.Skip()
+	}
+}
+
 func runURL(hostname, runID string) string {
 	return "https://" + hostname + "/app/runs/" + runID
 }

internal/integration/lock_file_test.go

@@ -15,7 +15,7 @@ import (
 // has to then generates hashes for linux, etc. It is notorious for causing
 // difficulties for users and it's no different for OTF.
 func TestLockFile(t *testing.T) {
-	t.Parallel()
+	integrationTest(t)
 
 	svc, org, ctx := setup(t, nil)