chore(CI): add retry for common errors

test/integration/appfactory/appfactory_test.go

@@ -16,14 +16,17 @@ package appfactory
 
 import (
 	"testing"
+	"time"
 
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
 	"github.com/stretchr/testify/assert"
+	"github.com/terraform-google-modules/enterprise-application/test/integration/testutils"
 )
 
 func TestAppfactory(t *testing.T) {
 	bootstrap := tft.NewTFBlueprintTest(t,
-		         tft.WithTFDir("../../../3-appfactory/apps"),
+		tft.WithTFDir("../../../3-appfactory/apps"),
+		tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute),
 	)
 
 	bootstrap.DefineVerify(func(assert *assert.Assertions) {

test/integration/bootstrap/bootstrap_test.go

@@ -16,14 +16,17 @@ package bootstrap
 
 import (
 	"testing"
+	"time"
 
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
 	"github.com/stretchr/testify/assert"
+	"github.com/terraform-google-modules/enterprise-application/test/integration/testutils"
 )
 
 func TestBootstrap(t *testing.T) {
 	bootstrap := tft.NewTFBlueprintTest(t,
-		         tft.WithTFDir("../../../1-bootstrap"),
+		tft.WithTFDir("../../../1-bootstrap"),
+		tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute),
 	)
 
 	bootstrap.DefineVerify(func(assert *assert.Assertions) {

test/integration/fleetscope/fleetscope_test.go

@@ -17,8 +17,10 @@ package fleetscope
 import (
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
+	"github.com/terraform-google-modules/enterprise-application/test/integration/testutils"
 )
 
 func TestFleetscope(t *testing.T) {
@@ -42,6 +44,7 @@ func TestFleetscope(t *testing.T) {
 			fleetscope := tft.NewTFBlueprintTest(t,
 				tft.WithTFDir(fmt.Sprintf("../../../4-fleetscope/envs/%s", envName)),
 				tft.WithVars(vars),
+				tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute),
 			)
 			fleetscope.Test()
 		})

test/integration/multitenant/multitenant_test.go

@@ -17,8 +17,10 @@ package multitenant
 import (
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
+	"github.com/terraform-google-modules/enterprise-application/test/integration/testutils"
 )
 
 func TestMultitenant(t *testing.T) {
@@ -33,6 +35,7 @@ func TestMultitenant(t *testing.T) {
 			t.Parallel()
 			multitenant := tft.NewTFBlueprintTest(t,
 				tft.WithTFDir(fmt.Sprintf("../../../2-multitenant/envs/%s", envName)),
+				tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute),
 			)
 			multitenant.Test()
 		})

test/integration/testutils/retry.go

@@ -0,0 +1,22 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testutils
+
+var (
+	RetryableTransientErrors = map[string]string{
+		// Error 409: unable to queue the operation
+		".*Error 409.*unable to queue the operation": "Unable to queue operation.",
+	}
+)