fix: Terraform provider updates (#710)

* update cdktf modules interface to support extension.
* make policy helpers public
* access roles for SSM index param
* redeploy images without tag updates

Co-authored-by: Jye Cusch <[email protected]>

cloud/aws/deploytf/.nitric/modules/api/main.tf

@@ -2,6 +2,7 @@ resource "aws_apigatewayv2_api" "api_gateway" {
   name          = var.name
   protocol_type = "HTTP"
   body = var.spec
+  fail_on_warnings = true
   tags = {
     "x-nitric-${var.stack_id}-name" = var.name,
     "x-nitric-${var.stack_id}-type" = "api",

cloud/aws/deploytf/.nitric/modules/service/main.tf

@@ -15,9 +15,13 @@ resource "aws_ecr_repository" "repo" {
 data "aws_ecr_authorization_token" "ecr_auth" {
 }
 
+data "docker_image" "latest" {
+  name = var.image
+}
+
 # Tag the provided docker image with the ECR repository url
 resource "docker_tag" "tag" {
-  source_image = var.image
+  source_image = data.docker_image.latest.repo_digest
   target_image = aws_ecr_repository.repo.repository_url
 }
 
@@ -79,7 +83,7 @@ resource "aws_iam_role_policy_attachment" "basic-execution" {
 
 # Attach vpc access execution role if subnets are provided
 resource "aws_iam_role_policy_attachment" "vpc-access" {
-  count = length(var.subnet_ids) > 0 ? 1 : 0
+  count      = length(var.subnet_ids) > 0 ? 1 : 0
   role       = aws_iam_role.role.name
   policy_arn = "arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole"
 }
@@ -90,8 +94,8 @@ resource "aws_lambda_function" "function" {
   role          = aws_iam_role.role.arn
   image_uri     = "${aws_ecr_repository.repo.repository_url}@${docker_registry_image.push.sha256_digest}"
   package_type  = "Image"
-  timeout = var.timeout
-  memory_size = var.memory
+  timeout       = var.timeout
+  memory_size   = var.memory
   ephemeral_storage {
     size = var.ephemeral_storage
   }

cloud/aws/deploytf/deploy.go

@@ -16,7 +16,6 @@ package deploytf
 
 import (
 	"embed"
-	"io/fs"
 
 	"github.com/aws/jsii-runtime-go"
 	ecrauth "github.com/cdktf/cdktf-provider-aws-go/aws/v19/dataawsecrauthorizationtoken"
@@ -91,8 +90,13 @@ func (a *NitricAwsTerraformProvider) Init(attributes map[string]interface{}) err
 //go:embed .nitric/modules/**/*
 var modules embed.FS
 
-func (a *NitricAwsTerraformProvider) CdkTfModules() (string, fs.FS, error) {
-	return ".nitric/modules", modules, nil
+func (a *NitricAwsTerraformProvider) CdkTfModules() ([]provider.ModuleDirectory, error) {
+	return []provider.ModuleDirectory{
+		{
+			ParentDir: ".nitric/modules",
+			Modules:   modules,
+		},
+	}, nil
 }
 
 func (a *NitricAwsTerraformProvider) RequiredProviders() map[string]interface{} {
@@ -142,7 +146,13 @@ func (a *NitricAwsTerraformProvider) Pre(stack cdktf.TerraformStack, resources [
 }
 
 func (a *NitricAwsTerraformProvider) Post(stack cdktf.TerraformStack) error {
-	return a.resourcesStore(stack)
+	// Give all the Services access to the resource index
+	accessRoleNames := []string{}
+	for _, service := range a.Services {
+		accessRoleNames = append(accessRoleNames, *service.RoleNameOutput())
+	}
+
+	return a.ResourcesStore(stack, accessRoleNames)
 }
 
 // // Post - Called after all resources have been created, before the Pulumi Context is concluded

cloud/aws/deploytf/policy.go

@@ -25,14 +25,7 @@ import (
 	"github.com/samber/lo"
 )
 
-// func md5Hash(b []byte) string {
-// 	hasher := md5.New() //#nosec G401 -- md5 used only to produce a unique ID from non-sensistive information (policy IDs)
-// 	hasher.Write(b)
-
-// 	return hex.EncodeToString(hasher.Sum(nil))
-// }
-
-var awsActionsMap map[resourcespb.Action][]string = map[resourcespb.Action][]string{
+var AwsActionsMap map[resourcespb.Action][]string = map[resourcespb.Action][]string{
 	resourcespb.Action_BucketFileList: {
 		"s3:ListBucket",
 	},
@@ -87,20 +80,20 @@ var awsActionsMap map[resourcespb.Action][]string = map[resourcespb.Action][]str
 	},
 }
 
-func actionsToAwsActions(actions []resourcespb.Action) []string {
+func ActionsToAwsActions(actions []resourcespb.Action) []string {
 	awsActions := make([]string, 0)
 
 	for _, a := range actions {
-		awsActions = append(awsActions, awsActionsMap[a]...)
+		awsActions = append(awsActions, AwsActionsMap[a]...)
 	}
 
 	awsActions = lo.Uniq(awsActions)
 
 	return awsActions
 }
 
-// // discover the arn of a deployed resource
-func (a *NitricAwsTerraformProvider) arnForResource(resource *deploymentspb.Resource) ([]*string, error) {
+// discover the arn of a deployed resource
+func (a *NitricAwsTerraformProvider) ArnForResource(resource *deploymentspb.Resource) ([]*string, error) {
 	switch resource.Id.Type {
 	case resourcespb.ResourceType_Bucket:
 		if b, ok := a.Buckets[resource.Id.Name]; ok {
@@ -149,13 +142,13 @@ func (a *NitricAwsTerraformProvider) roleForPrincipal(resource *deploymentspb.Re
 
 func (a *NitricAwsTerraformProvider) Policy(stack cdktf.TerraformStack, name string, config *deploymentspb.Policy) error {
 	// Get Actions
-	actions := actionsToAwsActions(config.Actions)
+	actions := ActionsToAwsActions(config.Actions)
 
 	// Get Targets
 	targetArns := make([]*string, 0, len(config.Resources))
 
 	for _, res := range config.Resources {
-		if arn, err := a.arnForResource(res); err == nil {
+		if arn, err := a.ArnForResource(res); err == nil {
 			targetArns = append(targetArns, arn...)
 		} else {
 			return fmt.Errorf("failed to create policy, unable to determine resource ARN: %w", err)

cloud/aws/deploytf/resources.go

@@ -24,7 +24,7 @@ import (
 	"github.com/nitrictech/nitric/cloud/aws/deploytf/generated/parameter"
 )
 
-func (a *NitricAwsTerraformProvider) resourcesStore(stack cdktf.TerraformStack) error {
+func (a *NitricAwsTerraformProvider) ResourcesStore(stack cdktf.TerraformStack, accessRoleNames []string) error {
 	index := common.NewResourceIndex()
 
 	for name, bucket := range a.Buckets {
@@ -76,12 +76,6 @@ func (a *NitricAwsTerraformProvider) resourcesStore(stack cdktf.TerraformStack)
 		return fmt.Errorf("failed to marshal resource index: %w", err)
 	}
 
-	// Give all the Services access to the resource index
-	accessRoleNames := []string{}
-	for _, service := range a.Services {
-		accessRoleNames = append(accessRoleNames, *service.RoleNameOutput())
-	}
-
 	parameter.NewParameter(stack, jsii.String("nitric_resources"), &parameter.ParameterConfig{
 		ParameterName:   jsii.Sprintf("/nitric/%s/resource-index", *a.Stack.StackIdOutput()),
 		ParameterValue:  jsii.String(string(indexJson)),

cloud/common/deploy/provider/terraform.go

@@ -21,7 +21,6 @@ import (
 	"io/fs"
 	"net"
 	"os"
-	"path/filepath"
 
 	goruntime "runtime"
 
@@ -36,14 +35,20 @@ import (
 	"google.golang.org/grpc/status"
 )
 
+type ModuleDirectory struct {
+	ParentDir string
+	Modules   fs.FS
+}
+
 type NitricTerraformProvider interface {
 	// Init - Initialize the provider with the given attributes, prior to any resource creation or Pulumi Context creation
 	Init(attributes map[string]interface{}) error
 	// Pre - Called prior to any resource creation, after the Pulumi Context has been established
 	Pre(stack cdktf.TerraformStack, resources []*deploymentspb.Resource) error
 
 	// CdkTfModules - Return the relative parent directory (root golang packed) and embedded modules directory
-	CdkTfModules() (string, fs.FS, error)
+	// CdkTfModules() (string, fs.FS, error)
+	CdkTfModules() ([]ModuleDirectory, error)
 
 	// RequiredProviders - Return a list of required providers for this provider
 	RequiredProviders() map[string]interface{}
@@ -127,53 +132,63 @@ func createTerraformStackForNitricProvider(req *deploymentspb.DeploymentUpReques
 
 	fullStackName := fmt.Sprintf("%s-%s", projectName, stackName)
 
-	parentDir, modules, err := nitricProvider.CdkTfModules()
+	modules, err := nitricProvider.CdkTfModules()
 	if err != nil {
 		return err
 	}
 
-	// modules dir
-	modulesDir := filepath.Join(parentDir)
-
-	err = os.MkdirAll(modulesDir, 0o750)
-	if err != nil {
-		return err
-	}
-	// cleanup the modules when we're done
-	// NOTE: Its importent to ensure that the modules are written to a temporary directory like .nitric
-	defer os.RemoveAll(modulesDir)
-
-	err = fs.WalkDir(modules, ".", func(path string, d fs.DirEntry, err error) error {
+	fses := []fs.FS{}
+	relativeModules := []string{}
+	for _, module := range modules {
+		relativeModules = append(relativeModules, module.ParentDir)
+		err = os.MkdirAll(module.ParentDir, 0o750)
 		if err != nil {
 			return err
 		}
 
-		if !d.IsDir() {
-			data, err := modules.Open(path)
+		defer os.RemoveAll(module.ParentDir)
+		fses = append(fses, module.Modules)
+	}
+
+	// modules dir
+
+	// cleanup the modules when we're done
+	// NOTE: Its important to ensure that the modules are written to a temporary directory like .nitric
+
+	for _, fsx := range fses {
+		err = fs.WalkDir(fsx, ".", func(path string, d fs.DirEntry, err error) error {
 			if err != nil {
 				return err
 			}
-			defer data.Close()
 
-			//#nosec G304 -- path unpacking known modules embedded fs
-			out, err := os.Create(path)
-			if err != nil {
+			if !d.IsDir() {
+				data, err := fsx.Open(path)
+				if err != nil {
+					return err
+				}
+				defer data.Close()
+
+				//#nosec G304 -- path unpacking known modules embedded fs
+				out, err := os.Create(path)
+				if err != nil {
+					return err
+				}
+				defer out.Close()
+
+				fmt.Println("Writing module to", path)
+				_, err = io.Copy(out, data)
 				return err
 			}
-			defer out.Close()
 
-			_, err = io.Copy(out, data)
+			return os.MkdirAll(path, 0o750)
+		})
+		if err != nil {
 			return err
 		}
-
-		return os.MkdirAll(path, 0o750)
-	})
-	if err != nil {
-		return err
 	}
 
 	appCtx := map[string]interface{}{
-		"cdktfRelativeModules": []string{filepath.Join(modulesDir)},
+		"cdktfRelativeModules": relativeModules,
 		// Ensure static output
 		"cdktfStaticModuleAssetHash": "nitric_modules",
 	}

cloud/gcp/deploytf/deploy.go

@@ -16,7 +16,6 @@ package deploytf
 
 import (
 	"embed"
-	"io/fs"
 
 	"github.com/aws/jsii-runtime-go"
 	dockerprovider "github.com/cdktf/cdktf-provider-docker-go/docker/v11/provider"
@@ -99,8 +98,13 @@ func (a *NitricGcpTerraformProvider) RequiredProviders() map[string]interface{}
 	}
 }
 
-func (a *NitricGcpTerraformProvider) CdkTfModules() (string, fs.FS, error) {
-	return ".nitric/modules", modules, nil
+func (a *NitricGcpTerraformProvider) CdkTfModules() ([]provider.ModuleDirectory, error) {
+	return []provider.ModuleDirectory{
+		{
+			ParentDir: ".nitric/modules",
+			Modules:   modules,
+		},
+	}, nil
 }
 
 func (a *NitricGcpTerraformProvider) prepareGcpProviders(stack cdktf.TerraformStack) {