Output chaining in ev2 pipelines

Makefile

@@ -15,7 +15,7 @@ all: test lint
 # There is currently no convenient way to run tests against a whole Go workspace
 # https://github.com/golang/go/issues/50745
 test:
-	go list -f '{{.Dir}}/...' -m |RUN_TEMPLATIZE_E2E=true xargs go test -timeout 120s -tags=$(GOTAGS) -cover
+	go list -f '{{.Dir}}/...' -m |RUN_TEMPLATIZE_E2E=true xargs go test -timeout 1200s -tags=$(GOTAGS) -cover
 .PHONY: test
 
 # There is currently no convenient way to run golangci-lint against a whole Go workspace

backend/pipeline.yaml

@@ -8,7 +8,7 @@ resourceGroups:
   - name: deploy
     action: Shell
     command: make deploy
-    env:
+    variables:
     - name: ARO_HCP_IMAGE_ACR
       configRef: svcAcrName
     - name: LOCATION

frontend/pipeline.yaml

@@ -9,10 +9,10 @@ resourceGroups:
     action: Shell
     command: make deploy
     dryRun:
-      envVars:
+      variables:
         - name: HELM_DRY_RUN
           value: "--dry-run=server --debug"
-    env:
+    variables:
     - name: ARO_HCP_IMAGE_ACR
       configRef: svcAcrName
     - name: LOCATION

maestro/server/pipeline.yaml

@@ -8,7 +8,7 @@ resourceGroups:
   - name: deploy
     action: Shell
     command: make deploy
-    env:
+    variables:
     - name: EVENTGRID_NAME
       configRef: maestro.eventGrid.name
     - name: REGION_RG

tooling/templatize/internal/end2end/e2e.go

@@ -1,12 +1,17 @@
 package testutil
 
 import (
+	"context"
 	"fmt"
-	"math/rand/v2"
 	"os"
 
+	"math/rand/v2"
+
 	"gopkg.in/yaml.v2"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
+
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/config"
 	"github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline"
 )
@@ -19,21 +24,28 @@ func shouldRunE2E() bool {
 
 type E2E interface {
 	SetConfig(updates config.Variables)
-	UseRandomRG()
+	UseRandomRG() func() error
+	AddBicepTemplate(template, templateFileName, paramfile, paramfileName string)
 	AddStep(step pipeline.Step)
 	SetOSArgs()
 	Persist() error
 }
 
+type bicepTemplate struct {
+	bicepFile     string
+	bicepFileName string
+	paramFile     string
+	paramFileName string
+}
+
 type e2eImpl struct {
-	config    config.Variables
-	makefile  string
-	pipeline  pipeline.Pipeline
-	bicepFile string
-	paramFile string
-	schema    string
-	tmpdir    string
-	rgName    string
+	config   config.Variables
+	makefile string
+	pipeline pipeline.Pipeline
+	biceps   []bicepTemplate
+	schema   string
+	tmpdir   string
+	rgName   string
 }
 
 var _ E2E = &e2eImpl{}
@@ -71,13 +83,14 @@ func newE2E(tmpdir string) e2eImpl {
 			},
 		},
 		rgName: defaultRgName,
+		biceps: []bicepTemplate{},
 	}
 
 	imp.SetOSArgs()
 	return imp
 }
 
-func (e *e2eImpl) UseRandomRG() {
+func (e *e2eImpl) UseRandomRG() func() error {
 
 	chars := []rune("abcdefghijklmnopqrstuvwxyz0123456789")
 	rg := "templatize-e2e-"
@@ -87,6 +100,23 @@ func (e *e2eImpl) UseRandomRG() {
 	}
 	e.rgName = rg
 	e.SetConfig(config.Variables{"defaults": config.Variables{"rg": rg}})
+
+	return func() error {
+		subsriptionID, err := pipeline.LookupSubscriptionID(context.Background(), "ARO Hosted Control Planes (EA Subscription 1)")
+		if err != nil {
+			return err
+		}
+		cred, err := azidentity.NewDefaultAzureCredential(nil)
+		if err != nil {
+			return err
+		}
+		rgClient, err := armresources.NewResourceGroupsClient(subsriptionID, cred, nil)
+		if err != nil {
+			return err
+		}
+		_, err = rgClient.BeginDelete(context.Background(), e.rgName, nil)
+		return err
+	}
 }
 
 func (e *e2eImpl) SetOSArgs() {
@@ -111,16 +141,28 @@ func (e *e2eImpl) SetConfig(updates config.Variables) {
 	config.MergeVariables(e.config, updates)
 }
 
-func (e *e2eImpl) Persist() error {
-	if e.bicepFile != "" && e.paramFile != "" {
-		err := os.WriteFile(e.tmpdir+"/test.bicep", []byte(e.bicepFile), 0644)
-		if err != nil {
-			return err
-		}
+func (e *e2eImpl) AddBicepTemplate(template, templateFileName, paramfile, paramfileName string) {
+	e.biceps = append(e.biceps, bicepTemplate{
+		bicepFile:     template,
+		bicepFileName: templateFileName,
+		paramFile:     paramfile,
+		paramFileName: paramfileName,
+	})
+}
 
-		err = os.WriteFile(e.tmpdir+"/test.bicepparm", []byte(e.paramFile), 0644)
-		if err != nil {
-			return err
+func (e *e2eImpl) Persist() error {
+	if len(e.biceps) != 0 {
+		for _, b := range e.biceps {
+
+			err := os.WriteFile(e.tmpdir+"/"+b.bicepFileName, []byte(b.bicepFile), 0644)
+			if err != nil {
+				return err
+			}
+
+			err = os.WriteFile(e.tmpdir+"/"+b.paramFileName, []byte(b.paramFile), 0644)
+			if err != nil {
+				return err
+			}
 		}
 	}
 

tooling/templatize/internal/end2end/e2e_test.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dns/armdns"
-	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
 )
 
 func persistAndRun(t *testing.T, e2eImpl E2E) {
@@ -39,7 +38,7 @@ func TestE2EMake(t *testing.T) {
 		Name:    "test",
 		Action:  "Shell",
 		Command: "make test",
-		Env: []pipeline.EnvVar{
+		Variables: []pipeline.Variable{
 			{
 				Name:      "TEST_ENV",
 				ConfigRef: "test_env",
@@ -95,18 +94,23 @@ func TestE2EArmDeploy(t *testing.T) {
 		Parameters: "test.bicepparm",
 	})
 
-	e2eImpl.UseRandomRG()
+	cleanup := e2eImpl.UseRandomRG()
+	defer func() {
+		err := cleanup()
+		assert.NilError(t, err)
+	}()
 
-	e2eImpl.bicepFile = `
+	bicepFile := `
 param zoneName string
 resource symbolicname 'Microsoft.Network/dnsZones@2018-05-01' = {
   location: 'global'
   name: zoneName
 }`
-	e2eImpl.paramFile = `
+	paramFile := `
 using 'test.bicep'
 param zoneName = 'e2etestarmdeploy.foo.bar.example.com'
 `
+	e2eImpl.AddBicepTemplate(bicepFile, "test.bicep", paramFile, "test.bicepparm")
 
 	persistAndRun(t, &e2eImpl)
 
@@ -117,29 +121,161 @@ param zoneName = 'e2etestarmdeploy.foo.bar.example.com'
 	cred, err := azidentity.NewDefaultAzureCredential(nil)
 	assert.NilError(t, err)
 
-	rgClient, err := armresources.NewResourceGroupsClient(subsriptionID, cred, nil)
-	assert.NilError(t, err)
-
-	existence, err := rgClient.CheckExistence(context.Background(), e2eImpl.rgName, nil)
-	assert.NilError(t, err)
-	assert.Assert(t, existence.Success)
-
 	zonesClient, err := armdns.NewZonesClient(subsriptionID, cred, nil)
 	assert.NilError(t, err)
 
 	zoneResp, err := zonesClient.Get(context.Background(), e2eImpl.rgName, "e2etestarmdeploy.foo.bar.example.com", nil)
 	assert.NilError(t, err)
 	assert.Equal(t, *zoneResp.Name, "e2etestarmdeploy.foo.bar.example.com")
+}
 
-	delResponse, err := zonesClient.BeginDelete(context.Background(), e2eImpl.rgName, "e2etestarmdeploy.foo.bar.example.com", nil)
-	assert.NilError(t, err)
+func TestE2EShell(t *testing.T) {
+	if !shouldRunE2E() {
+		t.Skip("Skipping end-to-end tests")
+	}
+
+	tmpDir := t.TempDir()
+
+	e2eImpl := newE2E(tmpDir)
+
+	e2eImpl.AddStep(pipeline.Step{
+		Name:    "readInput",
+		Action:  "Shell",
+		Command: "/usr/bin/echo ${PWD} > env.txt",
+	})
 
-	_, err = delResponse.PollUntilDone(context.Background(), nil)
+	persistAndRun(t, &e2eImpl)
+
+	io, err := os.ReadFile(tmpDir + "/env.txt")
 	assert.NilError(t, err)
+	assert.Equal(t, string(io), tmpDir+"\n")
+}
+
+func TestE2EArmDeployWithOutput(t *testing.T) {
+	if !shouldRunE2E() {
+		t.Skip("Skipping end-to-end tests")
+	}
+
+	tmpDir := t.TempDir()
+
+	e2eImpl := newE2E(tmpDir)
+	e2eImpl.AddStep(pipeline.Step{
+		Name:       "createZone",
+		Action:     "ARM",
+		Template:   "test.bicep",
+		Parameters: "test.bicepparm",
+	})
+
+	e2eImpl.AddStep(pipeline.Step{
+		Name:    "readInput",
+		Action:  "Shell",
+		Command: "echo ${zoneName} > env.txt",
+		Variables: []pipeline.Variable{
+			{
+				Name: "zoneName",
+				Input: &pipeline.Input{
+					Name: "zoneName",
+					Step: "createZone",
+				},
+			},
+		},
+	})
+
+	cleanup := e2eImpl.UseRandomRG()
+	defer func() {
+		err := cleanup()
+		assert.NilError(t, err)
+	}()
+
+	bicepFile := `
+param zoneName string
+output zoneName string = zoneName`
+	paramFile := `
+using 'test.bicep'
+param zoneName = 'e2etestarmdeploy.foo.bar.example.com'
+`
+	e2eImpl.AddBicepTemplate(bicepFile, "test.bicep", paramFile, "test.bicepparm")
+	persistAndRun(t, &e2eImpl)
 
-	rgDelResponse, err := rgClient.BeginDelete(context.Background(), e2eImpl.rgName, nil)
+	io, err := os.ReadFile(tmpDir + "/env.txt")
 	assert.NilError(t, err)
+	assert.Equal(t, string(io), "e2etestarmdeploy.foo.bar.example.com\n")
+}
+
+func TestE2EArmDeployWithOutputToArm(t *testing.T) {
+	if !shouldRunE2E() {
+		t.Skip("Skipping end-to-end tests")
+	}
 
-	_, err = rgDelResponse.PollUntilDone(context.Background(), nil)
+	tmpDir := t.TempDir()
+
+	e2eImpl := newE2E(tmpDir)
+	e2eImpl.AddStep(pipeline.Step{
+		Name:       "parameterA",
+		Action:     "ARM",
+		Template:   "testa.bicep",
+		Parameters: "testa.bicepparm",
+	})
+
+	e2eImpl.AddStep(pipeline.Step{
+		Name:       "parameterB",
+		Action:     "ARM",
+		Template:   "testb.bicep",
+		Parameters: "testb.bicepparm",
+		Variables: []pipeline.Variable{
+			{
+				Name: "parameterB",
+				Input: &pipeline.Input{
+					Name: "parameterA",
+					Step: "parameterA",
+				},
+			},
+		},
+	})
+
+	e2eImpl.AddStep(pipeline.Step{
+		Name:    "readInput",
+		Action:  "Shell",
+		Command: "echo ${end} > env.txt",
+		Variables: []pipeline.Variable{
+			{
+				Name: "end",
+				Input: &pipeline.Input{
+					Name: "parameterC",
+					Step: "parameterB",
+				},
+			},
+		},
+	})
+
+	e2eImpl.AddBicepTemplate(`
+param parameterA string
+output parameterA string = parameterA`,
+		"testa.bicep",
+		`
+using 'testa.bicep'
+param parameterA = 'Hello Bicep'`,
+		"testa.bicepparm")
+
+	e2eImpl.AddBicepTemplate(`
+param parameterB string
+output parameterC string = parameterB
+`,
+		"testb.bicep",
+		`
+using 'testb.bicep'
+param parameterB = '{{ .parameterB }}'
+`,
+		"testb.bicepparm")
+
+	cleanup := e2eImpl.UseRandomRG()
+	defer func() {
+		err := cleanup()
+		assert.NilError(t, err)
+	}()
+	persistAndRun(t, &e2eImpl)
+
+	io, err := os.ReadFile(tmpDir + "/env.txt")
 	assert.NilError(t, err)
+	assert.Equal(t, string(io), "Hello Bicep\n")
 }