diff --git a/tooling/templatize/pkg/config/config.go b/tooling/templatize/pkg/config/config.go index ee92704b0..7fb5bb50f 100644 --- a/tooling/templatize/pkg/config/config.go +++ b/tooling/templatize/pkg/config/config.go @@ -247,20 +247,28 @@ func (cp *configProviderImpl) loadConfig(configReplacements *ConfigReplacements) // PreprocessFile reads and processes a gotemplate // The path will be read as is. It parses the file as a template, and executes it with the provided variables. func PreprocessFile(templateFilePath string, vars map[string]any) ([]byte, error) { - tmpl := template.New("file") content, err := os.ReadFile(templateFilePath) if err != nil { return nil, fmt.Errorf("failed to read file %s: %w", templateFilePath, err) } + processedContent, err := PreprocessContent(content, vars) + if err != nil { + return nil, fmt.Errorf("failed to preprocess file %s: %w", templateFilePath, err) + } + return processedContent, nil +} - tmpl, err = tmpl.Parse(string(content)) +// PreprocessContent processes a gotemplate from memory +func PreprocessContent(content []byte, vars map[string]any) ([]byte, error) { + tmpl := template.New("file") + tmpl, err := tmpl.Parse(string(content)) if err != nil { - return nil, fmt.Errorf("failed to parse template %s: %w", templateFilePath, err) + return nil, fmt.Errorf("failed to parse template: %w", err) } var tmplBytes bytes.Buffer if err := tmpl.Option("missingkey=error").Execute(&tmplBytes, vars); err != nil { - return nil, fmt.Errorf("failed to execute template %s: %w", templateFilePath, err) + return nil, fmt.Errorf("failed to execute template: %w", err) } return tmplBytes.Bytes(), nil } diff --git a/tooling/templatize/pkg/config/config_test.go b/tooling/templatize/pkg/config/config_test.go index ddf4286f1..90419f297 100644 --- a/tooling/templatize/pkg/config/config_test.go +++ b/tooling/templatize/pkg/config/config_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/stretchr/testify/assert" + + "github.com/Azure/ARO-HCP/tooling/templatize/internal/testutil" ) func TestConfigProvider(t *testing.T) { @@ -231,3 +233,78 @@ func TestConvertToInterface(t *testing.T) { assert.IsType(t, expected, map[string]any{}) assert.IsType(t, expected["key2"], map[string]any{}) } + +func TestPreprocessContent(t *testing.T) { + fileContent, err := os.ReadFile("../../testdata/test.bicepparam") + assert.Nil(t, err) + + processed, err := PreprocessContent( + fileContent, + map[string]any{ + "regionRG": "bahamas", + "clusterService": map[string]any{ + "imageTag": "cs-image", + }, + }, + ) + assert.Nil(t, err) + testutil.CompareWithFixture(t, processed, testutil.WithExtension(".bicepparam")) +} + +func TestPreprocessContentMissingKey(t *testing.T) { + testCases := []struct { + name string + content string + vars map[string]any + shouldFail bool + }{ + { + name: "missing key", + content: "foo: {{ .bar }}", + vars: map[string]any{ + "baz": "bar", + }, + shouldFail: true, + }, + { + name: "missing nested key", + content: "foo: {{ .bar.baz }}", + vars: map[string]any{ + "baz": "bar", + }, + shouldFail: true, + }, + { + name: "no missing key", + content: "foo: {{ .bar }}", + vars: map[string]any{ + "bar": "bar", + }, + shouldFail: false, + }, + { + name: "no missing nested key", + content: "foo: {{ .bar.baz }}", + vars: map[string]any{ + "bar": map[string]any{ + "baz": "baz", + }, + }, + shouldFail: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := PreprocessContent( + []byte(tc.content), + tc.vars, + ) + if tc.shouldFail { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} diff --git a/tooling/templatize/pkg/ev2/pipeline.go b/tooling/templatize/pkg/ev2/pipeline.go index 1b5e854d5..b1dcc0179 100644 --- a/tooling/templatize/pkg/ev2/pipeline.go +++ b/tooling/templatize/pkg/ev2/pipeline.go @@ -1,6 +1,7 @@ package ev2 import ( + "fmt" "os" "path/filepath" @@ -10,73 +11,105 @@ import ( "github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline" ) -func PrecompilePipelineForEV2(pipelineFilePath string, vars config.Variables) (string, error) { - // switch to the pipeline file dir so all relative paths are resolved correctly - originalDir, err := os.Getwd() - if err != nil { - return "", nil - } - pipelineDir := filepath.Dir(pipelineFilePath) - err = os.Chdir(pipelineDir) +const precompiledPrefix = "ev2-precompiled-" + +func PrecompilePipelineFileForEV2(pipelineFilePath string, vars config.Variables) (string, error) { + precompiledPipeline, err := PrecompilePipelineForEV2(pipelineFilePath, vars) if err != nil { return "", err } - defer func() { - _ = os.Chdir(originalDir) - }() - // precompile the pipeline file - pipelineFileName := filepath.Base(pipelineFilePath) - p, err := pipeline.NewPipelineFromFile(pipelineFileName, vars) + // store as new file + pipelineBytes, err := yaml.Marshal(precompiledPipeline) if err != nil { return "", err } - err = processPipelineForEV2(p, vars) + err = os.WriteFile(precompiledPipeline.PipelineFilePath(), pipelineBytes, 0644) if err != nil { return "", err } - // store as new file - pipelineBytes, err := yaml.Marshal(p) + return precompiledPipeline.PipelineFilePath(), nil +} + +func PrecompilePipelineForEV2(pipelineFilePath string, vars config.Variables) (*pipeline.Pipeline, error) { + // load the pipeline and referenced files + originalPipeline, err := pipeline.NewPipelineFromFile(pipelineFilePath, vars) if err != nil { - return "", err + return nil, err } - newPipelineFileName := "ev2-precompiled-" + pipelineFileName - err = os.WriteFile(newPipelineFileName, pipelineBytes, 0644) + referencedFiles, err := readReferencedPipelineFiles(originalPipeline) if err != nil { - return "", err + return nil, fmt.Errorf("failed to read referenced files of pipeline %s: %w", originalPipeline.PipelineFilePath(), err) + } + + // precompile the pipeline and referenced files + processedPipeline, processedFiles, err := processPipelineForEV2(originalPipeline, referencedFiles, vars) + if err != nil { + return nil, err + } + + // store the processed files to disk relative to the pipeline directory + for filePath, content := range processedFiles { + absFilePath, err := processedPipeline.AbsoluteFilePath(filePath) + if err != nil { + return nil, fmt.Errorf("failed to get absolute file path for %q: %w", filePath, err) + } + err = os.WriteFile(absFilePath, content, 0644) + if err != nil { + return nil, fmt.Errorf("failed to write precompiled file %q: %w", filePath, err) + } } - return filepath.Join(pipelineDir, newPipelineFileName), nil + return processedPipeline, nil } -func processPipelineForEV2(p *pipeline.Pipeline, vars config.Variables) error { - _, scopeBoundVars := EV2Mapping(vars, []string{}) +func readReferencedPipelineFiles(p *pipeline.Pipeline) (map[string][]byte, error) { + referencedFiles := make(map[string][]byte) for _, rg := range p.ResourceGroups { for _, step := range rg.Steps { if step.Parameters != "" { - newParameterFilePath, err := precompileFileAndStore(step.Parameters, scopeBoundVars) + absFilePath, err := p.AbsoluteFilePath(step.Parameters) if err != nil { - return err + return nil, fmt.Errorf("failed to get absolute file path for %q: %w", step.Parameters, err) } - step.Parameters = newParameterFilePath + paramFileContent, err := os.ReadFile(absFilePath) + if err != nil { + return nil, fmt.Errorf("failed to read parameter file %q: %w", step.Parameters, err) + } + referencedFiles[step.Parameters] = paramFileContent } } } - return nil + return referencedFiles, nil } -func precompileFileAndStore(filePath string, vars map[string]interface{}) (string, error) { - preprocessedBytes, err := config.PreprocessFile(filePath, vars) +func processPipelineForEV2(p *pipeline.Pipeline, referencedFiles map[string][]byte, vars config.Variables) (*pipeline.Pipeline, map[string][]byte, error) { + processingPipeline, err := p.DeepCopy(buildPrefixedFilePath(p.PipelineFilePath(), precompiledPrefix)) if err != nil { - return "", err + return nil, nil, err } - newFilePath := buildPrefixedFilePath(filePath, "ev2-precompiled-") - err = os.WriteFile(newFilePath, preprocessedBytes, 0644) - if err != nil { - return "", err + processedFiles := make(map[string][]byte) + _, scopeBoundVars := EV2Mapping(vars, []string{}) + for _, rg := range processingPipeline.ResourceGroups { + for _, step := range rg.Steps { + // preprocess the parameters file with scopebinding variables + if step.Parameters != "" { + paramFileContent, ok := referencedFiles[step.Parameters] + if !ok { + return nil, nil, fmt.Errorf("parameter file %q not found", step.Parameters) + } + preprocessedBytes, err := config.PreprocessContent(paramFileContent, scopeBoundVars) + if err != nil { + return nil, nil, err + } + newParameterFilePath := buildPrefixedFilePath(step.Parameters, precompiledPrefix) + processedFiles[newParameterFilePath] = preprocessedBytes + step.Parameters = newParameterFilePath + } + } } - return newFilePath, nil + return processingPipeline, processedFiles, nil } func buildPrefixedFilePath(path, prefix string) string { diff --git a/tooling/templatize/pkg/ev2/pipeline_test.go b/tooling/templatize/pkg/ev2/pipeline_test.go index 540476335..48c4264d6 100644 --- a/tooling/templatize/pkg/ev2/pipeline_test.go +++ b/tooling/templatize/pkg/ev2/pipeline_test.go @@ -1,40 +1,42 @@ package ev2 import ( - "fmt" - "os" "testing" + "gopkg.in/yaml.v3" + + "github.com/Azure/ARO-HCP/tooling/templatize/internal/testutil" "github.com/Azure/ARO-HCP/tooling/templatize/pkg/config" "github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline" ) -func TestPrecompilePipelineForEV2(t *testing.T) { - defer func() { - _ = os.Remove("../../testdata/ev2-precompiled-pipeline.yaml") - _ = os.Remove("../../testdata/ev2-precompiled-test.bicepparam") - }() - +func TestProcessPipelineForEV2(t *testing.T) { configProvider := config.NewConfigProvider("../../testdata/config.yaml") - vars, err := configProvider.GetVariables("public", "int", "", newEv2ConfigReplacements()) + vars, err := configProvider.GetVariables("public", "int", "", NewEv2ConfigReplacements()) if err != nil { t.Errorf("failed to get variables: %v", err) } - newPipelinePath, err := PrecompilePipelineForEV2("../../testdata/pipeline.yaml", vars) + originalPipeline, err := pipeline.NewPipelineFromFile("../../testdata/pipeline.yaml", vars) + if err != nil { + t.Errorf("failed to read new pipeline: %v", err) + } + files := make(map[string][]byte) + files["test.bicepparam"] = []byte("param regionRG = '{{ .regionRG }}'") + + newPipeline, newFiles, err := processPipelineForEV2(originalPipeline, files, vars) if err != nil { t.Errorf("failed to precompile pipeline: %v", err) } - p, err := pipeline.NewPipelineFromFile(newPipelinePath, vars) + // verify pipeline + pipelineContent, err := yaml.Marshal(newPipeline) if err != nil { - t.Errorf("failed to read new pipeline: %v", err) + t.Errorf("failed to marshal processed pipeline: %v", err) } - fmt.Println(p) - expectedParamsPath := "ev2-precompiled-test.bicepparam" + testutil.CompareWithFixture(t, pipelineContent, testutil.WithExtension("pipeline.yaml")) - armStep := p.ResourceGroups[0].Steps[2] - if armStep.Parameters != expectedParamsPath { - t.Errorf("expected parameters path %v, but got %v", expectedParamsPath, armStep.Parameters) + // verify referenced files + for filePath, content := range newFiles { + testutil.CompareWithFixture(t, content, testutil.WithExtension(filePath)) } - // TODO improve test, check against fixture } diff --git a/tooling/templatize/pkg/ev2/utils.go b/tooling/templatize/pkg/ev2/utils.go index 76814efe8..b59d613b1 100644 --- a/tooling/templatize/pkg/ev2/utils.go +++ b/tooling/templatize/pkg/ev2/utils.go @@ -10,7 +10,7 @@ import ( // This package contains helper functions to extract EV2 conformant data from a config.yaml file. // -func newEv2ConfigReplacements() *config.ConfigReplacements { +func NewEv2ConfigReplacements() *config.ConfigReplacements { return config.NewConfigReplacements( "$location()", "$(regionShortName)", @@ -23,7 +23,7 @@ func newEv2ConfigReplacements() *config.ConfigReplacements { // The variable values are formatted to contain EV2 $location(), $stamp() and $(serviceConfigVar) variables. // This function is useful to get the variables to fill the `Settings` section of an EV2 `ServiceConfig.json“ func GetNonRegionalServiceConfigVariables(configProvider config.ConfigProvider, cloud, deployEnv string) (config.Variables, error) { - return configProvider.GetVariables(cloud, deployEnv, "", newEv2ConfigReplacements()) + return configProvider.GetVariables(cloud, deployEnv, "", NewEv2ConfigReplacements()) } // GetRegionalServiceConfigVariableOverrides returns the regional overrides of a config.yaml file. @@ -36,7 +36,7 @@ func GetRegionalServiceConfigVariableOverrides(configProvider config.ConfigProvi } overrides := make(map[string]config.Variables) for _, region := range regions { - regionOverrides, err := configProvider.GetRegionOverrides(cloud, deployEnv, region, newEv2ConfigReplacements()) + regionOverrides, err := configProvider.GetRegionOverrides(cloud, deployEnv, region, NewEv2ConfigReplacements()) if err != nil { return nil, err } @@ -49,7 +49,7 @@ func GetRegionalServiceConfigVariableOverrides(configProvider config.ConfigProvi // It uses the provided configProvider to fetch the variables, flattens them into a __VAR__ = $config(var) formatted map. // This function is useful to get the find/replace pairs for an EV2 `ScopeBinding.json` func ScopeBindingVariables(configProvider config.ConfigProvider, cloud, deployEnv string) (map[string]string, error) { - vars, err := configProvider.GetVariables(cloud, deployEnv, "", newEv2ConfigReplacements()) + vars, err := configProvider.GetVariables(cloud, deployEnv, "", NewEv2ConfigReplacements()) if err != nil { return nil, err } @@ -65,7 +65,7 @@ func ScopeBindingVariables(configProvider config.ConfigProvider, cloud, deployEn // while maintaining EV2 conformant system variables. // This function is useful to process a pipeline.yaml file so that it contains EV2 system variables. func PreprocessFileForEV2SystemVars(configProvider config.ConfigProvider, cloud, deployEnv string, templateFile string) ([]byte, error) { - vars, err := configProvider.GetVariables(cloud, deployEnv, "", newEv2ConfigReplacements()) + vars, err := configProvider.GetVariables(cloud, deployEnv, "", NewEv2ConfigReplacements()) if err != nil { return nil, err } @@ -77,7 +77,7 @@ func PreprocessFileForEV2SystemVars(configProvider config.ConfigProvider, cloud, // This function is useful to process bicepparam files so that they can be used within EV2 together // with scopebinding. func PreprocessFileForEV2ScopeBinding(configProvider config.ConfigProvider, cloud, deployEnv string, templateFile string) ([]byte, error) { - vars, err := configProvider.GetVariables(cloud, deployEnv, "", newEv2ConfigReplacements()) + vars, err := configProvider.GetVariables(cloud, deployEnv, "", NewEv2ConfigReplacements()) if err != nil { return nil, err } diff --git a/tooling/templatize/pkg/pipeline/arm.go b/tooling/templatize/pkg/pipeline/arm.go index f7fcf5a4b..dc7a196e7 100644 --- a/tooling/templatize/pkg/pipeline/arm.go +++ b/tooling/templatize/pkg/pipeline/arm.go @@ -10,7 +10,7 @@ import ( "github.com/go-logr/logr" ) -func (s *step) runArmStep(ctx context.Context, executionTarget *ExecutionTarget, options *PipelineRunOptions) error { +func (s *Step) runArmStep(ctx context.Context, executionTarget *ExecutionTarget, options *PipelineRunOptions) error { logger := logr.FromContextOrDiscard(ctx) // Transform Bicep to ARM @@ -59,7 +59,7 @@ func (s *step) runArmStep(ctx context.Context, executionTarget *ExecutionTarget, return nil } -func (s *step) ensureResourceGroupExists(ctx context.Context, executionTarget *ExecutionTarget) error { +func (s *Step) ensureResourceGroupExists(ctx context.Context, executionTarget *ExecutionTarget) error { // Create a new Azure identity client cred, err := azidentity.NewDefaultAzureCredential(nil) if err != nil { diff --git a/tooling/templatize/pkg/pipeline/common.go b/tooling/templatize/pkg/pipeline/common.go new file mode 100644 index 000000000..2a2584000 --- /dev/null +++ b/tooling/templatize/pkg/pipeline/common.go @@ -0,0 +1,30 @@ +package pipeline + +import ( + "fmt" + "path/filepath" + + "gopkg.in/yaml.v3" +) + +func (p *Pipeline) DeepCopy(newPipelineFilePath string) (*Pipeline, error) { + copy := new(Pipeline) + data, err := yaml.Marshal(p) + if err != nil { + return nil, fmt.Errorf("failed to marshal pipeline: %v", err) + } + err = yaml.Unmarshal(data, copy) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal pipeline: %v", err) + } + copy.pipelineFilePath = newPipelineFilePath + return copy, nil +} + +func (p *Pipeline) PipelineFilePath() string { + return p.pipelineFilePath +} + +func (p *Pipeline) AbsoluteFilePath(filePath string) (string, error) { + return filepath.Abs(filepath.Join(filepath.Dir(p.pipelineFilePath), filePath)) +} diff --git a/tooling/templatize/pkg/pipeline/common_test.go b/tooling/templatize/pkg/pipeline/common_test.go new file mode 100644 index 000000000..7b5d15b65 --- /dev/null +++ b/tooling/templatize/pkg/pipeline/common_test.go @@ -0,0 +1,85 @@ +package pipeline + +import ( + "path/filepath" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "gotest.tools/v3/assert" + + "github.com/Azure/ARO-HCP/tooling/templatize/pkg/config" +) + +func TestDeepCopy(t *testing.T) { + configProvider := config.NewConfigProvider("../../testdata/config.yaml") + vars, err := configProvider.GetVariables("public", "int", "", config.NewConfigReplacements("r", "sr", "s")) + if err != nil { + t.Errorf("failed to get variables: %v", err) + } + pipeline, err := NewPipelineFromFile("../../testdata/pipeline.yaml", vars) + if err != nil { + t.Errorf("failed to read new pipeline: %v", err) + } + + newPipelinePath := "new-pipeline.yaml" + pipelineCopy, err := pipeline.DeepCopy(newPipelinePath) + if err != nil { + t.Errorf("failed to copy pipeline: %v", err) + } + + assert.Assert(t, pipeline != pipelineCopy, "expected pipeline and copy to be different") + assert.Equal(t, pipelineCopy.PipelineFilePath(), newPipelinePath, "expected pipeline copy to have new path") + + if diff := cmp.Diff(pipeline, pipelineCopy, cmpopts.IgnoreUnexported(Pipeline{}, Step{})); diff != "" { + t.Errorf("got diffs after pipeline deep copy: %v", diff) + } +} + +func TestAbsoluteFilePath(t *testing.T) { + configProvider := config.NewConfigProvider("../../testdata/config.yaml") + vars, err := configProvider.GetVariables("public", "int", "", config.NewConfigReplacements("r", "sr", "s")) + if err != nil { + t.Errorf("failed to get variables: %v", err) + } + pipeline, err := NewPipelineFromFile("../../testdata/pipeline.yaml", vars) + if err != nil { + t.Errorf("failed to read new pipeline: %v", err) + } + + abspath := func(path string) string { + abs, _ := filepath.Abs(path) + return abs + } + testCases := []struct { + name string + relativeFile string + absoluteFile string + }{ + { + name: "basic", + relativeFile: "test.bicepparam", + absoluteFile: abspath("../../testdata/test.bicepparam"), + }, + { + name: "go one lower", + relativeFile: "../test.bicepparam", + absoluteFile: abspath("../../test.bicepparam"), + }, + { + name: "subdir", + relativeFile: "subdir/test.bicepparam", + absoluteFile: abspath("../../testdata/subdir/test.bicepparam"), + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + abs, err := pipeline.AbsoluteFilePath(tc.relativeFile) + if err != nil { + t.Errorf("failed to get absolute file path: %v", err) + } + assert.Equal(t, abs, tc.absoluteFile, "expected absolute file path to be correct") + }) + } + +} diff --git a/tooling/templatize/pkg/pipeline/inspect.go b/tooling/templatize/pkg/pipeline/inspect.go index 2374ce84f..7576b8277 100644 --- a/tooling/templatize/pkg/pipeline/inspect.go +++ b/tooling/templatize/pkg/pipeline/inspect.go @@ -8,7 +8,7 @@ import ( "github.com/Azure/ARO-HCP/tooling/templatize/pkg/config" ) -type StepInspectScope func(*step, *InspectOptions, io.Writer) error +type StepInspectScope func(*Step, *InspectOptions, io.Writer) error func NewStepInspectScopes() map[string]StepInspectScope { return map[string]StepInspectScope{ @@ -57,7 +57,7 @@ func (p *Pipeline) Inspect(ctx context.Context, options *InspectOptions, writer return fmt.Errorf("step %q not found", options.Step) } -func inspectVars(s *step, options *InspectOptions, writer io.Writer) error { +func inspectVars(s *Step, options *InspectOptions, writer io.Writer) error { var envVars map[string]string var err error switch s.Action { diff --git a/tooling/templatize/pkg/pipeline/inspect_test.go b/tooling/templatize/pkg/pipeline/inspect_test.go index f8be4c7b3..dc308715e 100644 --- a/tooling/templatize/pkg/pipeline/inspect_test.go +++ b/tooling/templatize/pkg/pipeline/inspect_test.go @@ -14,14 +14,14 @@ import ( func TestInspectVars(t *testing.T) { testCases := []struct { name string - caseStep *step + caseStep *Step options *InspectOptions expected string err string }{ { name: "basic", - caseStep: &step{ + caseStep: &Step{ Action: "Shell", Env: []EnvVar{ { @@ -40,7 +40,7 @@ func TestInspectVars(t *testing.T) { }, { name: "makefile", - caseStep: &step{ + caseStep: &Step{ Action: "Shell", Env: []EnvVar{ { @@ -59,12 +59,12 @@ func TestInspectVars(t *testing.T) { }, { name: "failed action", - caseStep: &step{Action: "Unknown"}, + caseStep: &Step{Action: "Unknown"}, err: "inspecting step variables not implemented for action type Unknown", }, { name: "failed format", - caseStep: &step{Action: "Shell"}, + caseStep: &Step{Action: "Shell"}, options: &InspectOptions{Format: "unknown"}, err: "unknown output format \"unknown\"", }, @@ -86,8 +86,8 @@ func TestInspectVars(t *testing.T) { func TestInspect(t *testing.T) { p := Pipeline{ - ResourceGroups: []*resourceGroup{{ - Steps: []*step{ + ResourceGroups: []*ResourceGroup{{ + Steps: []*Step{ { Name: "step1", }, @@ -98,7 +98,7 @@ func TestInspect(t *testing.T) { opts := NewInspectOptions(config.Variables{}, "", "step1", "scope", "format") opts.ScopeFunctions = map[string]StepInspectScope{ - "scope": func(s *step, o *InspectOptions, w io.Writer) error { + "scope": func(s *Step, o *InspectOptions, w io.Writer) error { assert.Equal(t, s.Name, "step1") return nil }, @@ -110,8 +110,8 @@ func TestInspect(t *testing.T) { func TestInspectWrongScope(t *testing.T) { p := Pipeline{ - ResourceGroups: []*resourceGroup{{ - Steps: []*step{ + ResourceGroups: []*ResourceGroup{{ + Steps: []*Step{ { Name: "step1", }, diff --git a/tooling/templatize/pkg/pipeline/run.go b/tooling/templatize/pkg/pipeline/run.go index c8fb29fac..f1c38abb5 100644 --- a/tooling/templatize/pkg/pipeline/run.go +++ b/tooling/templatize/pkg/pipeline/run.go @@ -78,7 +78,7 @@ func (p *Pipeline) Run(ctx context.Context, options *PipelineRunOptions) error { return nil } -func (rg *resourceGroup) run(ctx context.Context, options *PipelineRunOptions) error { +func (rg *ResourceGroup) run(ctx context.Context, options *PipelineRunOptions) error { // prepare execution context subscriptionID, err := lookupSubscriptionID(ctx, rg.Subscription) if err != nil { @@ -132,7 +132,7 @@ func (rg *resourceGroup) run(ctx context.Context, options *PipelineRunOptions) e return nil } -func (s *step) run(ctx context.Context, kubeconfigFile string, executionTarget *ExecutionTarget, options *PipelineRunOptions) error { +func (s *Step) run(ctx context.Context, kubeconfigFile string, executionTarget *ExecutionTarget, options *PipelineRunOptions) error { fmt.Println("\n---------------------") if options.DryRun { fmt.Println("This is a dry run!") @@ -169,7 +169,7 @@ func prepareKubeConfig(ctx context.Context, executionTarget *ExecutionTarget) (s return kubeconfigFile, nil } -func (s *step) description() string { +func (s *Step) description() string { var details []string switch s.Action { case "Shell": @@ -191,7 +191,7 @@ func (p *Pipeline) Validate() error { return nil } -func (rg *resourceGroup) Validate() error { +func (rg *ResourceGroup) Validate() error { if rg.Name == "" { return fmt.Errorf("resource group name is required") } diff --git a/tooling/templatize/pkg/pipeline/shell.go b/tooling/templatize/pkg/pipeline/shell.go index 23754a119..e77cb788f 100644 --- a/tooling/templatize/pkg/pipeline/shell.go +++ b/tooling/templatize/pkg/pipeline/shell.go @@ -12,7 +12,7 @@ import ( "github.com/Azure/ARO-HCP/tooling/templatize/pkg/utils" ) -func (s *step) createCommand(ctx context.Context, dryRun bool, envVars map[string]string) (*exec.Cmd, bool) { +func (s *Step) createCommand(ctx context.Context, dryRun bool, envVars map[string]string) (*exec.Cmd, bool) { var cmd *exec.Cmd if dryRun { if s.DryRun.Command == nil && s.DryRun.EnvVars == nil { @@ -33,7 +33,7 @@ func (s *step) createCommand(ctx context.Context, dryRun bool, envVars map[strin return cmd, false } -func (s *step) runShellStep(ctx context.Context, kubeconfigFile string, options *PipelineRunOptions) error { +func (s *Step) runShellStep(ctx context.Context, kubeconfigFile string, options *PipelineRunOptions) error { if s.outputFunc == nil { s.outputFunc = func(output string) { fmt.Println(output) @@ -73,7 +73,7 @@ func (s *step) runShellStep(ctx context.Context, kubeconfigFile string, options return nil } -func (s *step) mapStepVariables(vars config.Variables) (map[string]string, error) { +func (s *Step) mapStepVariables(vars config.Variables) (map[string]string, error) { envVars := make(map[string]string) for _, e := range s.Env { value, found := vars.GetByPath(e.ConfigRef) diff --git a/tooling/templatize/pkg/pipeline/shell_test.go b/tooling/templatize/pkg/pipeline/shell_test.go index 4ac7f6a6e..078ff2255 100644 --- a/tooling/templatize/pkg/pipeline/shell_test.go +++ b/tooling/templatize/pkg/pipeline/shell_test.go @@ -13,7 +13,7 @@ func TestCreateCommand(t *testing.T) { ctx := context.Background() testCases := []struct { name string - step *step + step *Step dryRun bool envVars map[string]string expectedCommand string @@ -23,7 +23,7 @@ func TestCreateCommand(t *testing.T) { }{ { name: "basic", - step: &step{ + step: &Step{ Command: []string{"/usr/bin/echo", "hello"}, }, expectedCommand: "/usr/bin/echo", @@ -31,9 +31,9 @@ func TestCreateCommand(t *testing.T) { }, { name: "dry-run", - step: &step{ + step: &Step{ Command: []string{"/usr/bin/echo", "hello"}, - DryRun: dryRun{ + DryRun: DryRun{ Command: []string{"/usr/bin/echo", "dry-run"}, }, }, @@ -43,9 +43,9 @@ func TestCreateCommand(t *testing.T) { }, { name: "dry-run-env", - step: &step{ + step: &Step{ Command: []string{"/usr/bin/echo"}, - DryRun: dryRun{ + DryRun: DryRun{ EnvVars: []EnvVar{ { Name: "DRY_RUN", @@ -61,7 +61,7 @@ func TestCreateCommand(t *testing.T) { }, { name: "dry-run fail", - step: &step{ + step: &Step{ Command: []string{"/usr/bin/echo"}, }, dryRun: true, @@ -90,7 +90,7 @@ func TestMapStepVariables(t *testing.T) { testCases := []struct { name string vars config.Variables - step step + step Step expected map[string]string err string }{ @@ -99,7 +99,7 @@ func TestMapStepVariables(t *testing.T) { vars: config.Variables{ "FOO": "bar", }, - step: step{ + step: Step{ Env: []EnvVar{ { Name: "BAZ", @@ -114,7 +114,7 @@ func TestMapStepVariables(t *testing.T) { { name: "missing", vars: config.Variables{}, - step: step{ + step: Step{ Env: []EnvVar{ { ConfigRef: "FOO", @@ -128,7 +128,7 @@ func TestMapStepVariables(t *testing.T) { vars: config.Variables{ "FOO": 42, }, - step: step{ + step: Step{ Env: []EnvVar{ { Name: "BAZ", @@ -157,7 +157,7 @@ func TestMapStepVariables(t *testing.T) { func TestRunShellStep(t *testing.T) { expectedOutput := "hello\n" - s := &step{ + s := &Step{ Command: []string{"echo", "hello"}, outputFunc: func(output string) { assert.Equal(t, output, expectedOutput) diff --git a/tooling/templatize/pkg/pipeline/types.go b/tooling/templatize/pkg/pipeline/types.go index 4dc9fb3df..2ef6595a1 100644 --- a/tooling/templatize/pkg/pipeline/types.go +++ b/tooling/templatize/pkg/pipeline/types.go @@ -4,37 +4,37 @@ type Pipeline struct { pipelineFilePath string ServiceGroup string `yaml:"serviceGroup"` RolloutName string `yaml:"rolloutName"` - ResourceGroups []*resourceGroup `yaml:"resourceGroups"` + ResourceGroups []*ResourceGroup `yaml:"resourceGroups"` } -type resourceGroup struct { +type ResourceGroup struct { Name string `yaml:"name"` Subscription string `yaml:"subscription"` - AKSCluster string `yaml:"aksCluster"` - Steps []*step `yaml:"steps"` + AKSCluster string `yaml:"aksCluster,omitempty"` + Steps []*Step `yaml:"steps"` } type outPutHandler func(string) -type step struct { +type Step struct { Name string `yaml:"name"` Action string `yaml:"action"` - Command []string `yaml:"command"` - Env []EnvVar `yaml:"env"` - Template string `yaml:"template"` - Parameters string `yaml:"parameters"` - DependsOn []string `yaml:"dependsOn"` - DryRun dryRun `yaml:"dryRun"` + Command []string `yaml:"command,omitempty"` + Env []EnvVar `yaml:"env,omitempty"` + Template string `yaml:"template,omitempty"` + Parameters string `yaml:"parameters,omitempty"` + DependsOn []string `yaml:"dependsOn,omitempty"` + DryRun DryRun `yaml:"dryRun,omitempty"` outputFunc outPutHandler } -type dryRun struct { - EnvVars []EnvVar `yaml:"envVars"` - Command []string `yaml:"command"` +type DryRun struct { + EnvVars []EnvVar `yaml:"envVars,omitempty"` + Command []string `yaml:"command,omitempty"` } type EnvVar struct { Name string `yaml:"name"` - ConfigRef string `yaml:"configRef"` - Value string `yaml:"value"` + ConfigRef string `yaml:"configRef,omitempty"` + Value string `yaml:"value,omitempty"` } diff --git a/tooling/templatize/testdata/zz_fixture_TestPreprocessContent.bicepparam b/tooling/templatize/testdata/zz_fixture_TestPreprocessContent.bicepparam new file mode 100644 index 000000000..4d6e5f22b --- /dev/null +++ b/tooling/templatize/testdata/zz_fixture_TestPreprocessContent.bicepparam @@ -0,0 +1,10 @@ +// copy from dev-infrastructure/configurations/region.bicepparam +using '../templates/region.bicep' + +// dns +param baseDNSZoneName = 'hcp.osadev.cloud' +param baseDNSZoneResourceGroup = 'global' + +// CS +param csImage = 'cs-image' +param regionRG = 'bahamas' diff --git a/tooling/templatize/testdata/zz_fixture_TestPreprocessFilebicepparam b/tooling/templatize/testdata/zz_fixture_TestPreprocessFilebicepparam new file mode 100644 index 000000000..4d6e5f22b --- /dev/null +++ b/tooling/templatize/testdata/zz_fixture_TestPreprocessFilebicepparam @@ -0,0 +1,10 @@ +// copy from dev-infrastructure/configurations/region.bicepparam +using '../templates/region.bicep' + +// dns +param baseDNSZoneName = 'hcp.osadev.cloud' +param baseDNSZoneResourceGroup = 'global' + +// CS +param csImage = 'cs-image' +param regionRG = 'bahamas' diff --git a/tooling/templatize/testdata/zz_fixture_TestProcessPipelineForEV2ev2-precompiled-test.bicepparam b/tooling/templatize/testdata/zz_fixture_TestProcessPipelineForEV2ev2-precompiled-test.bicepparam new file mode 100644 index 000000000..feb7facf9 --- /dev/null +++ b/tooling/templatize/testdata/zz_fixture_TestProcessPipelineForEV2ev2-precompiled-test.bicepparam @@ -0,0 +1 @@ +param regionRG = '__REGIONRG__' \ No newline at end of file diff --git a/tooling/templatize/testdata/zz_fixture_TestProcessPipelineForEV2pipeline.yaml b/tooling/templatize/testdata/zz_fixture_TestProcessPipelineForEV2pipeline.yaml new file mode 100644 index 000000000..f405fa1b0 --- /dev/null +++ b/tooling/templatize/testdata/zz_fixture_TestProcessPipelineForEV2pipeline.yaml @@ -0,0 +1,28 @@ +serviceGroup: Microsoft.Azure.ARO.Test +rolloutName: Test Rollout +resourceGroups: + - name: hcp-underlay-$(regionShortName) + subscription: hcp-$location() + aksCluster: aro-hcp-aks + steps: + - name: deploy + action: Shell + command: + - make + - deploy + env: + - name: MAESTRO_IMAGE + configRef: maestro_image + - name: dry-run + action: Shell + command: + - make + - deploy + dryRun: + envVars: + - name: DRY_RUN + value: A very dry one + - name: svc + action: ARM + template: templates/svc-cluster.bicep + parameters: ev2-precompiled-test.bicepparam