diff --git a/config/config.yaml b/config/config.yaml index 9e6db4b46..5eba0d0a3 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,11 +1,14 @@ defaults: region: {{ .ctx.region }} + # Subscriptions + serviceClusterSubscription: hcp-{{ .ctx.region }} + managementClusterSubscription: hcp-{{ .ctx.region }} # Resourcegroups globalRG: global - regionRG: hcp-underlay-{{ .ctx.region }}-{{ .ctx.regionStamp }} - serviceClusterRG: hcp-underlay-{{ .ctx.region }}-{{ .ctx.regionStamp }}-svc - managementClusterRG: hcp-underlay-{{ .ctx.region }}-{{ .ctx.regionStamp }}-mgmt-{{ .ctx.cxStamp }} - imageSyncRG: hcp-underlay-{{ .ctx.region }}-{{ .ctx.regionStamp }}-imagesync + regionRG: hcp-underlay-{{ .ctx.regionShort }} + serviceClusterRG: hcp-underlay-{{ .ctx.regionShort }}-svc + managementClusterRG: hcp-underlay-{{ .ctx.regionShort }}-mgmt-{{ .ctx.stamp }} + imageSyncRG: hcp-underlay-{{ .ctx.regionShort }}-imagesync # General AKS config kubernetesVersion: 1.30.5 @@ -19,30 +22,30 @@ defaults: serviceComponentAcrResourceGroups: global # SVC cluster specifics - svcEtcdKVName: {{ azureKeyVaultName "aro-hcp-etcd" 5 .ctx.region .ctx.regionStamp }} + svcEtcdKVName: arohcp-etcd-{{ .ctx.regionShort }} svcEtcdKVSoftDelete: true # MGMT cluster specifics - mgmtEtcdKVName: {{ azureKeyVaultName "aro-hcp-etcd" 5 .ctx.region .ctx.regionStamp .ctx.cxStamp }} + mgmtEtcdKVName: arohcp-etcd-{{ .ctx.regionShort }}-{{ .ctx.stamp }} mgmtEtcdKVSoftDelete: true # Frontend frontendCosmosDBDeploy: true frontendCosmosDBDisableLocalAuth: true - frontendCosmosDBName: {{ azureCosmosDBName "aro-hcp-rp" 5 .ctx.region .ctx.regionStamp }} + frontendCosmosDBName: arohcp-rp-{{ .ctx.regionShort }} # Maestro - maestroKeyVaultName: {{ azureKeyVaultName "maestro" 5 .ctx.region .ctx.regionStamp }} - maestroEventgridName: {{ azureEventGridName "maestro" 5 .ctx.region .ctx.regionStamp }} + maestroKeyVaultName: arohcp-maestro-{{ .ctx.regionShort }} + maestroEventgridName: arohcp-maestro-{{ .ctx.regionShort }} maestroEventGridMaxClientSessionsPerAuthName: '4' maestroCertDomain: 'selfsigned.maestro.keyvault.azure.com' - maestroPostgresName: {{ azurePostgresName "maestro" 5 .ctx.region .ctx.regionStamp }} + maestroPostgresName: arohcp-maestro-{{ .ctx.regionShort }} maestroPostgresServerVersion: '15' maestroPostgresServerStorageSizeGB: '32' maestroPostgresDeploy: true maestroPostgresPrivate: false maestroRestrictIstioIngress: true - maestroConsumerName: hcp-underlay-{{ .ctx.region }}-{{ .ctx.regionStamp }}-mgmt-{{ .ctx.cxStamp }} + maestroConsumerName: hcp-underlay-{{ .ctx.regionShort }}-mgmt-{{ .ctx.stamp }} # Hypershift hypershiftNamespace: hypershift @@ -51,7 +54,7 @@ defaults: externalDNSServiceAccountName: external-dns # Cluster Service - clusterServicePostgresName: {{ azurePostgresName "cs" 5 .ctx.region .ctx.regionStamp }} + clusterServicePostgresName: arohcp-cs-{{ .ctx.regionShort }} clusterServicePostgresDeploy: true clusterServicePostgresPrivate: false clusterServiceAcrRG: global @@ -66,20 +69,20 @@ defaults: ocMirrorImageTag: 7abc8af # Service KeyVault - serviceKeyVaultName: {{ azureKeyVaultName "aro-hcp-svc" 5 .ctx.region .ctx.regionStamp }} - serviceKeyVaultRG: hcp-underlay-{{ .ctx.region }}-svc-{{ .ctx.regionStamp }} + serviceKeyVaultName: arohcp-svc-{{ .ctx.regionShort }} + serviceKeyVaultRG: hcp-underlay-{{ .ctx.regionShort }} serviceKeyVaultRegion: {{ .ctx.region }} serviceKeyVaultSoftDelete: true serviceKeyVaultPrivate: true # Management Cluster KV - cxKeyVaultName: {{ azureKeyVaultName "aro-hcp-cx" 5 .ctx.region .ctx.regionStamp .ctx.cxStamp }} + cxKeyVaultName: arohcp-cx-{{ .ctx.regionShort }}-{{ .ctx.stamp }} cxKeyVaultSoftDelete: true cxKeyVaultPrivate: true - msiKeyVaultName: {{ azureKeyVaultName "aro-hcp-msi" 5 .ctx.region .ctx.regionStamp .ctx.cxStamp }} + msiKeyVaultName: arohcp-msi-{{ .ctx.regionShort }}-{{ .ctx.stamp }} msiKeyVaultSoftDelete: true msiKeyVaultPrivate: true - mgmtKeyVaultName: {{ azureKeyVaultName "aro-hcp-mgmt" 5 .ctx.region .ctx.regionStamp .ctx.cxStamp }} + mgmtKeyVaultName: arohcp-mgmt-{{ .ctx.regionShort }}-{{ .ctx.stamp }} mgmtKeyVaultSoftDelete: true mgmtKeyVaultPrivate: true @@ -90,6 +93,9 @@ clouds: # this configuration serves as a template for for all RH DEV subscription deployments # the following vars need approprivate overrides: defaults: + # Subscription + serviceClusterSubscription: ARO Hosted Control Planes (EA Subscription 1) + managementClusterSubscription: ARO Hosted Control Planes (EA Subscription 1) # DNS baseDnsZoneName: 'hcp.osadev.cloud' # MGMTM AKS nodepools - big enough for 2 HCPs @@ -136,11 +142,11 @@ clouds: # Shared Image Sync imageSyncRG: hcp-underlay-westus3-imagesync-dev # OIDC - oidcStorageAccountName: {{ azureStorageAccountName "arohcpoidc" 5 .ctx.region .ctx.regionStamp }} + oidcStorageAccountName: arohcpoidc{{ .ctx.regionShort }} # Metrics - monitoringWorkspaceName: 'aro-hcp-monitor-{{ uniqueString 5 .ctx.region .ctx.regionStamp}}' - grafanaName: 'aro-hcp-grafana-{{ uniqueString 5 .ctx.region .ctx.regionStamp}}' - monitoringMsiName: 'aro-hcp-metrics-msi-{{ uniqueString 5 .ctx.region .ctx.regionStamp }}' + monitoringWorkspaceName: 'arohcp-{{ .ctx.regionShort }}' + grafanaName: 'arohcp-{{ .ctx.regionShort }}' + monitoringMsiName: 'aro-hcp-metrics-msi-{{ .ctx.regionShort }}' grafanaAdminGroupPrincipalId: 6b6d3adf-8476-4727-9812-20ffdef2b85c # DEVOPS MSI aroDevopsMsiId: '/subscriptions/1d3378d3-5a3f-4712-85a1-2485495dfc4b/resourceGroups/global/providers/Microsoft.ManagedIdentity/userAssignedIdentities/aro-hcp-devops' @@ -153,6 +159,9 @@ clouds: mgmtUserAgentPoolMaxCount: 12 # DNS regionalDNSSubdomain: '{{ .ctx.region }}' + regions: + westus2: + mgmtUserAgentPoolMinCount: 5 cs-pr: # this is the cluster service PR check and full cycle test environment defaults: @@ -169,7 +178,7 @@ clouds: # Cluster Service clusterServicePostgresDeploy: false # DNS - regionalDNSSubdomain: '{{ .ctx.region }}-{{ uniqueString 5 .ctx.region .ctx.regionStamp }}' + regionalDNSSubdomain: '{{ .ctx.regionShort }}' # Hypershift # uncomment the following line if you want to install the hypershift operator # with CRD support for managedIdentities diff --git a/go.work.sum b/go.work.sum index a4332a4ac..5f7261401 100644 --- a/go.work.sum +++ b/go.work.sum @@ -354,13 +354,9 @@ github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7Oputl github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= -github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= -github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= -github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= -github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= github.com/Masterminds/squirrel v1.5.4 h1:uUcX/aBc8O7Fg9kaISIUsHXdKuqehiXAMQTYX8afzqM= github.com/Masterminds/squirrel v1.5.4/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= github.com/Masterminds/vcs v1.13.3 h1:IIA2aBdXvfbIM+yl/eTnL4hb1XwdpvuQLglAix1gweE= @@ -745,6 +741,8 @@ github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14j github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -929,8 +927,6 @@ github.com/howeyc/gopass v0.0.0-20210920133722-c8aef6fb66ef/go.mod h1:lADxMC39cJ github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= -github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI= -github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -1285,8 +1281,6 @@ github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh github.com/shibumi/go-pathspec v1.3.0/go.mod h1:Xutfslp817l2I1cZvgcfeMQJG5QnU2lh5tVaaMCl3jE= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= -github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/shurcooL/httpfs v0.0.0-20230704072500-f1e31cf0ba5c/go.mod h1:owqhoLW1qZoYLZzLnBw+QkPP9WZnjlSWihhxAJC1+/M= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sigstore/protobuf-specs v0.3.0 h1:E49qS++llp4psM+3NNVEb+C4AD422bT9VkOQIPrNLpA= @@ -1314,8 +1308,6 @@ github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= -github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cobra v0.0.6/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= diff --git a/maestro/server/Makefile b/maestro/server/Makefile index 2f97063fb..05f7e2e3a 100644 --- a/maestro/server/Makefile +++ b/maestro/server/Makefile @@ -3,6 +3,10 @@ DEPLOY_ENV ?= personal-dev $(shell ../../templatize.sh $(DEPLOY_ENV) config.tmpl.mk config.mk) include config.mk +hi: + @echo "Hello, World!" + kubectl get ns + deploy: kubectl create namespace maestro --dry-run=client -o json | kubectl apply -f - ISTO_VERSION=$(shell az aks show -n ${AKS_NAME} -g ${SVC_RG} --query serviceMeshProfile.istio.revisions[-1] -o tsv) && \ diff --git a/maestro/server/pipeline.yaml b/maestro/server/pipeline.yaml new file mode 100644 index 000000000..a4e478efa --- /dev/null +++ b/maestro/server/pipeline.yaml @@ -0,0 +1,30 @@ +serviceGroup: Microsoft.Azure.ARO.Test +rolloutName: Maestro Server Rollout +steps: +- name: deploy + subscription: {{ .serviceClusterSubscription }} + resourceGroup: {{ .serviceClusterRG }} + aksCluster: {{ .aksName }} + action: + type: Shell + command: ["/bin/bash", "-c", "make deploy"] + # we could infer the pwd from the location of this file and could avoid make -C + env: + - name: EVENTGRID_NAME + configRef: maestroEventgridName + - name: REGION_RG + configRef: regionRG + - name: SVC_RG + configRef: serviceClusterRG + - name: AKS_NAME + configRef: aksName + - name: IMAGE_BASE + configRef: maestroImageBase + - name: IMAGE_TAG + configRef: maestroImageTag + - name: USE_AZURE_DB + configRef: maestroPostgresDeploy + - name: ISTIO_RESTRICT_INGRESS + configRef: maestroRestrictIstioIngress + - name: KEYVAULT_NAME + configRef: maestroKeyVaultName diff --git a/templatize.sh b/templatize.sh index 31a023a92..f8e4f715a 100755 --- a/templatize.sh +++ b/templatize.sh @@ -61,10 +61,39 @@ while getopts "c:r:x:e:" opt; do esac done +# short names from EV2 prod ServiceConfig +case ${REGION} in + eastus) + REGION_SHORT="bl" + ;; + westus) + REGION_SHORT="by" + ;; + centralus) + REGION_SHORT="dm" + ;; + northcentralus) + REGION_SHORT="ch" + ;; + southcentralus) + REGION_SHORT="sn" + ;; + westus2) + REGION_SHORT="mwh" + ;; + westus3) + REGION_SHORT="usw3" + ;; + *) + echo "unsupported region: ${REGION}" + exit 1 +esac + if [ "$DEPLOY_ENV" == "personal-dev" ]; then - REGION_STAMP=${USER} + REGION_STAMP="${REGION_SHORT}${USER:0:4}" else - REGION_STAMP=${DEPLOY_ENV} + CLEAN_DEPLOY_ENV=$(echo "${DEPLOY_ENV}" | tr -cd '[:alnum:]') + REGION_STAMP="${CLEAN_DEPLOY_ENV}" fi TEMPLATIZE=${PROJECT_ROOT_DIR}/tooling/templatize/templatize @@ -79,8 +108,8 @@ if [ -n "$INPUT" ] && [ -n "$OUTPUT" ]; then --cloud=${CLOUD} \ --deploy-env=${DEPLOY_ENV} \ --region=${REGION} \ - --region-stamp=${REGION_STAMP} \ - --cx-stamp=${CXSTAMP} \ + --region-short=${REGION_STAMP} \ + --stamp=${CXSTAMP} \ --input=${INPUT} \ --output=${OUTPUT} \ ${EXTRA_ARGS} @@ -90,7 +119,7 @@ else --cloud=${CLOUD} \ --deploy-env=${DEPLOY_ENV} \ --region=${REGION} \ - --region-stamp=${REGION_STAMP} \ - --cx-stamp=${CXSTAMP} \ + --region-short=${REGION_STAMP} \ + --stamp=${CXSTAMP} \ ${EXTRA_ARGS} fi diff --git a/tooling/templatize/cmd/generate/generate.go b/tooling/templatize/cmd/generate/cmd.go similarity index 100% rename from tooling/templatize/cmd/generate/generate.go rename to tooling/templatize/cmd/generate/cmd.go diff --git a/tooling/templatize/cmd/generate/generate_test.go b/tooling/templatize/cmd/generate/cmd_test.go similarity index 78% rename from tooling/templatize/cmd/generate/generate_test.go rename to tooling/templatize/cmd/generate/cmd_test.go index 2f39319d4..231508555 100644 --- a/tooling/templatize/cmd/generate/generate_test.go +++ b/tooling/templatize/cmd/generate/cmd_test.go @@ -9,21 +9,22 @@ import ( "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" - "github.com/Azure/ARO-HCP/tooling/templatize/internal/config" + options "github.com/Azure/ARO-HCP/tooling/templatize/cmd" + "github.com/Azure/ARO-HCP/tooling/templatize/pkg/config" ) func TestExecuteTemplate(t *testing.T) { for _, testCase := range []struct { - name string - config config.Variables - input string + name string + vars config.Variables + input string expected string expectedError bool }{ { name: "happy case generates a file", - config: config.Variables{ + vars: config.Variables{ "region_maestro_keyvault": "kv", "region_eventgrid_namespace": "ns", }, @@ -36,7 +37,7 @@ param maestroEventGridMaxClientSessionsPerAuthName = 4`, }, { name: "referencing unset variable errors", - config: config.Variables{ + vars: config.Variables{ "region_maestro_keyvault": "kv", }, input: `param maestroKeyVaultName = '{{ .region_maestro_keyvault }}' @@ -49,10 +50,10 @@ param maestroEventGridMaxClientSessionsPerAuthName = 4`, output := &bytes.Buffer{} opts := GenerationOptions{ completedGenerationOptions: &completedGenerationOptions{ - Config: testCase.config, - Input: fstest.MapFS{"test": &fstest.MapFile{Data: []byte(testCase.input)}}, - InputFile: "test", - Output: &nopCloser{Writer: output}, + InputFS: fstest.MapFS{"test": &fstest.MapFile{Data: []byte(testCase.input)}}, + InputFile: "test", + OutputFile: &nopCloser{Writer: output}, + RolloutOptions: options.NewRolloutOptions(testCase.vars), }, } err := opts.ExecuteTemplate() diff --git a/tooling/templatize/cmd/generate/options.go b/tooling/templatize/cmd/generate/options.go index 20b229319..59b79ba74 100644 --- a/tooling/templatize/cmd/generate/options.go +++ b/tooling/templatize/cmd/generate/options.go @@ -4,7 +4,6 @@ import ( "fmt" "io" "io/fs" - "log" "os" "path/filepath" "text/template" @@ -13,23 +12,26 @@ import ( "github.com/spf13/cobra" options "github.com/Azure/ARO-HCP/tooling/templatize/cmd" - "github.com/Azure/ARO-HCP/tooling/templatize/internal/config" + "github.com/Azure/ARO-HCP/tooling/templatize/pkg/ev2" ) func DefaultGenerationOptions() *RawGenerationOptions { - return &RawGenerationOptions{} + return &RawGenerationOptions{ + RolloutOptions: options.DefaultRolloutOptions(), + } } func BindGenerationOptions(opts *RawGenerationOptions, cmd *cobra.Command) error { - err := options.BindOptions(&opts.RawOptions, cmd) + err := options.BindRolloutOptions(opts.RolloutOptions, cmd) if err != nil { return fmt.Errorf("failed to bind raw options: %w", err) } cmd.Flags().StringVar(&opts.Input, "input", opts.Input, "input file path") cmd.Flags().StringVar(&opts.Output, "output", opts.Output, "output file path") + cmd.Flags().BoolVar(&opts.EV2Placeholders, "ev2-placeholders", opts.EV2Placeholders, "generate EV2 placeholders") for _, flag := range []string{"config-file", "input", "output"} { - if err := cmd.MarkFlagFilename("config-file"); err != nil { + if err := cmd.MarkFlagFilename(flag); err != nil { return fmt.Errorf("failed to mark flag %q as a file: %w", flag, err) } } @@ -38,26 +40,16 @@ func BindGenerationOptions(opts *RawGenerationOptions, cmd *cobra.Command) error // RawGenerationOptions holds input values. type RawGenerationOptions struct { - options.RawOptions - Input string - Output string -} - -func (o *RawGenerationOptions) Validate() (*ValidatedGenerationOptions, error) { - if _, err := o.RawOptions.Validate(); err != nil { - return nil, fmt.Errorf("validation failed for raw options: %w", err) - } - - return &ValidatedGenerationOptions{ - validatedGenerationOptions: &validatedGenerationOptions{ - RawGenerationOptions: o, - }, - }, nil + RolloutOptions *options.RawRolloutOptions + Input string + Output string + EV2Placeholders bool } // validatedGenerationOptions is a private wrapper that enforces a call of Validate() before Complete() can be invoked. type validatedGenerationOptions struct { *RawGenerationOptions + *options.ValidatedRolloutOptions } type ValidatedGenerationOptions struct { @@ -65,11 +57,46 @@ type ValidatedGenerationOptions struct { *validatedGenerationOptions } +// completedGenerationOptions is a private wrapper that enforces a call of Complete() before config generation can be invoked. +type completedGenerationOptions struct { + *options.RolloutOptions + InputFS fs.FS + InputFile string + OutputFile io.Writer +} + +type GenerationOptions struct { + // Embed a private pointer that cannot be instantiated outside of this package. + *completedGenerationOptions +} + +func (o *RawGenerationOptions) Validate() (*ValidatedGenerationOptions, error) { + validatedRolloutOptions, err := o.RolloutOptions.Validate() + if err != nil { + return nil, fmt.Errorf("validation failed for raw options: %w", err) + } + + if _, err := os.Stat(o.Input); os.IsNotExist(err) { + return nil, fmt.Errorf("input file %s does not exist", o.Input) + } + + return &ValidatedGenerationOptions{ + validatedGenerationOptions: &validatedGenerationOptions{ + RawGenerationOptions: o, + ValidatedRolloutOptions: validatedRolloutOptions, + }, + }, nil +} + func (o *ValidatedGenerationOptions) Complete() (*GenerationOptions, error) { - cfg := config.NewConfigProvider(o.ConfigFile, o.Region, o.RegionStamp, o.CXStamp) - vars, err := cfg.GetVariables(o.Cloud, o.DeployEnv, o.ExtraVars) + completed, err := o.ValidatedRolloutOptions.Complete() if err != nil { - return nil, fmt.Errorf("failed to get variables for cloud %s: %w", o.Cloud, err) + return nil, err + } + + if o.EV2Placeholders { + _, vars := ev2.EV2Mapping(completed.Config, []string{}) + completed.Config = vars } inputFile := filepath.Base(o.Input) @@ -78,37 +105,24 @@ func (o *ValidatedGenerationOptions) Complete() (*GenerationOptions, error) { return nil, fmt.Errorf("failed to create output directory %s: %w", o.Output, err) } - output, err := os.Create(o.Output) + outputFile, err := os.Create(o.Output) if err != nil { return nil, fmt.Errorf("failed to create output file %s: %w", o.Input, err) } return &GenerationOptions{ completedGenerationOptions: &completedGenerationOptions{ - Config: vars, - Input: os.DirFS(filepath.Dir(o.Input)), - InputFile: inputFile, - Output: output, + RolloutOptions: completed, + InputFS: os.DirFS(filepath.Dir(o.Input)), + InputFile: inputFile, + OutputFile: outputFile, }, }, nil } -// completedGenerationOptions is a private wrapper that enforces a call of Complete() before config generation can be invoked. -type completedGenerationOptions struct { - Config config.Variables - Input fs.FS - InputFile string - Output io.WriteCloser -} - -type GenerationOptions struct { - // Embed a private pointer that cannot be instantiated outside of this package. - *completedGenerationOptions -} - func (opts *GenerationOptions) ExecuteTemplate() error { tmpl := template.New(opts.InputFile).Funcs(sprig.FuncMap()) - content, err := fs.ReadFile(opts.Input, opts.InputFile) + content, err := fs.ReadFile(opts.InputFS, opts.InputFile) if err != nil { return err } @@ -118,10 +132,5 @@ func (opts *GenerationOptions) ExecuteTemplate() error { return err } - defer func() { - if err := opts.Output.Close(); err != nil { - log.Printf("error closing output: %v\n", err) - } - }() - return tmpl.Option("missingkey=error").ExecuteTemplate(opts.Output, opts.InputFile, opts.Config) + return tmpl.Option("missingkey=error").ExecuteTemplate(opts.OutputFile, opts.InputFile, opts.RolloutOptions.Config) } diff --git a/tooling/templatize/cmd/generate/options_test.go b/tooling/templatize/cmd/generate/options_test.go index 3fd96d475..229506ab9 100644 --- a/tooling/templatize/cmd/generate/options_test.go +++ b/tooling/templatize/cmd/generate/options_test.go @@ -14,13 +14,15 @@ import ( func TestRawOptions(t *testing.T) { tmpdir := t.TempDir() opts := &RawGenerationOptions{ - RawOptions: options.RawOptions{ - ConfigFile: "../../testdata/config.yaml", - Cloud: "public", - DeployEnv: "dev", + RolloutOptions: &options.RawRolloutOptions{ Region: "uksouth", - RegionStamp: "1", - CXStamp: "cx", + RegionShort: "abcde", + Stamp: "fghij", + BaseOptions: &options.RawOptions{ + ConfigFile: "../../testdata/config.yaml", + Cloud: "public", + DeployEnv: "dev", + }, }, Input: "../../testdata/helm.sh", Output: fmt.Sprintf("%s/helm.sh", tmpdir), diff --git a/tooling/templatize/cmd/inspect/inspect.go b/tooling/templatize/cmd/inspect/cmd.go similarity index 85% rename from tooling/templatize/cmd/inspect/inspect.go rename to tooling/templatize/cmd/inspect/cmd.go index 8edb69449..0490a7f57 100644 --- a/tooling/templatize/cmd/inspect/inspect.go +++ b/tooling/templatize/cmd/inspect/cmd.go @@ -11,7 +11,8 @@ import ( ) func NewCommand() *cobra.Command { - opts := options.DefaultOptions() + opts := options.DefaultRolloutOptions() + format := "json" cmd := &cobra.Command{ Use: "inspect", @@ -21,14 +22,14 @@ func NewCommand() *cobra.Command { return dumpConfig(format, opts) }, } - if err := options.BindOptions(opts, cmd); err != nil { + if err := options.BindRolloutOptions(opts, cmd); err != nil { log.Fatal(err) } cmd.Flags().StringVar(&format, "format", format, "output format (json, yaml)") return cmd } -func dumpConfig(format string, opts *options.RawOptions) error { +func dumpConfig(format string, opts *options.RawRolloutOptions) error { validated, err := opts.Validate() if err != nil { return err diff --git a/tooling/templatize/cmd/inspect/options.go b/tooling/templatize/cmd/inspect/options.go deleted file mode 100644 index eda521fdc..000000000 --- a/tooling/templatize/cmd/inspect/options.go +++ /dev/null @@ -1 +0,0 @@ -package inspect diff --git a/tooling/templatize/cmd/options.go b/tooling/templatize/cmd/options.go index 2d4e66793..28fa6116d 100644 --- a/tooling/templatize/cmd/options.go +++ b/tooling/templatize/cmd/options.go @@ -6,7 +6,7 @@ import ( "github.com/spf13/cobra" "k8s.io/apimachinery/pkg/util/sets" - "github.com/Azure/ARO-HCP/tooling/templatize/internal/config" + "github.com/Azure/ARO-HCP/tooling/templatize/pkg/config" ) func DefaultOptions() *RawOptions { @@ -17,22 +17,14 @@ func BindOptions(opts *RawOptions, cmd *cobra.Command) error { cmd.Flags().StringVar(&opts.ConfigFile, "config-file", opts.ConfigFile, "config file path") cmd.Flags().StringVar(&opts.Cloud, "cloud", opts.Cloud, "the cloud (public, fairfax)") cmd.Flags().StringVar(&opts.DeployEnv, "deploy-env", opts.DeployEnv, "the deploy environment") - cmd.Flags().StringVar(&opts.Region, "region", opts.Region, "resources location") - cmd.Flags().StringVar(&opts.RegionStamp, "region-stamp", opts.RegionStamp, "region stamp") - cmd.Flags().StringVar(&opts.CXStamp, "cx-stamp", opts.CXStamp, "CX stamp") - cmd.Flags().StringToStringVar(&opts.ExtraVars, "extra-args", opts.ExtraVars, "Extra arguments to be used config templating") return nil } // RawGenerationOptions holds input values. type RawOptions struct { - ConfigFile string - Cloud string - DeployEnv string - Region string - RegionStamp string - CXStamp string - ExtraVars map[string]string + ConfigFile string + Cloud string + DeployEnv string } func (o *RawOptions) Validate() (*ValidatedOptions, error) { @@ -59,22 +51,22 @@ type ValidatedOptions struct { } func (o *ValidatedOptions) Complete() (*Options, error) { - cfg := config.NewConfigProvider(o.ConfigFile, o.Region, o.RegionStamp, o.CXStamp) - vars, err := cfg.GetVariables(o.Cloud, o.DeployEnv, o.ExtraVars) + configProvider := config.NewConfigProvider(o.ConfigFile) + err := configProvider.Validate(o.Cloud, o.DeployEnv) if err != nil { - return nil, fmt.Errorf("failed to get variables for cloud %s: %w", o.Cloud, err) + return nil, fmt.Errorf("failed to validate config: %w", err) } return &Options{ completedOptions: &completedOptions{ - Config: vars, + ConfigProvider: configProvider, }, }, nil } // completedGenerationOptions is a private wrapper that enforces a call of Complete() before config generation can be invoked. type completedOptions struct { - Config config.Variables + ConfigProvider config.ConfigProvider } type Options struct { diff --git a/tooling/templatize/cmd/rolloutoptions.go b/tooling/templatize/cmd/rolloutoptions.go new file mode 100644 index 000000000..a8757e63a --- /dev/null +++ b/tooling/templatize/cmd/rolloutoptions.go @@ -0,0 +1,116 @@ +package options + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/Azure/ARO-HCP/tooling/templatize/pkg/config" +) + +func DefaultRolloutOptions() *RawRolloutOptions { + return &RawRolloutOptions{ + BaseOptions: DefaultOptions(), + } +} + +func NewRolloutOptions(config config.Variables) *RolloutOptions { + return &RolloutOptions{ + completedRolloutOptions: &completedRolloutOptions{ + Config: config, + }, + } +} + +func EV2RolloutOptions() *RawRolloutOptions { + return &RawRolloutOptions{ + Region: "$location()", + RegionShort: "$(regionShort)", + Stamp: "$stamp()", + } +} + +func BindRolloutOptions(opts *RawRolloutOptions, cmd *cobra.Command) error { + err := BindOptions(opts.BaseOptions, cmd) + if err != nil { + return fmt.Errorf("failed to bind options: %w", err) + } + cmd.Flags().StringVar(&opts.Region, "region", opts.Region, "resources location") + cmd.Flags().StringVar(&opts.RegionShort, "region-short", opts.RegionShort, "short region string") + cmd.Flags().StringVar(&opts.Stamp, "stamp", opts.Stamp, "stamp") + cmd.Flags().StringToStringVar(&opts.ExtraVars, "extra-args", opts.ExtraVars, "Extra arguments to be used config templating") + return nil +} + +// RawRolloutOptions holds input values. +type RawRolloutOptions struct { + Region string + RegionShort string + Stamp string + ExtraVars map[string]string + BaseOptions *RawOptions +} + +// validatedRolloutOptions is a private wrapper that enforces a call of Validate() before Complete() can be invoked. +type validatedRolloutOptions struct { + *RawRolloutOptions + *ValidatedOptions +} + +type ValidatedRolloutOptions struct { + // Embed a private pointer that cannot be instantiated outside of this package. + *validatedRolloutOptions +} + +type completedRolloutOptions struct { + *ValidatedRolloutOptions + Options *Options + Config config.Variables +} + +type RolloutOptions struct { + // Embed a private pointer that cannot be instantiated outside of this package. + *completedRolloutOptions +} + +func (o *RawRolloutOptions) Validate() (*ValidatedRolloutOptions, error) { + validatedBaseOptions, err := o.BaseOptions.Validate() + if err != nil { + return nil, err + } + + return &ValidatedRolloutOptions{ + validatedRolloutOptions: &validatedRolloutOptions{ + RawRolloutOptions: o, + ValidatedOptions: validatedBaseOptions, + }, + }, nil +} + +func (o *ValidatedRolloutOptions) Complete() (*RolloutOptions, error) { + completed, err := o.ValidatedOptions.Complete() + if err != nil { + return nil, err + } + + variables, err := completed.ConfigProvider.GetVariables(o.Cloud, o.DeployEnv, o.Region, config.NewConfigReplacements(o.Region, o.RegionShort, o.Stamp)) + if err != nil { + return nil, fmt.Errorf("failed to get variables: %w", err) + } + extraVars := make(map[string]interface{}) + for k, v := range o.ExtraVars { + extraVars[k] = v + } + err = variables.AddNested("extraVars", extraVars) + if err != nil { + return nil, fmt.Errorf("failed to add extraVars: %w", err) + } + + return &RolloutOptions{ + completedRolloutOptions: &completedRolloutOptions{ + ValidatedRolloutOptions: o, + Options: completed, + Config: variables, + }, + }, nil +} diff --git a/tooling/templatize/cmd/run/cmd.go b/tooling/templatize/cmd/run/cmd.go new file mode 100644 index 000000000..b7e21459f --- /dev/null +++ b/tooling/templatize/cmd/run/cmd.go @@ -0,0 +1,42 @@ +package run + +import ( + "context" + "log" + "os" + "os/signal" + "syscall" + + "github.com/spf13/cobra" +) + +func NewCommand() *cobra.Command { + opts := DefaultOptions() + cmd := &cobra.Command{ + Use: "run-pipeline", + Short: "run a pipeline.yaml file towards an Azure Resourcegroup / AKS cluster", + Long: "run a pipeline.yaml file towards an Azure Resourcegroup / AKS cluster", + RunE: func(cmd *cobra.Command, args []string) error { + ctx, stop := signal.NotifyContext(cmd.Context(), os.Interrupt, syscall.SIGTERM) + defer stop() + + return runPipeline(ctx, opts) + }, + } + if err := BindOptions(opts, cmd); err != nil { + log.Fatal(err) + } + return cmd +} + +func runPipeline(ctx context.Context, opts *RawRunOptions) error { + validated, err := opts.Validate() + if err != nil { + return err + } + completed, err := validated.Complete() + if err != nil { + return err + } + return completed.RunPipeline(ctx) +} diff --git a/tooling/templatize/cmd/run/options.go b/tooling/templatize/cmd/run/options.go new file mode 100644 index 000000000..d33d16cdd --- /dev/null +++ b/tooling/templatize/cmd/run/options.go @@ -0,0 +1,119 @@ +package run + +import ( + "context" + "fmt" + "os" + + "github.com/spf13/cobra" + + options "github.com/Azure/ARO-HCP/tooling/templatize/cmd" + "github.com/Azure/ARO-HCP/tooling/templatize/pkg/config" + "github.com/Azure/ARO-HCP/tooling/templatize/pkg/pipeline" +) + +func DefaultOptions() *RawRunOptions { + return &RawRunOptions{ + RolloutOptions: options.DefaultRolloutOptions(), + } +} + +func BindOptions(opts *RawRunOptions, cmd *cobra.Command) error { + err := options.BindRolloutOptions(opts.RolloutOptions, cmd) + if err != nil { + return fmt.Errorf("failed to bind options: %w", err) + } + cmd.Flags().StringVar(&opts.PipelineFile, "pipeline-file", opts.PipelineFile, "pipeline file path") + + for _, flag := range []string{"pipeline-file"} { + if err := cmd.MarkFlagFilename(flag); err != nil { + return fmt.Errorf("failed to mark flag %q as a file: %w", flag, err) + } + if err := cmd.MarkFlagRequired(flag); err != nil { + return fmt.Errorf("failed to mark flag %q as required: %w", flag, err) + } + } + return nil +} + +// RawRunOptions holds input values. +type RawRunOptions struct { + RolloutOptions *options.RawRolloutOptions + PipelineFile string +} + +// validatedRunOptions is a private wrapper that enforces a call of Validate() before Complete() can be invoked. +type validatedRunOptions struct { + *RawRunOptions + *options.ValidatedRolloutOptions +} + +type ValidatedRunOptions struct { + // Embed a private pointer that cannot be instantiated outside of this package. + *validatedRunOptions +} + +// completedRunOptions is a private wrapper that enforces a call of Complete() before config generation can be invoked. +type completedRunOptions struct { + RolloutOptions *options.RolloutOptions + Pipeline *pipeline.Pipeline +} + +type RunOptions struct { + // Embed a private pointer that cannot be instantiated outside of this package. + *completedRunOptions +} + +func (o *RawRunOptions) Validate() (*ValidatedRunOptions, error) { + validatedRolloutOptions, err := o.RolloutOptions.Validate() + if err != nil { + return nil, err + } + + if _, err := os.Stat(o.PipelineFile); os.IsNotExist(err) { + return nil, fmt.Errorf("pipeline file %s does not exist", o.PipelineFile) + } + + return &ValidatedRunOptions{ + validatedRunOptions: &validatedRunOptions{ + RawRunOptions: o, + ValidatedRolloutOptions: validatedRolloutOptions, + }, + }, nil +} + +func (o *ValidatedRunOptions) Complete() (*RunOptions, error) { + completed, err := o.ValidatedRolloutOptions.Complete() + if err != nil { + return nil, err + } + + pipeline, err := pipeline.NewPipelineFromFile(o.PipelineFile, completed.Config) + if err != nil { + return nil, fmt.Errorf("failed to load pipeline file %s: %w", o.PipelineFile, err) + } + + return &RunOptions{ + completedRunOptions: &completedRunOptions{ + RolloutOptions: completed, + Pipeline: pipeline, + }, + }, nil +} + +func (o *RunOptions) RunPipeline(ctx context.Context) error { + variables, err := o.RolloutOptions.Options.ConfigProvider.GetVariables( + o.RolloutOptions.Cloud, + o.RolloutOptions.DeployEnv, + o.RolloutOptions.Region, + config.NewConfigReplacements( + o.RolloutOptions.Region, + o.RolloutOptions.RegionShort, + o.RolloutOptions.Stamp, + ), + ) + if err != nil { + return err + } + return o.Pipeline.Run(ctx, variables) +} diff --git a/tooling/templatize/go.mod b/tooling/templatize/go.mod index 73bc01893..1aec6b507 100644 --- a/tooling/templatize/go.mod +++ b/tooling/templatize/go.mod @@ -3,29 +3,43 @@ module github.com/Azure/ARO-HCP/tooling/templatize go 1.23.0 require ( + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 github.com/Masterminds/sprig/v3 v3.3.0 github.com/google/go-cmp v0.6.0 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 + gotest.tools v2.2.0+incompatible k8s.io/apimachinery v0.31.1 sigs.k8s.io/yaml v1.4.0 ) require ( dario.cat/mergo v1.0.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/huandu/xstrings v1.5.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/spf13/cast v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - golang.org/x/crypto v0.26.0 // indirect + golang.org/x/crypto v0.27.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect ) diff --git a/tooling/templatize/go.sum b/tooling/templatize/go.sum index 412df81cc..d656fc95b 100644 --- a/tooling/templatize/go.sum +++ b/tooling/templatize/go.sum @@ -1,16 +1,42 @@ dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 h1:JZg6HRh6W6U4OLl6lk7BZ7BLisIzM9dG1R50zUk9C/M= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0/go.mod h1:YL1xnZ6QejvQHWJrX/AvhFl4WW4rqHVoKspWNVwFk0M= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 h1:B/dfvscEQtew9dVuoxqxrUKKv8Ih2f55PydknDamU+g= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0/go.mod h1:fiPSssYvltE08HJchL04dOy+RD4hgrjph0cwGGMntdI= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvULkDNfdXOgrjtg6UYJPFBJyuEcRCAw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0 h1:figxyQZXzZQIcP3njhC68bYUiTw45J8/SsHaLW8Ax0M= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice v1.0.0/go.mod h1:TmlMW4W5OvXOmOyKNnor8nlMMiO1ctIyzmHme/VHsrA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1 h1:7CBQ+Ei8SP2c6ydQTGCCrS35bDxgTMfoP2miAwK++OU= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.1.1/go.mod h1:c/wcGeGx5FUPbM/JltUYHZcKmigwyVLJlDq+4HdtXaw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 h1:wxQx2Bt4xzPIKvW59WQf1tJNx/ZZKPfN+EhPX3Z6CYY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0/go.mod h1:TpiwjwnW/khS0LKs4vW5UmmT9OWcxaveS8U7+tlknzo= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe3tPhs= github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -20,16 +46,26 @@ github.com/huandu/xstrings v1.5.0 h1:2ag3IFq9ZDANvthTwTiqSSZLjDc+BedvHPAp5tJy2TI github.com/huandu/xstrings v1.5.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs= +github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4= +github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -43,14 +79,24 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= k8s.io/apimachinery v0.31.1 h1:mhcUBbj7KUjaVhyXILglcVjuS4nYXiwC+KKFBgIVy7U= k8s.io/apimachinery v0.31.1/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/tooling/templatize/internal/config/config.go b/tooling/templatize/internal/config/config.go deleted file mode 100644 index 0b208a569..000000000 --- a/tooling/templatize/internal/config/config.go +++ /dev/null @@ -1,137 +0,0 @@ -package config - -import ( - "bytes" - "fmt" - "os" - "reflect" - "text/template" - - "gopkg.in/yaml.v3" - - "github.com/Azure/ARO-HCP/tooling/templatize/internal/naming" -) - -type Provider interface { - GetVariables(cloud, deployEnv string) (Variables, error) -} - -func NewConfigProvider(config, region, regionStamp, cxStamp string) *configProviderImpl { - return &configProviderImpl{ - config: config, - region: region, - regionStamp: regionStamp, - cxStamp: cxStamp, - } -} - -func interfaceToVariables(i interface{}) (Variables, bool) { - // Helper, that reduces need for reflection calls, i.e. MapIndex - // from: https://github.com/peterbourgon/mergemap/blob/master/mergemap.go - value := reflect.ValueOf(i) - if value.Kind() == reflect.Map { - m := Variables{} - for _, k := range value.MapKeys() { - m[k.String()] = value.MapIndex(k).Interface() - } - return m, true - } - return Variables{}, false -} - -// Merges variables, returns merged variables -// However the return value is only used for recursive updates on the map -// The actual merged variables are updated in the base map -func mergeVariables(base, override Variables) Variables { - for k, newValue := range override { - if baseValue, exists := base[k]; exists { - srcMap, srcMapOk := interfaceToVariables(newValue) - dstMap, dstMapOk := interfaceToVariables(baseValue) - if srcMapOk && dstMapOk { - newValue = mergeVariables(dstMap, srcMap) - } - } - base[k] = newValue - } - - return base -} - -// get the variables toke effect finally for cloud/deployEnv/region -func (cp *configProviderImpl) GetVariables(cloud, deployEnv string, extraVars map[string]string) (Variables, error) { - variableOverrides, err := cp.loadConfig(cloud, deployEnv) - variables := Variables{} - - if err == nil { - for k, v := range variableOverrides.Defaults { - variables[k] = v - } - if cloudOverride, ok := variableOverrides.Overrides[cloud]; ok { - mergeVariables(variables, cloudOverride.Defaults) - if deployEnvOverride, ok := cloudOverride.Overrides[deployEnv]; ok { - mergeVariables(variables, deployEnvOverride.Defaults) - if regionOverride, ok := deployEnvOverride.Overrides[cp.region]; ok { - mergeVariables(variables, regionOverride) - } - } else { - return nil, fmt.Errorf("the deployment env %s is not found under cloud %s in %s", deployEnv, cloud, cp.config) - } - } - } - - if _, exists := variables["extraVars"]; exists { - return nil, fmt.Errorf("extraVars is a reserved key and cannot be used in the config file") - } - - if len(extraVars) > 0 { - variables["extraVars"] = extraVars - } - return variables, err -} - -func (cp *configProviderImpl) loadConfig(cloud, deployEnv string) (*VariableOverrides, error) { - vars := map[string]interface{}{ - "ctx": map[string]interface{}{ - "region": cp.region, - "cloud": cloud, - "deployEnv": deployEnv, - "regionStamp": cp.regionStamp, - "cxStamp": cp.cxStamp, - }, - } - - functions := template.FuncMap{ - "azureEventGridName": naming.AzureEventGridName, - "azurePostgresName": naming.AzurePostgresName, - "azureKeyVaultName": naming.AzureKeyVaultName, - "azureStorageAccountName": naming.AzureStorageAccountName, - "azureCosmosDBName": naming.AzureCosmosDBName, - "uniqueString": naming.UniqueString, - } - - // parse, execute and unmarshal the config file as a template to generate the final config file - tmpl := template.New("configTemplate").Funcs(functions) - content, err := os.ReadFile(cp.config) - if err != nil { - return nil, err - } - - tmpl, err = tmpl.Parse(string(content)) - if err != nil { - return nil, err - } - - var tmplBytes bytes.Buffer - if err := tmpl.Option("missingkey=error").Execute(&tmplBytes, vars); err != nil { - return nil, err - } - - currentVariableOverrides := &VariableOverrides{} - if err := yaml.Unmarshal(tmplBytes.Bytes(), currentVariableOverrides); err == nil { - cp.baseVariableOverrides = currentVariableOverrides - } else { - return nil, err - } - - return cp.baseVariableOverrides, err -} diff --git a/tooling/templatize/internal/config/types.go b/tooling/templatize/internal/config/types.go deleted file mode 100644 index aafc3e593..000000000 --- a/tooling/templatize/internal/config/types.go +++ /dev/null @@ -1,29 +0,0 @@ -package config - -type configProviderImpl struct { - baseVariableOverrides *VariableOverrides - config string - region string - regionStamp string - cxStamp string -} - -type Variables map[string]interface{} - -type VariableOverrides struct { - Defaults Variables `yaml:"defaults"` - // key is the cloud alias - Overrides map[string]*CloudVariableOverride `yaml:"clouds"` -} - -type CloudVariableOverride struct { - Defaults Variables `yaml:"defaults"` - // key is the deploy env - Overrides map[string]*DeployEnvVariableOverride `yaml:"environments"` -} - -type DeployEnvVariableOverride struct { - Defaults Variables `yaml:"defaults"` - // key is the region name - Overrides map[string]Variables `yaml:"regions"` -} diff --git a/tooling/templatize/internal/naming/azure.go b/tooling/templatize/internal/naming/azure.go deleted file mode 100644 index 3c99eb933..000000000 --- a/tooling/templatize/internal/naming/azure.go +++ /dev/null @@ -1,21 +0,0 @@ -package naming - -func AzureEventGridName(prefix string, suffixLength int, suffixDigestArgs ...string) (string, error) { - return suffixedName(prefix, "-", 24, suffixLength, suffixDigestArgs...) -} - -func AzurePostgresName(prefix string, suffixLength int, suffixDigestArgs ...string) (string, error) { - return suffixedName(prefix, "-", 60, suffixLength, suffixDigestArgs...) -} - -func AzureKeyVaultName(prefix string, suffixLength int, suffixDigestArgs ...string) (string, error) { - return suffixedName(prefix, "-", 24, suffixLength, suffixDigestArgs...) -} - -func AzureStorageAccountName(prefix string, suffixLength int, suffixDigestArgs ...string) (string, error) { - return suffixedName(prefix, "", 24, suffixLength, suffixDigestArgs...) -} - -func AzureCosmosDBName(prefix string, suffixLength int, suffixDigestArgs ...string) (string, error) { - return suffixedName(prefix, "-", 44, suffixLength, suffixDigestArgs...) -} diff --git a/tooling/templatize/internal/naming/common.go b/tooling/templatize/internal/naming/common.go deleted file mode 100644 index 7822ebb87..000000000 --- a/tooling/templatize/internal/naming/common.go +++ /dev/null @@ -1,39 +0,0 @@ -package naming - -import ( - "crypto/sha256" - "encoding/hex" - "fmt" -) - -func suffixDigest(length int, strs ...string) (string, error) { - combined := "" - for _, s := range strs { - combined += s - } - hash := sha256.Sum256([]byte(combined)) - hashedString := hex.EncodeToString(hash[:]) - if len(hashedString) < length { - return "", fmt.Errorf("suffix digest does not have the required length of %d", length) - } - return hashedString[:length], nil -} - -func suffixedName(prefix string, suffixDelim string, maxLength int, suffixLength int, suffixDigestArgs ...string) (string, error) { - name := prefix - if len(suffixDigestArgs) > 0 { - suffixDigest, err := suffixDigest(suffixLength, suffixDigestArgs...) - if err != nil { - return "", err - } - name = prefix + suffixDelim + suffixDigest - } - if len(name) > maxLength { - return "", fmt.Errorf("name '%s' is too long, max length is %d", name, maxLength) - } - return name, nil -} - -func UniqueString(length int, digestArgs ...string) (string, error) { - return suffixDigest(length, digestArgs...) -} diff --git a/tooling/templatize/internal/naming/common_test.go b/tooling/templatize/internal/naming/common_test.go deleted file mode 100644 index 686c66c5c..000000000 --- a/tooling/templatize/internal/naming/common_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package naming - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestSuffixedName(t *testing.T) { - for _, testCase := range []struct { - name string - prefix string - suffixDigestArgs []string - maxLength int - suffixLength int - expected string - errorExpected bool - }{ - { - name: "no suffix", - prefix: "prefix", - suffixDigestArgs: []string{}, - maxLength: 10, - expected: "prefix", - }, - { - name: "no suffix - too long", - prefix: "prefix", - suffixDigestArgs: []string{}, - maxLength: 4, - expected: "", - errorExpected: true, - }, - { - name: "with suffix", - prefix: "prefix", - suffixDigestArgs: []string{"arg1"}, - maxLength: 10, - suffixLength: 3, - expected: "prefix-84f", - }, - { - name: "with suffix - too long", - prefix: "prefix", - suffixDigestArgs: []string{"arg1"}, - maxLength: 4, - suffixLength: 3, - expected: "", - errorExpected: true, - }, - { - name: "with multiple suffix args", - prefix: "prefix", - suffixDigestArgs: []string{"arg1", "arg2"}, - maxLength: 10, - suffixLength: 3, - expected: "prefix-cb9", - }, - } { - t.Run(testCase.name, func(t *testing.T) { - resourceName, err := suffixedName(testCase.prefix, "-", testCase.maxLength, testCase.suffixLength, testCase.suffixDigestArgs...) - if testCase.errorExpected { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } - assert.Equal(t, testCase.expected, resourceName) - }) - } -} diff --git a/tooling/templatize/main.go b/tooling/templatize/main.go index 3982de580..a7075646b 100644 --- a/tooling/templatize/main.go +++ b/tooling/templatize/main.go @@ -2,12 +2,12 @@ package main import ( "log" - "os" "github.com/spf13/cobra" "github.com/Azure/ARO-HCP/tooling/templatize/cmd/generate" "github.com/Azure/ARO-HCP/tooling/templatize/cmd/inspect" + "github.com/Azure/ARO-HCP/tooling/templatize/cmd/run" ) func main() { @@ -20,17 +20,11 @@ func main() { CompletionOptions: cobra.CompletionOptions{ HiddenDefaultCmd: true, }, - RunE: func(cmd *cobra.Command, args []string) error { - err := cmd.Help() - if err != nil { - return err - } - os.Exit(1) - return nil - }, } cmd.AddCommand(generate.NewCommand()) cmd.AddCommand(inspect.NewCommand()) + cmd.AddCommand(run.NewCommand()) + cmd.SetHelpCommand(&cobra.Command{Hidden: true}) if err := cmd.Execute(); err != nil { log.Fatal(err) diff --git a/tooling/templatize/pkg/config/config.go b/tooling/templatize/pkg/config/config.go new file mode 100644 index 000000000..58eafdcca --- /dev/null +++ b/tooling/templatize/pkg/config/config.go @@ -0,0 +1,195 @@ +package config + +import ( + "bytes" + "fmt" + "os" + "reflect" + "text/template" + + "gopkg.in/yaml.v3" +) + +func DefaultConfigReplacements() *ConfigReplacements { + return NewConfigReplacements("", "", "") +} + +func NewConfigReplacements(regionReplacement, regionShortReplacement, stampReplacement string) *ConfigReplacements { + return &ConfigReplacements{ + RegionReplacement: regionReplacement, + RegionShortReplacement: regionShortReplacement, + StampReplacement: stampReplacement, + } +} + +type ConfigReplacements struct { + RegionReplacement string + RegionShortReplacement string + StampReplacement string +} + +func (c *ConfigReplacements) AsMap() map[string]interface{} { + return map[string]interface{}{ + "ctx": map[string]interface{}{ + "region": c.RegionReplacement, + "regionShort": c.RegionShortReplacement, + "stamp": c.StampReplacement, + }, + } +} + +type ConfigProvider interface { + Validate(cloud, deployEnv string) error + GetVariables(cloud, deployEnv, region string, configReplacements *ConfigReplacements) (Variables, error) + GetDeployEnvVariables(cloud, deployEnv string, configReplacements *ConfigReplacements) (Variables, error) + GetRegions(cloud, deployEnv string) ([]string, error) + GetRegionOverrides(cloud, deployEnv, region string, configReplacements *ConfigReplacements) (Variables, error) +} + +func NewConfigProvider(config string) ConfigProvider { + return &configProviderImpl{ + config: config, + } +} + +func interfaceToVariables(i interface{}) (Variables, bool) { + // Helper, that reduces need for reflection calls, i.e. MapIndex + // from: https://github.com/peterbourgon/mergemap/blob/master/mergemap.go + value := reflect.ValueOf(i) + if value.Kind() == reflect.Map { + m := Variables{} + for _, k := range value.MapKeys() { + v := value.MapIndex(k).Interface() + if nestedMap, ok := interfaceToVariables(v); ok { + m[k.String()] = nestedMap + } else { + m[k.String()] = v + } + } + return m, true + } + return Variables{}, false +} + +// Merges variables, returns merged variables +// However the return value is only used for recursive updates on the map +// The actual merged variables are updated in the base map +func mergeVariables(base, override Variables) Variables { + for k, newValue := range override { + if baseValue, exists := base[k]; exists { + srcMap, srcMapOk := interfaceToVariables(newValue) + dstMap, dstMapOk := interfaceToVariables(baseValue) + if srcMapOk && dstMapOk { + newValue = mergeVariables(dstMap, srcMap) + } + } + base[k] = newValue + } + + return base +} + +func (cp *configProviderImpl) GetVariables(cloud, deployEnv, region string, configReplacements *ConfigReplacements) (Variables, error) { + variables, err := cp.GetDeployEnvVariables(cloud, deployEnv, configReplacements) + if err != nil { + return nil, err + } + + // region overrides + regionOverrides, err := cp.GetRegionOverrides(cloud, deployEnv, region, configReplacements) + if err != nil { + return nil, err + } + mergeVariables(variables, regionOverrides) + + return variables, nil +} + +func (cp *configProviderImpl) Validate(cloud, deployEnv string) error { + config, err := cp.loadConfig(DefaultConfigReplacements()) + if err != nil { + return err + } + if ok := config.HasCloud(cloud); !ok { + return fmt.Errorf("the cloud %s is not found in the config", cloud) + } + + if ok := config.HasDeployEnv(cloud, deployEnv); !ok { + return fmt.Errorf("the deployment env %s is not found under cloud %s", deployEnv, cloud) + } + return nil +} + +func (cp *configProviderImpl) GetDeployEnvVariables(cloud, deployEnv string, configReplacements *ConfigReplacements) (Variables, error) { + config, err := cp.loadConfig(configReplacements) + if err != nil { + return nil, err + } + err = cp.Validate(cloud, deployEnv) + if err != nil { + return nil, err + } + + variables := Variables{} + mergeVariables(variables, config.GetDefaults()) + mergeVariables(variables, config.GetCloudOverrides(cloud)) + mergeVariables(variables, config.GetDeployEnvOverrides(cloud, deployEnv)) + + return variables, nil +} + +func (cp *configProviderImpl) GetRegions(cloud, deployEnv string) ([]string, error) { + config, err := cp.loadConfig(DefaultConfigReplacements()) + if err != nil { + return nil, err + } + err = cp.Validate(cloud, deployEnv) + if err != nil { + return nil, err + } + regions := config.GetRegions(cloud, deployEnv) + return regions, nil +} + +func (cp *configProviderImpl) GetRegionOverrides(cloud, deployEnv, region string, configReplacements *ConfigReplacements) (Variables, error) { + config, err := cp.loadConfig(configReplacements) + if err != nil { + return nil, err + } + return config.GetRegionOverrides(cloud, deployEnv, region), nil +} + +func (cp *configProviderImpl) loadConfig(configReplacements *ConfigReplacements) (VariableOverrides, error) { + // TODO validate that field names are unique regardless of casing + // parse, execute and unmarshal the config file as a template to generate the final config file + bytes, err := PreprocessFile(cp.config, configReplacements.AsMap()) + if err != nil { + return nil, err + } + + currentVariableOverrides := NewVariableOverrides() + if err := yaml.Unmarshal(bytes, currentVariableOverrides); err == nil { + return currentVariableOverrides, nil + } else { + return nil, err + } +} + +func PreprocessFile(templateFilePath string, vars map[string]interface{}) ([]byte, error) { + tmpl := template.New("file") + content, err := os.ReadFile(templateFilePath) + if err != nil { + return nil, err + } + + tmpl, err = tmpl.Parse(string(content)) + if err != nil { + return nil, err + } + + var tmplBytes bytes.Buffer + if err := tmpl.Option("missingkey=error").Execute(&tmplBytes, vars); err != nil { + return nil, err + } + return tmplBytes.Bytes(), nil +} diff --git a/tooling/templatize/internal/config/config_test.go b/tooling/templatize/pkg/config/config_test.go similarity index 74% rename from tooling/templatize/internal/config/config_test.go rename to tooling/templatize/pkg/config/config_test.go index 22d9b6ee9..14827ccb0 100644 --- a/tooling/templatize/internal/config/config_test.go +++ b/tooling/templatize/pkg/config/config_test.go @@ -9,12 +9,12 @@ import ( func TestConfigProvider(t *testing.T) { region := "uksouth" - regionStamp := "1" - cxStamp := "cx" + regionShort := "uks" + stamp := "1" - configProvider := NewConfigProvider("../../testdata/config.yaml", region, regionStamp, cxStamp) + configProvider := NewConfigProvider("../../testdata/config.yaml") - variables, err := configProvider.GetVariables("public", "int", map[string]string{}) + variables, err := configProvider.GetVariables("public", "int", region, NewConfigReplacements(region, regionShort, stamp)) assert.NoError(t, err) assert.NotNil(t, variables) @@ -28,7 +28,7 @@ func TestConfigProvider(t *testing.T) { assert.Equal(t, "aro-hcp-int.azurecr.io/maestro-server:the-stable-one", variables["maestro_image"]) // key is in the config file, default, varaible value - assert.Equal(t, fmt.Sprintf("hcp-underlay-%s-%s", region, regionStamp), variables["region_resourcegroup"]) + assert.Equal(t, fmt.Sprintf("hcp-underlay-%s-%s", region, stamp), variables["region_resourcegroup"]) } func TestInterfaceToVariable(t *testing.T) { @@ -61,6 +61,20 @@ func TestInterfaceToVariable(t *testing.T) { "key2": "value2", }, }, + { + name: "nested map", + i: map[string]interface{}{ + "key1": map[string]interface{}{ + "key2": "value2", + }, + }, + ok: true, + expecetedVariables: Variables{ + "key1": Variables{ + "key2": "value2", + }, + }, + }, } for _, tc := range testCases { @@ -118,6 +132,18 @@ func TestMergeVariable(t *testing.T) { override: Variables{"key1": Variables{"key2": "value3"}}, expected: Variables{"key1": Variables{"key2": "value3"}}, }, + { + name: "override nested sub map", + base: Variables{"key1": Variables{"key2": Variables{"key3": "value3"}}}, + override: Variables{"key1": Variables{"key2": Variables{"key3": "value4"}}}, + expected: Variables{"key1": Variables{"key2": Variables{"key3": "value4"}}}, + }, + { + name: "override nested sub map multiple levels", + base: Variables{"key1": Variables{"key2": Variables{"key3": "value3"}}}, + override: Variables{"key1": Variables{"key2": Variables{"key4": "value4"}}, "key5": "value5"}, + expected: Variables{"key1": Variables{"key2": Variables{"key3": "value3", "key4": "value4"}}, "key5": "value5"}, + }, } for _, tc := range testCases { diff --git a/tooling/templatize/pkg/config/types.go b/tooling/templatize/pkg/config/types.go new file mode 100644 index 000000000..1538fbca4 --- /dev/null +++ b/tooling/templatize/pkg/config/types.go @@ -0,0 +1,113 @@ +package config + +import "fmt" + +type configProviderImpl struct { + config string +} + +type Variables map[string]interface{} + +func (v Variables) AddNested(key string, other map[string]interface{}) error { + if _, exists := v[key]; exists { + return fmt.Errorf("%s exists already in Variables", key) + } + other[key] = other + return nil +} + +func NewVariableOverrides() VariableOverrides { + return &variableOverrides{} +} + +type VariableOverrides interface { + GetDefaults() Variables + GetCloudOverrides(cloud string) Variables + GetDeployEnvOverrides(cloud, deployEnv string) Variables + GetRegionOverrides(cloud, deployEnv, region string) Variables + GetRegions(cloud, deployEnv string) []string + HasCloud(cloud string) bool + HasDeployEnv(cloud, deployEnv string) bool +} + +type variableOverrides struct { + Defaults Variables `yaml:"defaults"` + // key is the cloud alias + Overrides map[string]*struct { + Defaults Variables `yaml:"defaults"` + // key is the deploy env + Overrides map[string]*struct { + Defaults Variables `yaml:"defaults"` + // key is the region name + Overrides map[string]Variables `yaml:"regions"` + } `yaml:"environments"` + } `yaml:"clouds"` +} + +func (vo *variableOverrides) GetDefaults() Variables { + return vo.Defaults +} + +func (vo *variableOverrides) HasCloud(cloud string) bool { + _, ok := vo.Overrides[cloud] + return ok +} + +func (vo *variableOverrides) GetCloudOverrides(cloud string) Variables { + if cloudOverride, ok := vo.Overrides[cloud]; ok { + return cloudOverride.Defaults + } + return Variables{} +} + +func (vo *variableOverrides) HasDeployEnv(cloud, deployEnv string) bool { + if cloudOverride, ok := vo.Overrides[cloud]; ok { + _, ok := cloudOverride.Overrides[deployEnv] + return ok + } + return false +} + +func (vo *variableOverrides) GetDeployEnvOverrides(cloud, deployEnv string) Variables { + if cloudOverride, ok := vo.Overrides[cloud]; ok { + if deployEnvOverride, ok := cloudOverride.Overrides[deployEnv]; ok { + return deployEnvOverride.Defaults + } + } + return Variables{} +} + +func (vo *variableOverrides) GetRegions(cloud, deployEnv string) []string { + deployEnvOverrides, err := vo.getAllDeployEnvRegionOverrides(cloud, deployEnv) + if err != nil { + return []string{} + } + regions := make([]string, 0, len(deployEnvOverrides)) + for region := range deployEnvOverrides { + regions = append(regions, region) + } + return regions +} + +func (vo *variableOverrides) getAllDeployEnvRegionOverrides(cloud, deployEnv string) (map[string]Variables, error) { + if cloudOverride, ok := vo.Overrides[cloud]; ok { + if deployEnvOverride, ok := cloudOverride.Overrides[deployEnv]; ok { + return deployEnvOverride.Overrides, nil + } else { + return nil, fmt.Errorf("deploy env %s not found under cloud %s in config", deployEnv, cloud) + } + } + return nil, fmt.Errorf("cloud %s not found in config", cloud) +} + +func (vo *variableOverrides) GetRegionOverrides(cloud, deployEnv, region string) Variables { + regionOverrides, err := vo.getAllDeployEnvRegionOverrides(cloud, deployEnv) + if err != nil { + return Variables{} + } + if regionOverrides, ok := regionOverrides[region]; ok { + return regionOverrides + } else { + return Variables{} + } +} diff --git a/tooling/templatize/pkg/ev2/mapping.go b/tooling/templatize/pkg/ev2/mapping.go new file mode 100644 index 000000000..b70494d6f --- /dev/null +++ b/tooling/templatize/pkg/ev2/mapping.go @@ -0,0 +1,27 @@ +package ev2 + +import ( + "fmt" + "strings" +) + +func EV2Mapping(input map[string]interface{}, prefix []string) (map[string]string, map[string]interface{}) { + output := map[string]string{} + replaced := map[string]interface{}{} + for key, value := range input { + nestedKey := append(prefix, key) + nested, ok := value.(map[string]interface{}) + if ok { + flattened, replacement := EV2Mapping(nested, nestedKey) + for index, what := range flattened { + output[index] = what + } + replaced[key] = replacement + } else { + placeholder := fmt.Sprintf("__%s__", strings.ToUpper(strings.Join(nestedKey, "_"))) + output[placeholder] = strings.Join(nestedKey, ".") + replaced[key] = placeholder + } + } + return output, replaced +} diff --git a/tooling/templatize/pkg/ev2/mapping_test.go b/tooling/templatize/pkg/ev2/mapping_test.go new file mode 100644 index 000000000..d507d1dfb --- /dev/null +++ b/tooling/templatize/pkg/ev2/mapping_test.go @@ -0,0 +1,46 @@ +package ev2 + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + "github.com/Azure/ARO-HCP/tooling/templatize/pkg/config" +) + +func TestMapping(t *testing.T) { + testData := config.Variables{ + "key1": "value1", + "key2": 42, + "key3": true, + "parent": map[string]interface{}{ + "nested": "nestedvalue", + "deeper": map[string]interface{}{ + "deepest": "deepestvalue", + }, + }, + } + expectedFlattened := map[string]string{ + "__KEY1__": "key1", + "__KEY2__": "key2", + "__KEY3__": "key3", + "__PARENT_NESTED__": "parent.nested", + "__PARENT_DEEPER_DEEPEST__": "parent.deeper.deepest", + } + expecetedReplace := map[string]interface{}{ + "key1": "__KEY1__", + "key2": "__KEY2__", + "key3": "__KEY3__", + "parent": map[string]interface{}{ + "nested": "__PARENT_NESTED__", + "deeper": map[string]interface{}{"deepest": "__PARENT_DEEPER_DEEPEST__"}, + }, + } + flattened, replace := EV2Mapping(testData, []string{}) + if diff := cmp.Diff(expectedFlattened, flattened); diff != "" { + t.Errorf("got incorrect flattened: %v", diff) + } + if diff := cmp.Diff(expecetedReplace, replace); diff != "" { + t.Errorf("got incorrect replace: %v", diff) + } +} diff --git a/tooling/templatize/pkg/ev2/utils.go b/tooling/templatize/pkg/ev2/utils.go new file mode 100644 index 000000000..6d55d3506 --- /dev/null +++ b/tooling/templatize/pkg/ev2/utils.go @@ -0,0 +1,73 @@ +package ev2 + +import ( + "fmt" + + "github.com/Azure/ARO-HCP/tooling/templatize/pkg/config" +) + +// +// This package contains helper functions to extract EV2 conformant data from a config.yaml file. +// + +func newEv2ConfigReplacements() *config.ConfigReplacements { + return config.NewConfigReplacements( + "$location()", + "$(regionShortName)", + "$stamp()", + ) +} + +// GetNonRegionalServiceConfigVariables returns all non-regional configuration variables of a config.yaml file. +// Non regional means: global variables + cloud overrides + deployment environment overrides - but not regional overrides. +// The variable values are formatted to contain EV2 $location(), $stamp() and $(serviceConfigVar) variables. +// This function is useful to get the variables to fill the `Settings` section of an EV2 `ServiceConfig.json“ +func GetNonRegionalServiceConfigVariables(configProvider config.ConfigProvider, cloud, deployEnv string) (config.Variables, error) { + return configProvider.GetVariables(cloud, deployEnv, "", newEv2ConfigReplacements()) +} + +// GetRegionalServiceConfigVariableOverrides returns the regional overrides of a config.yaml file. +// The variable values are formatted to contain EV2 $location(), $stamp() and $(serviceConfigVar) variables. +// This function is useful to get the variables to fill the `Geographies/Regions` section of an EV2 `ServiceConfig.json` +func GetRegionalServiceConfigVariableOverrides(configProvider config.ConfigProvider, cloud, deployEnv string) (map[string]config.Variables, error) { + regions, err := configProvider.GetRegions(cloud, deployEnv) + if err != nil { + return nil, err + } + overrides := make(map[string]config.Variables) + for _, region := range regions { + regionOverrides, err := configProvider.GetRegionOverrides(cloud, deployEnv, region, newEv2ConfigReplacements()) + if err != nil { + return nil, err + } + overrides[region] = regionOverrides + } + return overrides, nil +} + +// ScopeBindingVariables retrieves and processes configuration variables for a given cloud and deployment environment. +// It uses the provided configProvider to fetch the variables, flattens them into a __VAR__ = $config(var) formatted map. +// This function is useful to get the find/replace pairs for an EV2 `ScopeBinding.json` +func ScopeBindingVariables(configProvider config.ConfigProvider, cloud, deployEnv string) (map[string]string, error) { + vars, err := configProvider.GetVariables(cloud, deployEnv, "", newEv2ConfigReplacements()) + if err != nil { + return nil, err + } + flattened, _ := EV2Mapping(vars, []string{}) + variables := make(map[string]string) + for key, value := range flattened { + variables[key] = fmt.Sprintf("$config(%s)", value) + } + return variables, nil +} + +// PreprocessFileForEV2 processes an arbitrary gotemplate file and replaces all config.yaml references +// while maintaining EV2 conformant system variables. +// This function is useful to process a pipeline.yaml file so that it contains EV2 system variables. +func PreprocessFileForEV2(configProvider config.ConfigProvider, cloud, deployEnv string, templateFile string) ([]byte, error) { + vars, err := configProvider.GetVariables(cloud, deployEnv, "", newEv2ConfigReplacements()) + if err != nil { + return nil, err + } + return config.PreprocessFile(templateFile, vars) +} diff --git a/tooling/templatize/pkg/ev2/utils_test.go b/tooling/templatize/pkg/ev2/utils_test.go new file mode 100644 index 000000000..5f51ae21e --- /dev/null +++ b/tooling/templatize/pkg/ev2/utils_test.go @@ -0,0 +1,18 @@ +package ev2 + +import ( + "testing" + + "github.com/Azure/ARO-HCP/tooling/templatize/internal/testutil" + "github.com/Azure/ARO-HCP/tooling/templatize/pkg/config" +) + +func TestPreprocessFileForEV2(t *testing.T) { + configProvider := config.NewConfigProvider("../../testdata/config.yaml") + content, err := PreprocessFileForEV2(configProvider, "public", "int", "../../testdata/pipeline.yaml") + if err != nil { + t.Fatalf("PreprocessFileForEV2 failed: %v", err) + } + testutil.CompareWithFixture(t, content, testutil.WithExtension(".yaml")) + +} diff --git a/tooling/templatize/pkg/pipeline/executiontarget.go b/tooling/templatize/pkg/pipeline/executiontarget.go new file mode 100644 index 000000000..afe02f50f --- /dev/null +++ b/tooling/templatize/pkg/pipeline/executiontarget.go @@ -0,0 +1,103 @@ +package pipeline + +import ( + "context" + "fmt" + "os" + "os/exec" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions" +) + +func lookupSubscriptionID(ctx context.Context, subscriptionName string) (string, error) { + // Create a new Azure identity client + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return "", fmt.Errorf("failed to obtain a credential: %v", err) + } + + // Create a new subscriptions client + client, err := armsubscriptions.NewClient(cred, nil) + if err != nil { + return "", fmt.Errorf("failed to create subscriptions client: %v", err) + } + + // List subscriptions and find the one with the matching name + pager := client.NewListPager(nil) + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return "", fmt.Errorf("failed to get next page of subscriptions: %v", err) + } + for _, sub := range page.Value { + if sub.DisplayName != nil && *sub.DisplayName == subscriptionName { + return *sub.SubscriptionID, nil + } + } + } + + return "", fmt.Errorf("subscription with name %q not found", subscriptionName) +} + +type StepExecutionTarget struct { + SubscriptionName string + SubscriptionID string + ResourceGroup string + AKSClusterName string +} + +func (target *StepExecutionTarget) KubeConfig(ctx context.Context) (string, error) { + if target.AKSClusterName == "" { + return "", fmt.Errorf("AKSClusterName is required to build a kubeconfig") + } + + // Create a new Azure identity client + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return "", fmt.Errorf("failed to obtain a credential: %v", err) + } + + // Create a new AKS client + client, err := armcontainerservice.NewManagedClustersClient(target.SubscriptionID, cred, nil) + if err != nil { + return "", fmt.Errorf("failed to create AKS client: %v", err) + } + + // Get the cluster access credentials + resp, err := client.ListClusterUserCredentials(ctx, target.ResourceGroup, target.AKSClusterName, nil) + if err != nil { + return "", fmt.Errorf("failed to get cluster access credentials: %v", err) + } + if len(resp.Kubeconfigs) == 0 { + return "", fmt.Errorf("no kubeconfig found") + } + kubeconfigContent := resp.Kubeconfigs[0].Value + + // store the kubeconfig content into a temporary file + // generate a unique temporary filename + tmpfile, err := os.CreateTemp("", "kubeconfig-*.yaml") + if err != nil { + return "", fmt.Errorf("failed to create temporary file for kubeconfig: %v", err) + } + defer tmpfile.Close() + + // store the kubeconfig content into the temporary file + if _, err := tmpfile.Write([]byte(kubeconfigContent)); err != nil { + return "", fmt.Errorf("failed to write to temporary kubeconfigfile %s: %v", tmpfile.Name(), err) + } + + // Run kubelogin to transform the kubeconfig + cmd := exec.CommandContext(ctx, "kubelogin", "convert-kubeconfig", "-l", "azurecli", "--kubeconfig", tmpfile.Name()) + output, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("failed to run kubelogin: %s %v", string(output), err) + } + + return tmpfile.Name(), nil +} + +func (target *StepExecutionTarget) Name() string { + return fmt.Sprintf("SUB %s / RG %s / AKS %s", target.SubscriptionName, target.ResourceGroup, target.AKSClusterName) +} diff --git a/tooling/templatize/pkg/pipeline/run.go b/tooling/templatize/pkg/pipeline/run.go new file mode 100644 index 000000000..3b9cf059b --- /dev/null +++ b/tooling/templatize/pkg/pipeline/run.go @@ -0,0 +1,82 @@ +package pipeline + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "gopkg.in/yaml.v3" + + "github.com/Azure/ARO-HCP/tooling/templatize/pkg/config" +) + +func NewPipelineFromFile(pipelineFilePath string, vars config.Variables) (*Pipeline, error) { + bytes, err := config.PreprocessFile(pipelineFilePath, vars) + if err != nil { + return nil, err + } + + pipeline := &Pipeline{ + pipelineFilePath: pipelineFilePath, + } + err = yaml.Unmarshal(bytes, pipeline) + if err != nil { + return nil, err + } + return pipeline, nil +} + +func (p *Pipeline) Run(ctx context.Context, vars config.Variables) error { + // set working directory to the pipeline file directory for the duration of + // the execution + originalDir, err := os.Getwd() + if err != nil { + return err + } + dir := filepath.Dir(p.pipelineFilePath) + err = os.Chdir(dir) + if err != nil { + return err + } + defer func() { + err := os.Chdir(originalDir) + if err != nil { + fmt.Printf("failed to reset directory: %v\n", err) + } + }() + + for _, step := range p.Steps { + err := step.run(ctx, vars) + if err != nil { + return err + } + } + return nil +} + +func (s *step) executionTarget(ctx context.Context) (*StepExecutionTarget, error) { + subscriptionID, err := lookupSubscriptionID(ctx, s.Subscription) + if err != nil { + return nil, err + } + return &StepExecutionTarget{ + SubscriptionName: s.Subscription, + SubscriptionID: subscriptionID, + ResourceGroup: s.ResourceGroup, + AKSClusterName: s.AKSClusterName, + }, nil +} + +func (s *step) run(ctx context.Context, vars config.Variables) error { + executionTarget, err := s.executionTarget(ctx) + if err != nil { + return err + } + switch s.Action.Type { + case "Shell": + return s.runShellStep(ctx, executionTarget, vars) + default: + return fmt.Errorf("unsupported action type %q", s.Action.Type) + } +} diff --git a/tooling/templatize/pkg/pipeline/shell.go b/tooling/templatize/pkg/pipeline/shell.go new file mode 100644 index 000000000..6a6acdbba --- /dev/null +++ b/tooling/templatize/pkg/pipeline/shell.go @@ -0,0 +1,55 @@ +package pipeline + +import ( + "context" + "fmt" + "os" + "os/exec" + + "github.com/Azure/ARO-HCP/tooling/templatize/pkg/config" +) + +func (s *step) runShellStep(ctx context.Context, executionTarget *StepExecutionTarget, vars config.Variables) error { + fmt.Printf("Execution target: %s\n", executionTarget.Name()) + fmt.Printf("Shell command: %v\n", s.Action.Command) + + // prepare kubeconfig + kubeconfigFile, err := executionTarget.KubeConfig(ctx) + if err != nil { + return fmt.Errorf("failed to build kubeconfig: %w", err) + } + + // schedule the deletion of the kubeconfig file after the command execution + defer func() { + if err := os.Remove(kubeconfigFile); err != nil { + fmt.Printf("Warning: failed to delete kubeconfig file %s: %v\n", kubeconfigFile, err) + } + }() + + // build ENV vars + envVars := os.Environ() + if executionTarget.AKSClusterName != "" { + kubeconfigFile, err := executionTarget.KubeConfig(ctx) + if err != nil { + return fmt.Errorf("failed to build kubeconfig for %s: %w", executionTarget.Name(), err) + } + envVars = append(envVars, fmt.Sprintf("KUBECONFIG=%s", kubeconfigFile)) + } + for _, e := range s.Action.Env { + value := vars[e.ConfigRef] // todo nested lookups + envVars = append(envVars, fmt.Sprintf("%s=%s", e.Name, value)) + } + + // execute the shell command with the environment variables + cmd := exec.CommandContext(ctx, s.Action.Command[0], s.Action.Command[1:]...) + cmd.Env = append(cmd.Env, envVars...) + output, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to execute shell command: %s %w", string(output), err) + } + + // print the output of the command + fmt.Println(string(output)) + + return nil +} diff --git a/tooling/templatize/pkg/pipeline/types.go b/tooling/templatize/pkg/pipeline/types.go new file mode 100644 index 000000000..7b749794d --- /dev/null +++ b/tooling/templatize/pkg/pipeline/types.go @@ -0,0 +1,26 @@ +package pipeline + +type Pipeline struct { + pipelineFilePath string + RolloutName string `yaml:"rolloutName"` + Steps []*step `yaml:"steps"` +} + +type step struct { + Name string `yaml:"name"` + Subscription string `yaml:"subscription"` + ResourceGroup string `yaml:"resourceGroup"` + AKSClusterName string `yaml:"aksCluster"` + Action action `yaml:"action"` +} + +type action struct { + Type string `yaml:"type"` + Command []string `yaml:"command"` + Env []EnvVar `yaml:"env"` +} + +type EnvVar struct { + Name string `yaml:"name"` + ConfigRef string `yaml:"configRef"` +} diff --git a/tooling/templatize/serviceconfig.json b/tooling/templatize/serviceconfig.json new file mode 100644 index 000000000..01c9b2ee7 --- /dev/null +++ b/tooling/templatize/serviceconfig.json @@ -0,0 +1,507 @@ +{ + "Settings": {}, + "Geographies": [ + { + "Name": "Asia Pacific", + "Settings": {}, + "Regions": [ + { + "Name": "apacsoutheast2", + "Settings": {} + }, + { + "Name": "eastasia", + "Settings": {} + }, + { + "Name": "southeastasia", + "Settings": {} + } + ] + }, + { + "Name": "Australia", + "Settings": {}, + "Regions": [ + { + "Name": "australiacentral", + "Settings": {} + }, + { + "Name": "australiacentral2", + "Settings": {} + }, + { + "Name": "australiaeast", + "Settings": {} + }, + { + "Name": "australiasoutheast", + "Settings": {} + } + ] + }, + { + "Name": "Austria", + "Settings": {}, + "Regions": [ + { + "Name": "austriaeast", + "Settings": {} + } + ] + }, + { + "Name": "Belgium", + "Settings": {}, + "Regions": [ + { + "Name": "belgiumcentral", + "Settings": {} + } + ] + }, + { + "Name": "Brazil", + "Settings": {}, + "Regions": [ + { + "Name": "brazilnortheast", + "Settings": {} + }, + { + "Name": "brazilsouth", + "Settings": {} + }, + { + "Name": "brazilsoutheast", + "Settings": {} + } + ] + }, + { + "Name": "Canada", + "Settings": {}, + "Regions": [ + { + "Name": "canadacentral", + "Settings": {} + }, + { + "Name": "canadaeast", + "Settings": {} + } + ] + }, + { + "Name": "Canary (US)", + "Settings": {}, + "Regions": [ + { + "Name": "centraluseuap", + "Settings": {} + }, + { + "Name": "eastus2euap", + "Settings": {} + } + ] + }, + { + "Name": "Chile", + "Settings": {}, + "Regions": [ + { + "Name": "chilecentral", + "Settings": {} + } + ] + }, + { + "Name": "Denmark", + "Settings": {}, + "Regions": [ + { + "Name": "denmarkeast", + "Settings": {} + } + ] + }, + { + "Name": "Europe", + "Settings": {}, + "Regions": [ + { + "Name": "northeurope", + "Settings": {} + }, + { + "Name": "westeurope", + "Settings": {} + } + ] + }, + { + "Name": "France", + "Settings": {}, + "Regions": [ + { + "Name": "francecentral", + "Settings": {} + }, + { + "Name": "francesouth", + "Settings": {} + } + ] + }, + { + "Name": "Germany", + "Settings": {}, + "Regions": [ + { + "Name": "germanynorth", + "Settings": {} + }, + { + "Name": "germanywestcentral", + "Settings": {} + } + ] + }, + { + "Name": "India", + "Settings": {}, + "Regions": [ + { + "Name": "centralindia", + "Settings": {} + }, + { + "Name": "jioindiacentral", + "Settings": {} + }, + { + "Name": "jioindiawest", + "Settings": {} + }, + { + "Name": "southindia", + "Settings": {} + }, + { + "Name": "westindia", + "Settings": {} + } + ] + }, + { + "Name": "Indonesia", + "Settings": {}, + "Regions": [ + { + "Name": "indonesiacentral", + "Settings": {} + } + ] + }, + { + "Name": "Israel", + "Settings": {}, + "Regions": [ + { + "Name": "israelcentral", + "Settings": {} + }, + { + "Name": "israelnorthwest", + "Settings": {} + } + ] + }, + { + "Name": "Italy", + "Settings": {}, + "Regions": [ + { + "Name": "italynorth", + "Settings": {} + } + ] + }, + { + "Name": "Japan", + "Settings": {}, + "Regions": [ + { + "Name": "japaneast", + "Settings": {} + }, + { + "Name": "japanwest", + "Settings": {} + } + ] + }, + { + "Name": "Korea", + "Settings": {}, + "Regions": [ + { + "Name": "koreacentral", + "Settings": {} + }, + { + "Name": "koreasouth", + "Settings": {} + }, + { + "Name": "koreasouth2", + "Settings": {} + } + ] + }, + { + "Name": "Malaysia", + "Settings": {}, + "Regions": [ + { + "Name": "malaysiasouth", + "Settings": {} + }, + { + "Name": "malaysiawest", + "Settings": {} + } + ] + }, + { + "Name": "Mexico", + "Settings": {}, + "Regions": [ + { + "Name": "mexicocentral", + "Settings": {} + } + ] + }, + { + "Name": "New Zealand", + "Settings": {}, + "Regions": [ + { + "Name": "newzealandnorth", + "Settings": {} + } + ] + }, + { + "Name": "Norway", + "Settings": {}, + "Regions": [ + { + "Name": "norwayeast", + "Settings": {} + }, + { + "Name": "norwaywest", + "Settings": {} + } + ] + }, + { + "Name": "Poland", + "Settings": {}, + "Regions": [ + { + "Name": "polandcentral", + "Settings": {} + } + ] + }, + { + "Name": "Qatar", + "Settings": {}, + "Regions": [ + { + "Name": "qatarcentral", + "Settings": {} + } + ] + }, + { + "Name": "South Africa", + "Settings": {}, + "Regions": [ + { + "Name": "southafricanorth", + "Settings": {} + }, + { + "Name": "southafricawest", + "Settings": {} + } + ] + }, + { + "Name": "Spain", + "Settings": {}, + "Regions": [ + { + "Name": "spaincentral", + "Settings": {} + } + ] + }, + { + "Name": "Stage (US)", + "Settings": {}, + "Regions": [ + { + "Name": "eastusslv", + "Settings": {} + }, + { + "Name": "eastusstg", + "Settings": {} + }, + { + "Name": "southcentralusstg", + "Settings": {} + } + ] + }, + { + "Name": "Sweden", + "Settings": {}, + "Regions": [ + { + "Name": "swedencentral", + "Settings": {} + }, + { + "Name": "swedensouth", + "Settings": {} + } + ] + }, + { + "Name": "Switzerland", + "Settings": {}, + "Regions": [ + { + "Name": "switzerlandnorth", + "Settings": {} + }, + { + "Name": "switzerlandwest", + "Settings": {} + } + ] + }, + { + "Name": "Taiwan", + "Settings": {}, + "Regions": [ + { + "Name": "taiwannorth", + "Settings": {} + }, + { + "Name": "taiwannorthwest", + "Settings": {} + } + ] + }, + { + "Name": "UAE", + "Settings": {}, + "Regions": [ + { + "Name": "uaecentral", + "Settings": {} + }, + { + "Name": "uaenorth", + "Settings": {} + } + ] + }, + { + "Name": "United Kingdom", + "Settings": {}, + "Regions": [ + { + "Name": "uksouth", + "Settings": {} + }, + { + "Name": "ukwest", + "Settings": {} + } + ] + }, + { + "Name": "United States", + "Settings": {}, + "Regions": [ + { + "Name": "centralus", + "Settings": {} + }, + { + "Name": "eastus", + "Settings": {} + }, + { + "Name": "eastus2", + "Settings": {} + }, + { + "Name": "northcentralus", + "Settings": {} + }, + { + "Name": "southcentralus", + "Settings": {} + }, + { + "Name": "southcentralus2", + "Settings": {} + }, + { + "Name": "southeastus", + "Settings": {} + }, + { + "Name": "southeastus3", + "Settings": {} + }, + { + "Name": "southeastus5", + "Settings": {} + }, + { + "Name": "southwestus", + "Settings": {} + }, + { + "Name": "westcentralus", + "Settings": {} + }, + { + "Name": "westus", + "Settings": {} + }, + { + "Name": "westus2", + "Settings": {} + }, + { + "Name": "westus3", + "Settings": {} + } + ] + } + ] +} diff --git a/tooling/templatize/testdata/config.yaml b/tooling/templatize/testdata/config.yaml index 0eb03e986..0c39106bf 100644 --- a/tooling/templatize/testdata/config.yaml +++ b/tooling/templatize/testdata/config.yaml @@ -1,9 +1,10 @@ defaults: tenantId: "72f988bf-86f1-41af-91ab-2d7cd011db47" - region_resourcegroup: hcp-underlay-{{ .ctx.region }}-{{ .ctx.regionStamp }} - region_eventgrid_namespace: maestro-eventgrid-{{ .ctx.region }}-{{ .ctx.regionStamp }} + region_resourcegroup: hcp-underlay-{{ .ctx.region }}-{{ .ctx.stamp }} + region_eventgrid_namespace: maestro-eventgrid-{{ .ctx.region }}-{{ .ctx.stamp }} aks_name: aro-hcp-aks maestro_msi: "maestro-server" + subscription: hcp-svc-{{ .ctx.region }} clouds: fairfax: defaults: @@ -12,9 +13,9 @@ clouds: environments: dev: defaults: - region_resourcegroup: hcp-underlay-{{ .ctx.region }}-{{ .ctx.regionStamp }} - region_maestro_keyvault: maestro-kv-{{ .ctx.region }}-{{ .ctx.regionStamp }} - svc_resourcegroup: hcp-underlay-{{ .ctx.region }}-svc-{{ .ctx.regionStamp }} + region_resourcegroup: hcp-underlay-{{ .ctx.region }}-{{ .ctx.stamp }} + region_maestro_keyvault: maestro-kv-{{ .ctx.region }}-{{ .ctx.stamp }} + svc_resourcegroup: hcp-underlay-{{ .ctx.region }}-svc-{{ .ctx.stamp }} maestro_helm_chart: ../maestro/deploy/helm/server maestro_image: aro-hcp-dev.azurecr.io/maestro-server:the-new-one int: diff --git a/tooling/templatize/testdata/pipeline.yaml b/tooling/templatize/testdata/pipeline.yaml new file mode 100644 index 000000000..9e724be62 --- /dev/null +++ b/tooling/templatize/testdata/pipeline.yaml @@ -0,0 +1,12 @@ +serviceGroup: Microsoft.Azure.ARO.Test +rolloutName: Maestro Server Rollout +steps: +- name: deploy + subscription: {{ .subscription }} + resourceGroup: {{ .region_resourcegroup }} + action: + type: Shell + command: ["make", "hi"] + env: + - name: EVENTGRID_NAME + configRef: maestroEventgridName diff --git a/tooling/templatize/testdata/zz_fixture_TestPreprocessFileForEV2.yaml b/tooling/templatize/testdata/zz_fixture_TestPreprocessFileForEV2.yaml new file mode 100644 index 000000000..a5b8e413b --- /dev/null +++ b/tooling/templatize/testdata/zz_fixture_TestPreprocessFileForEV2.yaml @@ -0,0 +1,12 @@ +serviceGroup: Microsoft.Azure.ARO.Test +rolloutName: Maestro Server Rollout +steps: +- name: deploy + subscription: hcp-svc-$location() + resourceGroup: hcp-underlay-$location()-$stamp() + action: + type: Shell + command: ["make", "hi"] + env: + - name: EVENTGRID_NAME + configRef: maestroEventgridName diff --git a/tooling/templatize/testdata/zz_fixture_TestRawOptions.sh b/tooling/templatize/testdata/zz_fixture_TestRawOptions.sh index 198bb1a47..ba6e22a7e 100644 --- a/tooling/templatize/testdata/zz_fixture_TestRawOptions.sh +++ b/tooling/templatize/testdata/zz_fixture_TestRawOptions.sh @@ -1,10 +1,10 @@ # copy from maestro/Makefile#L14 deploy-server: TENANT_ID="72f988bf-86f1-41af-91ab-2d7cd011db47" - REGION_RG="hcp-underlay-uksouth-1" - EVENTGRID_NS="maestro-eventgrid-uksouth-1" - MAESTRO_KV="maestro-kv-uksouth-1" - SERVICE_RG="hcp-underlay-uksouth-svc-1" + REGION_RG="hcp-underlay-uksouth-fghij" + EVENTGRID_NS="maestro-eventgrid-uksouth-fghij" + MAESTRO_KV="maestro-kv-uksouth-fghij" + SERVICE_RG="hcp-underlay-uksouth-svc-fghij" AKS="aro-hcp-aks" MAESTRO_MI="maestro-server" HELM_CHART="../maestro/deploy/helm/server"