diff --git a/backend/pipeline.yaml b/backend/pipeline.yaml index 2fb5a91d2..68ca02805 100644 --- a/backend/pipeline.yaml +++ b/backend/pipeline.yaml @@ -1,5 +1,6 @@ -serviceGroup: Microsoft.Azure.ARO.Test -rolloutName: RP - Backend +$schema: "pipeline.schema.v1" +serviceGroup: Microsoft.Azure.ARO.HCP.RP.Backend +rolloutName: RP Backend Rollout resourceGroups: - name: {{ .svc.rg }} subscription: {{ .svc.subscription }} diff --git a/dev-infrastructure/mgmt-pipeline.yaml b/dev-infrastructure/mgmt-pipeline.yaml index 9f8747241..5bda12bf3 100644 --- a/dev-infrastructure/mgmt-pipeline.yaml +++ b/dev-infrastructure/mgmt-pipeline.yaml @@ -1,4 +1,5 @@ -serviceGroup: Microsoft.Azure.ARO.Test +$schema: "pipeline.schema.v1" +serviceGroup: Microsoft.Azure.ARO.HCP.Management.Infra rolloutName: Management Cluster Rollout resourceGroups: - name: {{ .svc.rg }} @@ -16,14 +17,17 @@ resourceGroups: action: ARM template: templates/mgmt-cluster.bicep parameters: configurations/mgmt-cluster.tmpl.bicepparam - inputs: - - name: clusterServiceMIResourceId - step: regionOutput - output: cs.msi.resourceID + variables: + - name: mgmt.clusterServiceResourceId + input: + step: regionOutput + name: cs + dependsOn: + - regionOutput - name: enable-metrics action: Shell command: scripts/enable-aks-metrics.sh - env: + variables: - name: RESOURCEGROUP configRef: mgmt.rg - name: AKS_NAME diff --git a/dev-infrastructure/region-pipeline.yaml b/dev-infrastructure/region-pipeline.yaml index 69125cdeb..b5e48291b 100644 --- a/dev-infrastructure/region-pipeline.yaml +++ b/dev-infrastructure/region-pipeline.yaml @@ -1,3 +1,4 @@ +$schema: "pipeline.schema.v1" serviceGroup: Microsoft.Azure.ARO.HCP.Region rolloutName: Region Rollout resourceGroups: diff --git a/dev-infrastructure/svc-pipeline.yaml b/dev-infrastructure/svc-pipeline.yaml index 7db3cb0cc..b3941307a 100644 --- a/dev-infrastructure/svc-pipeline.yaml +++ b/dev-infrastructure/svc-pipeline.yaml @@ -1,3 +1,4 @@ +$schema: "pipeline.schema.v1" serviceGroup: Microsoft.Azure.ARO.HCP.Service.Infra rolloutName: Service Cluster Rollout resourceGroups: @@ -12,7 +13,7 @@ resourceGroups: - name: enable-metrics action: Shell command: scripts/enable-aks-metrics.sh - env: + variables: - name: RESOURCEGROUP configRef: svc.rg - name: AKS_NAME diff --git a/dev-infrastructure/templates/outputs/region.bicep b/dev-infrastructure/templates/outputs/region.bicep index 2f4bbaa54..da5bbaaad 100644 --- a/dev-infrastructure/templates/outputs/region.bicep +++ b/dev-infrastructure/templates/outputs/region.bicep @@ -3,8 +3,4 @@ resource csMSI 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = { location: resourceGroup().location } -output cs object = { - msi: { - resourceID: csMSI.id - } -} +output cs string = csMSI.id diff --git a/frontend/pipeline.yaml b/frontend/pipeline.yaml index f342f5302..85516de2a 100644 --- a/frontend/pipeline.yaml +++ b/frontend/pipeline.yaml @@ -1,5 +1,6 @@ -serviceGroup: Microsoft.Azure.ARO.Test -rolloutName: RP - Frontend +$schema: "pipeline.schema.v1" +serviceGroup: Microsoft.Azure.ARO.HCP.RP.Frontend +rolloutName: RP Frontend Rollout resourceGroups: - name: {{ .svc.rg }} subscription: {{ .svc.subscription }} diff --git a/tooling/templatize/pkg/pipeline/pipeline.schema.v1.json b/tooling/templatize/pkg/pipeline/pipeline.schema.v1.json new file mode 100644 index 000000000..6a3c16dc9 --- /dev/null +++ b/tooling/templatize/pkg/pipeline/pipeline.schema.v1.json @@ -0,0 +1,228 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "pipeline.schema.v1", + "type": "object", + "definitions": { + "variable": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "input": { + "type": "object", + "additionalProperties": false, + "properties": { + "step": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + "required": [ + "step", + "name" + ] + }, + "configRef": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "oneOf": [ + { + "required": [ + "name", + "input" + ] + }, + { + "required": [ + "name", + "configRef" + ] + }, + { + "required": [ + "name", + "value" + ] + } + ], + "required": [ + "name" + ] + } + }, + "properties": { + "serviceGroup": { + "type": "string" + }, + "rolloutName": { + "type": "string" + }, + "resourceGroups": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "subscription": { + "type": "string" + }, + "steps": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "action": { + "type": "string", + "enum": ["ARM", "Shell"] + }, + "template": { + "type": "string" + }, + "parameters": { + "type": "string" + }, + "deploymentLevel": { + "type": "string", + "enum": ["ResourceGroup", "Subscription"] + }, + "command": { + "type": "string" + }, + "variables": { + "type": "array", + "items": { + "$ref": "#/definitions/variable" + } + }, + "dependsOn": { + "type": "array", + "items": { + "type": "string" + } + }, + "dryRun": { + "type": "object" + } + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "action": { + "type": "string", + "enum": ["ARM"] + }, + "template": { + "type": "string" + }, + "parameters": { + "type": "string" + }, + "variables": { + "type": "array", + "items": { + "$ref": "#/definitions/variable" + } + }, + "dependsOn": { + "type": "array", + "items": { + "type": "string" + } + }, + "deploymentLevel": { + "type": "string", + "enum": ["ResourceGroup", "Subscription"] + } + }, + "required": [ + "template", + "parameters" + ] + }, + { + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "action": { + "type": "string", + "enum": ["Shell"] + }, + "command": { + "type": "string" + }, + "variables": { + "type": "array", + "items": { + "$ref": "#/definitions/variable" + } + }, + "dependsOn": { + "type": "array", + "items": { + "type": "string" + } + }, + "dryRun": { + "type": "object", + "additionalProperties": false, + "properties": { + "command": { + "type": "string" + }, + "envVars": { + "type": "array", + "items": { + "$ref": "#/definitions/variable" + } + } + } + } + }, + "required": [ + "command" + ] + } + ], + "required": [ + "name", + "action" + ] + } + }, + "aksCluster": { + "type": "string" + } + }, + "required": [ + "name", + "subscription", + "steps" + ] + } + } + }, + "required": [ + "serviceGroup", + "rolloutName", + "resourceGroups" + ] +} diff --git a/tooling/templatize/pkg/pipeline/run.go b/tooling/templatize/pkg/pipeline/run.go index c376427d4..171230d8b 100644 --- a/tooling/templatize/pkg/pipeline/run.go +++ b/tooling/templatize/pkg/pipeline/run.go @@ -19,6 +19,12 @@ func NewPipelineFromFile(pipelineFilePath string, vars config.Variables) (*Pipel if err != nil { return nil, fmt.Errorf("failed to preprocess pipeline file %w", err) } + + err = ValidatePipelineSchema(bytes) + if err != nil { + return nil, fmt.Errorf("failed to validate pipeline schema: %w", err) + } + absPath, err := filepath.Abs(pipelineFilePath) if err != nil { return nil, fmt.Errorf("failed to get absolute path for pipeline file %q: %w", pipelineFilePath, err) @@ -199,49 +205,6 @@ func (s *Step) description() string { return fmt.Sprintf("Step %s\n Kind: %s\n %s", s.Name, s.Action, strings.Join(details, "\n ")) } -func (p *Pipeline) Validate() error { - // collect all steps from all resourcegroups and fail if there are duplicates - stepMap := make(map[string]*Step) - for _, rg := range p.ResourceGroups { - for _, step := range rg.Steps { - if _, ok := stepMap[step.Name]; ok { - return fmt.Errorf("duplicate step name %q", step.Name) - } - stepMap[step.Name] = step - } - } - - // validate dependsOn for a step exists - for _, step := range stepMap { - for _, dep := range step.DependsOn { - if _, ok := stepMap[dep]; !ok { - return fmt.Errorf("invalid dependency on step %s: dependency %s does not exist", step.Name, dep) - } - } - } - - // todo check for circular dependencies - - // validate resource groups - for _, rg := range p.ResourceGroups { - err := rg.Validate() - if err != nil { - return err - } - } - return nil -} - -func (rg *ResourceGroup) Validate() error { - if rg.Name == "" { - return fmt.Errorf("resource group name is required") - } - if rg.Subscription == "" { - return fmt.Errorf("subscription is required") - } - return nil -} - func getInputValues(configuredVariables []Variable, inputs map[string]output) (map[string]any, error) { values := make(map[string]any) for _, i := range configuredVariables { diff --git a/tooling/templatize/pkg/pipeline/validation.go b/tooling/templatize/pkg/pipeline/validation.go new file mode 100644 index 000000000..033a909b5 --- /dev/null +++ b/tooling/templatize/pkg/pipeline/validation.go @@ -0,0 +1,118 @@ +package pipeline + +import ( + _ "embed" + "encoding/json" + "fmt" + + "github.com/santhosh-tekuri/jsonschema/v6" + "gopkg.in/yaml.v3" +) + +//go:embed pipeline.schema.v1.json +var pipelineSchemaV1Content []byte + +func ValidatePipelineSchema(pipelineContent []byte) error { + // unmarshal pipeline content + pipelineMap := make(map[string]interface{}) + err := yaml.Unmarshal(pipelineContent, &pipelineMap) + if err != nil { + return fmt.Errorf("failed to unmarshal pipeline YAML content: %v", err) + } + + // load pipeline schema + pipelineSchema, schemaUrl, err := getSchemaForPipeline(pipelineMap) + if err != nil { + return fmt.Errorf("failed to load pipeline schema: %v", err) + } + + // validate pipeline schema + err = pipelineSchema.Validate(pipelineMap) + if err != nil { + return fmt.Errorf("pipeline is not compliant with schema %s: %v", schemaUrl, err) + } + return nil +} + +func getSchemaForPipeline(pipelineMap map[string]interface{}) (*jsonschema.Schema, string, error) { + schemaRef, ok := pipelineMap["$schema"].(string) + if !ok { + return nil, "", fmt.Errorf("pipeline $schema reference is missing - add $schema: pipeline.schema.v1") + } + + switch schemaRef { + case "pipeline.schema.v1": + return compileSchema(pipelineSchemaV1Content) + default: + return nil, "", fmt.Errorf("unsupported schema reference: %s", schemaRef) + } +} + +func compileSchema(schemaBytes []byte) (*jsonschema.Schema, string, error) { + // parse schema content + schemaMap := make(map[string]interface{}) + err := json.Unmarshal(schemaBytes, &schemaMap) + if err != nil { + return nil, "", fmt.Errorf("failed to unmarshal schema content: %v", err) + } + schemaUrl, ok := schemaMap["title"].(string) + if !ok { + return nil, "", fmt.Errorf("failed to get schema title") + } + + // compile schema + c := jsonschema.NewCompiler() + err = c.AddResource(schemaUrl, schemaMap) + if err != nil { + return nil, "", fmt.Errorf("failed to add schema resource %s: %v", schemaUrl, err) + } + pipelineSchema, err := c.Compile(schemaUrl) + if err != nil { + return nil, "", fmt.Errorf("failed to compile schema %s: %v", schemaUrl, err) + } + + return pipelineSchema, schemaUrl, nil +} + +func (p *Pipeline) Validate() error { + // collect all steps from all resourcegroups and fail if there are duplicates + stepMap := make(map[string]*Step) + for _, rg := range p.ResourceGroups { + for _, step := range rg.Steps { + if _, ok := stepMap[step.Name]; ok { + return fmt.Errorf("duplicate step name %q", step.Name) + } + stepMap[step.Name] = step + } + } + + // validate dependsOn for a step exists + for _, step := range stepMap { + for _, dep := range step.DependsOn { + if _, ok := stepMap[dep]; !ok { + return fmt.Errorf("invalid dependency on step %s: dependency %s does not exist", step.Name, dep) + } + } + } + + // todo check for circular dependencies + + // validate resource groups + for _, rg := range p.ResourceGroups { + err := rg.Validate() + if err != nil { + return err + } + } + return nil +} + +func (rg *ResourceGroup) Validate() error { + if rg.Name == "" { + return fmt.Errorf("resource group name is required") + } + if rg.Subscription == "" { + return fmt.Errorf("subscription is required") + } + return nil +}