Skip to content

Commit

Permalink
pipeline jsonschema (#930)
Browse files Browse the repository at this point in the history 
* pipeline jsonschema

added a schema definition and validation for pipeline.yaml files

Signed-off-by: Gerd Oberlechner <[email protected]>

* adapt test

Signed-off-by: Gerd Oberlechner <[email protected]>

* schema loading tests

Signed-off-by: Gerd Oberlechner <[email protected]>

---------

Signed-off-by: Gerd Oberlechner <[email protected]>
  • Loading branch information
@geoberle
geoberle authored Dec 5, 2024
1 parent 254dce0 commit c84097c
Show file tree
Hide file tree
Showing 15 changed files with 644 additions and 212 deletions.
5 changes: 3 additions & 2 deletions backend/pipeline.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
serviceGroup: Microsoft.Azure.ARO.Test
rolloutName: RP - Backend
$schema: "pipeline.schema.v1"
serviceGroup: Microsoft.Azure.ARO.HCP.RP.Backend
rolloutName: RP Backend Rollout
resourceGroups:
- name: {{ .svc.rg }}
subscription: {{ .svc.subscription }}
Expand Down
16 changes: 10 additions & 6 deletions dev-infrastructure/mgmt-pipeline.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
serviceGroup: Microsoft.Azure.ARO.Test
$schema: "pipeline.schema.v1"
serviceGroup: Microsoft.Azure.ARO.HCP.Management.Infra
rolloutName: Management Cluster Rollout
resourceGroups:
- name: {{ .svc.rg }}
Expand All @@ -16,14 +17,17 @@ resourceGroups:
action: ARM
template: templates/mgmt-cluster.bicep
parameters: configurations/mgmt-cluster.tmpl.bicepparam
inputs:
- name: clusterServiceMIResourceId
step: regionOutput
output: cs.msi.resourceID
variables:
- name: mgmt.clusterServiceResourceId
input:
step: regionOutput
name: cs
dependsOn:
- regionOutput
- name: enable-metrics
action: Shell
command: scripts/enable-aks-metrics.sh
env:
variables:
- name: RESOURCEGROUP
configRef: mgmt.rg
- name: AKS_NAME
Expand Down
1 change: 1 addition & 0 deletions dev-infrastructure/region-pipeline.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
$schema: "pipeline.schema.v1"
serviceGroup: Microsoft.Azure.ARO.HCP.Region
rolloutName: Region Rollout
resourceGroups:
Expand Down
3 changes: 2 additions & 1 deletion dev-infrastructure/svc-pipeline.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
$schema: "pipeline.schema.v1"
serviceGroup: Microsoft.Azure.ARO.HCP.Service.Infra
rolloutName: Service Cluster Rollout
resourceGroups:
Expand All @@ -12,7 +13,7 @@ resourceGroups:
- name: enable-metrics
action: Shell
command: scripts/enable-aks-metrics.sh
env:
variables:
- name: RESOURCEGROUP
configRef: svc.rg
- name: AKS_NAME
Expand Down
6 changes: 1 addition & 5 deletions dev-infrastructure/templates/outputs/region.bicep
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,4 @@ resource csMSI 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
location: resourceGroup().location
}

output cs object = {
msi: {
resourceID: csMSI.id
}
}
output cs string = csMSI.id
5 changes: 3 additions & 2 deletions frontend/pipeline.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
serviceGroup: Microsoft.Azure.ARO.Test
rolloutName: RP - Frontend
$schema: "pipeline.schema.v1"
serviceGroup: Microsoft.Azure.ARO.HCP.RP.Frontend
rolloutName: RP Frontend Rollout
resourceGroups:
- name: {{ .svc.rg }}
subscription: {{ .svc.subscription }}
Expand Down
6 changes: 4 additions & 2 deletions tooling/templatize/internal/end2end/e2e_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package testutil
import (
"context"
"os"
"path/filepath"
"testing"

"gotest.tools/v3/assert"
Expand Down Expand Up @@ -134,14 +135,15 @@ func TestE2EShell(t *testing.T) {
t.Skip("Skipping end-to-end tests")
}

tmpDir := t.TempDir()
tmpDir, err := filepath.EvalSymlinks(t.TempDir())
assert.NilError(t, err)

e2eImpl := newE2E(tmpDir)

e2eImpl.AddStep(pipeline.Step{
Name: "readInput",
Action: "Shell",
Command: "/usr/bin/echo ${PWD} > env.txt",
Command: "/bin/echo ${PWD} > env.txt",
})

persistAndRun(t, &e2eImpl)
Expand Down
228 changes: 228 additions & 0 deletions tooling/templatize/pkg/pipeline/pipeline.schema.v1.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "pipeline.schema.v1",
"type": "object",
"definitions": {
"variable": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"input": {
"type": "object",
"additionalProperties": false,
"properties": {
"step": {
"type": "string"
},
"name": {
"type": "string"
}
},
"required": [
"step",
"name"
]
},
"configRef": {
"type": "string"
},
"value": {
"type": "string"
}
},
"oneOf": [
{
"required": [
"name",
"input"
]
},
{
"required": [
"name",
"configRef"
]
},
{
"required": [
"name",
"value"
]
}
],
"required": [
"name"
]
}
},
"properties": {
"serviceGroup": {
"type": "string"
},
"rolloutName": {
"type": "string"
},
"resourceGroups": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"subscription": {
"type": "string"
},
"steps": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
"name": {
"type": "string"
},
"action": {
"type": "string",
"enum": ["ARM", "Shell"]
},
"template": {
"type": "string"
},
"parameters": {
"type": "string"
},
"deploymentLevel": {
"type": "string",
"enum": ["ResourceGroup", "Subscription"]
},
"command": {
"type": "string"
},
"variables": {
"type": "array",
"items": {
"$ref": "#/definitions/variable"
}
},
"dependsOn": {
"type": "array",
"items": {
"type": "string"
}
},
"dryRun": {
"type": "object"
}
},
"oneOf": [
{
"additionalProperties": false,
"properties": {
"name": {
"type": "string"
},
"action": {
"type": "string",
"enum": ["ARM"]
},
"template": {
"type": "string"
},
"parameters": {
"type": "string"
},
"variables": {
"type": "array",
"items": {
"$ref": "#/definitions/variable"
}
},
"dependsOn": {
"type": "array",
"items": {
"type": "string"
}
},
"deploymentLevel": {
"type": "string",
"enum": ["ResourceGroup", "Subscription"]
}
},
"required": [
"template",
"parameters"
]
},
{
"additionalProperties": false,
"properties": {
"name": {
"type": "string"
},
"action": {
"type": "string",
"enum": ["Shell"]
},
"command": {
"type": "string"
},
"variables": {
"type": "array",
"items": {
"$ref": "#/definitions/variable"
}
},
"dependsOn": {
"type": "array",
"items": {
"type": "string"
}
},
"dryRun": {
"type": "object",
"additionalProperties": false,
"properties": {
"command": {
"type": "string"
},
"variables": {
"type": "array",
"items": {
"$ref": "#/definitions/variable"
}
}
}
}
},
"required": [
"command"
]
}
],
"required": [
"name",
"action"
]
}
},
"aksCluster": {
"type": "string"
}
},
"required": [
"name",
"subscription",
"steps"
]
}
}
},
"required": [
"serviceGroup",
"rolloutName",
"resourceGroups"
]
}
49 changes: 6 additions & 43 deletions tooling/templatize/pkg/pipeline/run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@ func NewPipelineFromFile(pipelineFilePath string, vars config.Variables) (*Pipel
if err != nil {
return nil, fmt.Errorf("failed to preprocess pipeline file %w", err)
}

err = ValidatePipelineSchema(bytes)
if err != nil {
return nil, fmt.Errorf("failed to validate pipeline schema: %w", err)
}

absPath, err := filepath.Abs(pipelineFilePath)
if err != nil {
return nil, fmt.Errorf("failed to get absolute path for pipeline file %q: %w", pipelineFilePath, err)
Expand Down Expand Up @@ -200,49 +206,6 @@ func (s *Step) description() string {
return fmt.Sprintf("Step %s\n Kind: %s\n %s", s.Name, s.Action, strings.Join(details, "\n "))
}

func (p *Pipeline) Validate() error {
// collect all steps from all resourcegroups and fail if there are duplicates
stepMap := make(map[string]*Step)
for _, rg := range p.ResourceGroups {
for _, step := range rg.Steps {
if _, ok := stepMap[step.Name]; ok {
return fmt.Errorf("duplicate step name %q", step.Name)
}
stepMap[step.Name] = step
}
}

// validate dependsOn for a step exists
for _, step := range stepMap {
for _, dep := range step.DependsOn {
if _, ok := stepMap[dep]; !ok {
return fmt.Errorf("invalid dependency on step %s: dependency %s does not exist", step.Name, dep)
}
}
}

// todo check for circular dependencies

// validate resource groups
for _, rg := range p.ResourceGroups {
err := rg.Validate()
if err != nil {
return err
}
}
return nil
}

func (rg *ResourceGroup) Validate() error {
if rg.Name == "" {
return fmt.Errorf("resource group name is required")
}
if rg.Subscription == "" {
return fmt.Errorf("subscription is required")
}
return nil
}

func getInputValues(configuredVariables []Variable, inputs map[string]output) (map[string]any, error) {
values := make(map[string]any)
for _, i := range configuredVariables {
Expand Down
Loading

0 comments on commit c84097c

Please sign in to comment.