Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

pipeline jsonschema #930

Merged
merged 3 commits into from
Dec 5, 2024
Merged

pipeline jsonschema #930

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
45b3907
pipeline jsonschema
geoberle Dec 5, 2024
926c6c3
adapt test
geoberle Dec 5, 2024
35de620
schema loading tests
geoberle Dec 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions backend/pipeline.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
serviceGroup: Microsoft.Azure.ARO.Test
rolloutName: RP - Backend
$schema: "pipeline.schema.v1"
serviceGroup: Microsoft.Azure.ARO.HCP.RP.Backend
rolloutName: RP Backend Rollout
resourceGroups:
- name: {{ .svc.rg }}
subscription: {{ .svc.subscription }}
Expand Down
16 changes: 10 additions & 6 deletions dev-infrastructure/mgmt-pipeline.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
serviceGroup: Microsoft.Azure.ARO.Test
$schema: "pipeline.schema.v1"
serviceGroup: Microsoft.Azure.ARO.HCP.Management.Infra
rolloutName: Management Cluster Rollout
resourceGroups:
- name: {{ .svc.rg }}
Expand All @@ -16,14 +17,17 @@ resourceGroups:
action: ARM
template: templates/mgmt-cluster.bicep
parameters: configurations/mgmt-cluster.tmpl.bicepparam
inputs:
- name: clusterServiceMIResourceId
step: regionOutput
output: cs.msi.resourceID
variables:
- name: mgmt.clusterServiceResourceId
input:
step: regionOutput
name: cs
dependsOn:
- regionOutput
- name: enable-metrics
action: Shell
command: scripts/enable-aks-metrics.sh
env:
variables:
- name: RESOURCEGROUP
configRef: mgmt.rg
- name: AKS_NAME
Expand Down
1 change: 1 addition & 0 deletions dev-infrastructure/region-pipeline.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
$schema: "pipeline.schema.v1"
serviceGroup: Microsoft.Azure.ARO.HCP.Region
rolloutName: Region Rollout
resourceGroups:
Expand Down
3 changes: 2 additions & 1 deletion dev-infrastructure/svc-pipeline.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
$schema: "pipeline.schema.v1"
serviceGroup: Microsoft.Azure.ARO.HCP.Service.Infra
rolloutName: Service Cluster Rollout
resourceGroups:
Expand All @@ -12,7 +13,7 @@ resourceGroups:
- name: enable-metrics
action: Shell
command: scripts/enable-aks-metrics.sh
env:
variables:
- name: RESOURCEGROUP
configRef: svc.rg
- name: AKS_NAME
Expand Down
6 changes: 1 addition & 5 deletions dev-infrastructure/templates/outputs/region.bicep
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,4 @@ resource csMSI 'Microsoft.ManagedIdentity/userAssignedIdentities@2023-01-31' = {
location: resourceGroup().location
}

output cs object = {
msi: {
resourceID: csMSI.id
}
}
output cs string = csMSI.id
5 changes: 3 additions & 2 deletions frontend/pipeline.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
serviceGroup: Microsoft.Azure.ARO.Test
rolloutName: RP - Frontend
$schema: "pipeline.schema.v1"
serviceGroup: Microsoft.Azure.ARO.HCP.RP.Frontend
rolloutName: RP Frontend Rollout
resourceGroups:
- name: {{ .svc.rg }}
subscription: {{ .svc.subscription }}
Expand Down
6 changes: 4 additions & 2 deletions tooling/templatize/internal/end2end/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package testutil
import (
"context"
"os"
"path/filepath"
"testing"

"gotest.tools/v3/assert"
Expand Down Expand Up @@ -134,14 +135,15 @@ func TestE2EShell(t *testing.T) {
t.Skip("Skipping end-to-end tests")
}

tmpDir := t.TempDir()
tmpDir, err := filepath.EvalSymlinks(t.TempDir())
geoberle marked this conversation as resolved.
Show resolved Hide resolved
assert.NilError(t, err)

e2eImpl := newE2E(tmpDir)

e2eImpl.AddStep(pipeline.Step{
Name: "readInput",
Action: "Shell",
Command: "/usr/bin/echo ${PWD} > env.txt",
Command: "/bin/echo ${PWD} > env.txt",
})

persistAndRun(t, &e2eImpl)
Expand Down
228 changes: 228 additions & 0 deletions tooling/templatize/pkg/pipeline/pipeline.schema.v1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,228 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "pipeline.schema.v1",
"type": "object",
"definitions": {
"variable": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"input": {
"type": "object",
"additionalProperties": false,
"properties": {
"step": {
"type": "string"
},
"name": {
"type": "string"
}
},
"required": [
"step",
"name"
]
},
"configRef": {
"type": "string"
},
"value": {
"type": "string"
}
},
"oneOf": [
{
"required": [
"name",
"input"
]
},
{
"required": [
"name",
"configRef"
]
},
{
"required": [
"name",
"value"
]
}
],
"required": [
"name"
]
}
},
"properties": {
"serviceGroup": {
"type": "string"
},
"rolloutName": {
"type": "string"
},
"resourceGroups": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"subscription": {
"type": "string"
},
"steps": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"properties": {
"name": {
"type": "string"
},
"action": {
"type": "string",
"enum": ["ARM", "Shell"]
},
"template": {
"type": "string"
},
"parameters": {
"type": "string"
},
"deploymentLevel": {
"type": "string",
"enum": ["ResourceGroup", "Subscription"]
},
"command": {
"type": "string"
},
"variables": {
"type": "array",
"items": {
"$ref": "#/definitions/variable"
}
},
"dependsOn": {
"type": "array",
"items": {
"type": "string"
}
},
"dryRun": {
"type": "object"
}
},
"oneOf": [
{
"additionalProperties": false,
"properties": {
"name": {
"type": "string"
},
"action": {
"type": "string",
"enum": ["ARM"]
},
"template": {
"type": "string"
},
"parameters": {
"type": "string"
},
"variables": {
"type": "array",
"items": {
"$ref": "#/definitions/variable"
}
},
"dependsOn": {
"type": "array",
"items": {
"type": "string"
}
},
"deploymentLevel": {
"type": "string",
"enum": ["ResourceGroup", "Subscription"]
}
},
"required": [
"template",
"parameters"
]
},
{
"additionalProperties": false,
"properties": {
"name": {
"type": "string"
},
"action": {
"type": "string",
"enum": ["Shell"]
},
"command": {
"type": "string"
},
"variables": {
"type": "array",
"items": {
"$ref": "#/definitions/variable"
}
},
"dependsOn": {
"type": "array",
"items": {
"type": "string"
}
},
"dryRun": {
"type": "object",
"additionalProperties": false,
"properties": {
"command": {
"type": "string"
},
"variables": {
"type": "array",
"items": {
"$ref": "#/definitions/variable"
}
}
}
}
},
"required": [
"command"
]
}
],
"required": [
"name",
"action"
]
}
},
"aksCluster": {
"type": "string"
}
},
"required": [
"name",
"subscription",
"steps"
]
}
}
},
"required": [
"serviceGroup",
"rolloutName",
"resourceGroups"
]
}
49 changes: 6 additions & 43 deletions tooling/templatize/pkg/pipeline/run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@ func NewPipelineFromFile(pipelineFilePath string, vars config.Variables) (*Pipel
if err != nil {
return nil, fmt.Errorf("failed to preprocess pipeline file %w", err)
}

err = ValidatePipelineSchema(bytes)
if err != nil {
return nil, fmt.Errorf("failed to validate pipeline schema: %w", err)
}

absPath, err := filepath.Abs(pipelineFilePath)
if err != nil {
return nil, fmt.Errorf("failed to get absolute path for pipeline file %q: %w", pipelineFilePath, err)
Expand Down Expand Up @@ -199,49 +205,6 @@ func (s *Step) description() string {
return fmt.Sprintf("Step %s\n Kind: %s\n %s", s.Name, s.Action, strings.Join(details, "\n "))
}

func (p *Pipeline) Validate() error {
// collect all steps from all resourcegroups and fail if there are duplicates
stepMap := make(map[string]*Step)
for _, rg := range p.ResourceGroups {
for _, step := range rg.Steps {
if _, ok := stepMap[step.Name]; ok {
return fmt.Errorf("duplicate step name %q", step.Name)
}
stepMap[step.Name] = step
}
}

// validate dependsOn for a step exists
for _, step := range stepMap {
for _, dep := range step.DependsOn {
if _, ok := stepMap[dep]; !ok {
return fmt.Errorf("invalid dependency on step %s: dependency %s does not exist", step.Name, dep)
}
}
}

// todo check for circular dependencies

// validate resource groups
for _, rg := range p.ResourceGroups {
err := rg.Validate()
if err != nil {
return err
}
}
return nil
}

func (rg *ResourceGroup) Validate() error {
if rg.Name == "" {
return fmt.Errorf("resource group name is required")
}
if rg.Subscription == "" {
return fmt.Errorf("subscription is required")
}
return nil
}

func getInputValues(configuredVariables []Variable, inputs map[string]output) (map[string]any, error) {
values := make(map[string]any)
for _, i := range configuredVariables {
Expand Down
Loading
Loading