Merge pull request #1234 from equinor/feat-custom-health-checks-and-p…

…robes

* feat: Add Healtchchecks to RadixConfig

* sync RadixDeployment to KubeDeployment with Healtchchecks

* sync RadixApplication to RadixDeployment with Healtchchecks

* fix tests

* fix linting

* Add RA validation

* Start testing

* replace k8s types with radix, add pointers where optional

* bump chart

* fix correct error component name

* fix correct error component name

* simplify structures

* bump charts

* simpler validateProbe, remove unneeded tests

* fix typo

charts/radix-operator/Chart.yaml

@@ -1,7 +1,7 @@
 apiVersion: v2
 name: radix-operator
-version: 1.47.0
-appVersion: 1.67.0
+version: 1.48.0
+appVersion: 1.68.0
 kubeVersion: ">=1.24.0"
 description: Radix Operator
 keywords:

pkg/apis/deployment/jobschedulercomponent.go

@@ -21,6 +21,10 @@ func newJobSchedulerComponent(jobComponent *radixv1.RadixDeployJobComponent, rd
 	}
 }
 
+func (js *jobSchedulerComponent) GetHealthChecks() *radixv1.RadixHealthChecks {
+	return nil
+}
+
 func (js *jobSchedulerComponent) GetImage() string {
 	containerRegistry := os.Getenv(defaults.ContainerRegistryEnvironmentVariable)
 	radixJobScheduler := os.Getenv(defaults.OperatorRadixJobSchedulerEnvironmentVariable)

pkg/apis/deployment/kubedeployment.go

@@ -315,11 +315,17 @@ func (deploy *Deployment) setDesiredDeploymentProperties(ctx context.Context, de
 	}
 	desiredDeployment.Spec.Template.Spec.Containers[0].VolumeMounts = volumeMounts
 
-	readinessProbe, err := getReadinessProbeForComponent(deployComponent)
-	if err != nil {
-		return err
+	if hc := deployComponent.GetHealthChecks(); hc != nil {
+		desiredDeployment.Spec.Template.Spec.Containers[0].ReadinessProbe = hc.ReadinessProbe.MapToCoreProbe()
+		desiredDeployment.Spec.Template.Spec.Containers[0].LivenessProbe = hc.LivenessProbe.MapToCoreProbe()
+		desiredDeployment.Spec.Template.Spec.Containers[0].StartupProbe = hc.StartupProbe.MapToCoreProbe()
+	} else {
+		readinessProbe, err := getDefaultReadinessProbeForComponent(deployComponent)
+		if err != nil {
+			return err
+		}
+		desiredDeployment.Spec.Template.Spec.Containers[0].ReadinessProbe = readinessProbe
 	}
-	desiredDeployment.Spec.Template.Spec.Containers[0].ReadinessProbe = readinessProbe
 
 	environmentVariables, err := GetEnvironmentVariablesForRadixOperator(ctx, deploy.kubeutil, appName, deploy.radixDeployment, deployComponent)
 	if err != nil {
@@ -449,7 +455,7 @@ func (deploy *Deployment) isEligibleForGarbageCollectComponent(componentName Rad
 	return componentType != commonComponent.GetType()
 }
 
-func getReadinessProbeForComponent(component v1.RadixCommonDeployComponent) (*corev1.Probe, error) {
+func getDefaultReadinessProbeForComponent(component v1.RadixCommonDeployComponent) (*corev1.Probe, error) {
 	if len(component.GetPorts()) == 0 {
 		return nil, nil
 	}

pkg/apis/deployment/kubedeployment_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/equinor/radix-operator/pkg/apis/test"
 	"github.com/equinor/radix-operator/pkg/apis/utils"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -22,15 +23,15 @@ func teardownReadinessProbe() {
 func TestGetReadinessProbe_MissingDefaultEnvVars(t *testing.T) {
 	teardownReadinessProbe()
 
-	probe, err := getReadinessProbeForComponent(&v1.RadixDeployComponent{Ports: []v1.ComponentPort{{Name: "http", Port: int32(80)}}})
+	probe, err := getDefaultReadinessProbeForComponent(&v1.RadixDeployComponent{Ports: []v1.ComponentPort{{Name: "http", Port: int32(80)}}})
 	assert.Error(t, err)
 	assert.Nil(t, probe)
 }
 
 func TestGetReadinessProbe_Custom(t *testing.T) {
 	test.SetRequiredEnvironmentVariables()
 
-	probe, err := getReadinessProbeForComponent(&v1.RadixDeployComponent{Ports: []v1.ComponentPort{{Name: "http", Port: int32(5000)}}})
+	probe, err := getDefaultReadinessProbeForComponent(&v1.RadixDeployComponent{Ports: []v1.ComponentPort{{Name: "http", Port: int32(5000)}}})
 	assert.Nil(t, err)
 
 	assert.Equal(t, int32(5), probe.InitialDelaySeconds)
@@ -39,6 +40,57 @@ func TestGetReadinessProbe_Custom(t *testing.T) {
 
 	teardownReadinessProbe()
 }
+func TestComponentWithoutCustomHealthChecks(t *testing.T) {
+	tu, client, kubeUtil, radixclient, kedaClient, prometheusclient, _, certClient := SetupTest(t)
+	rd, _ := ApplyDeploymentWithSync(tu, client, kubeUtil, radixclient, kedaClient, prometheusclient, certClient,
+		utils.ARadixDeployment().
+			WithComponents(utils.NewDeployComponentBuilder().
+				WithName("comp1")).
+			WithAppName("any-app").
+			WithEnvironment("test"))
+
+	component := rd.GetComponentByName("comp1")
+	assert.Nil(t, component.HealthChecks)
+}
+func TestComponentWithCustomHealthChecks(t *testing.T) {
+	tu, client, kubeUtil, radixclient, kedaClient, prometheusclient, _, certClient := SetupTest(t)
+	createProbe := func(handler v1.RadixProbeHandler, seconds int32) *v1.RadixProbe {
+		return &v1.RadixProbe{
+			RadixProbeHandler:   handler,
+			InitialDelaySeconds: seconds,
+			TimeoutSeconds:      seconds + 1,
+			PeriodSeconds:       seconds + 2,
+			SuccessThreshold:    seconds + 3,
+			FailureThreshold:    seconds + 4,
+			// TerminationGracePeriodSeconds: pointers.Ptr(int64(seconds + 5)),
+		}
+	}
+
+	readynessProbe := createProbe(v1.RadixProbeHandler{HTTPGet: &v1.RadixProbeHTTPGetAction{
+		Port: 5000,
+	}}, 10)
+
+	livenessProbe := createProbe(v1.RadixProbeHandler{TCPSocket: &v1.RadixProbeTCPSocketAction{
+		Port: 5000,
+	}}, 20)
+	startuProbe := createProbe(v1.RadixProbeHandler{Exec: &v1.RadixProbeExecAction{
+		Command: []string{"echo", "hello"},
+	}}, 30)
+
+	rd, _ := ApplyDeploymentWithSync(tu, client, kubeUtil, radixclient, kedaClient, prometheusclient, certClient,
+		utils.ARadixDeployment().
+			WithComponents(utils.NewDeployComponentBuilder().
+				WithName("comp1").
+				WithHealthChecks(startuProbe, readynessProbe, livenessProbe)).
+			WithAppName("any-app").
+			WithEnvironment("test"))
+
+	component := rd.GetComponentByName("comp1")
+	require.NotNil(t, component.HealthChecks)
+	assert.Equal(t, readynessProbe, component.HealthChecks.ReadinessProbe)
+	assert.Equal(t, livenessProbe, component.HealthChecks.LivenessProbe)
+	assert.Equal(t, startuProbe, component.HealthChecks.StartupProbe)
+}
 
 func Test_UpdateResourcesInDeployment(t *testing.T) {
 	origRequests := map[string]string{"cpu": "10m", "memory": "100M"}

pkg/apis/deployment/radixcomponent.go

@@ -74,6 +74,7 @@ func GetRadixComponentsForEnv(ctx context.Context, radixApplication *radixv1.Rad
 		deployComponent.AlwaysPullImageOnDeploy = getRadixComponentAlwaysPullImageOnDeployFlag(&radixComponent, environmentSpecificConfig)
 		deployComponent.ExternalDNS = getExternalDNSAliasForComponentEnvironment(radixApplication, componentName, env)
 		deployComponent.SecretRefs = getRadixCommonComponentRadixSecretRefs(&radixComponent, environmentSpecificConfig)
+		deployComponent.HealthChecks = getRadixCommonComponentHealthChecks(&radixComponent, environmentSpecificConfig)
 		deployComponent.PublicPort = getRadixComponentPort(&radixComponent)
 		deployComponent.Authentication = auth
 		deployComponent.Identity = identity
@@ -93,6 +94,34 @@ func GetRadixComponentsForEnv(ctx context.Context, radixApplication *radixv1.Rad
 	return deployComponents, nil
 }
 
+func getRadixCommonComponentHealthChecks(r *radixv1.RadixComponent, config *radixv1.RadixEnvironmentConfig) *radixv1.RadixHealthChecks {
+	if r.HealthChecks == nil && (config == nil || config.HealthChecks == nil) {
+		return nil
+	}
+	hc := &radixv1.RadixHealthChecks{}
+	if r.HealthChecks != nil {
+		hc.StartupProbe = r.HealthChecks.StartupProbe.DeepCopy()
+		hc.ReadinessProbe = r.HealthChecks.ReadinessProbe.DeepCopy()
+		hc.LivenessProbe = r.HealthChecks.LivenessProbe.DeepCopy()
+	}
+
+	if config == nil || config.HealthChecks == nil {
+		return hc
+	}
+
+	if config.HealthChecks.ReadinessProbe != nil {
+		hc.ReadinessProbe = config.HealthChecks.ReadinessProbe.DeepCopy()
+	}
+	if config.HealthChecks.LivenessProbe != nil {
+		hc.LivenessProbe = config.HealthChecks.LivenessProbe.DeepCopy()
+	}
+	if config.HealthChecks.StartupProbe != nil {
+		hc.StartupProbe = config.HealthChecks.StartupProbe.DeepCopy()
+	}
+
+	return hc
+}
+
 func getRadixComponentNetwork(component *radixv1.RadixComponent, environmentConfig *radixv1.RadixEnvironmentConfig) (*radixv1.Network, error) {
 	var dst *radixv1.Network
 	if component.Network != nil {

pkg/apis/deployment/radixcomponent_test.go

@@ -1048,6 +1048,87 @@ func Test_GetRadixComponents_Monitoring(t *testing.T) {
 	}
 }
 
+func Test_GetRadixComponents_CustomHealthChecks(t *testing.T) {
+	createProbe := func(handler radixv1.RadixProbeHandler, seconds int32) *radixv1.RadixProbe {
+		return &radixv1.RadixProbe{
+			RadixProbeHandler:   handler,
+			InitialDelaySeconds: seconds,
+			TimeoutSeconds:      seconds + 1,
+			PeriodSeconds:       seconds + 2,
+			SuccessThreshold:    seconds + 3,
+			FailureThreshold:    seconds + 4,
+			// TerminationGracePeriodSeconds: pointers.Ptr(int64(seconds + 5)),
+		}
+	}
+
+	httpProbe := radixv1.RadixProbeHandler{HTTPGet: &radixv1.RadixProbeHTTPGetAction{Port: 5000, Path: "/healthz", Scheme: corev1.URISchemeHTTP}}
+	execProbe := radixv1.RadixProbeHandler{Exec: &radixv1.RadixProbeExecAction{Command: []string{"/bin/sh", "-c", "/healthz /healthz"}}}
+	tcpProbe := radixv1.RadixProbeHandler{TCPSocket: &radixv1.RadixProbeTCPSocketAction{Port: 8000}}
+
+	testCases := []struct {
+		description      string
+		compHealthChecks *radixv1.RadixHealthChecks
+		envHealthChecks  *radixv1.RadixHealthChecks
+
+		expectedHealthChecks *radixv1.RadixHealthChecks
+	}{
+		{"No configuration set results in default readieness probe", nil, nil, nil},
+		{
+			description:          "component has healthchecks, no env config",
+			compHealthChecks:     &radixv1.RadixHealthChecks{LivenessProbe: createProbe(tcpProbe, 30), ReadinessProbe: createProbe(execProbe, 10), StartupProbe: createProbe(httpProbe, 20)},
+			expectedHealthChecks: &radixv1.RadixHealthChecks{LivenessProbe: createProbe(tcpProbe, 30), ReadinessProbe: createProbe(execProbe, 10), StartupProbe: createProbe(httpProbe, 20)},
+		},
+		{
+			"Env healthchecks, no component healthchecks",
+			nil,
+			&radixv1.RadixHealthChecks{LivenessProbe: createProbe(tcpProbe, 1), ReadinessProbe: createProbe(execProbe, 10), StartupProbe: createProbe(httpProbe, 20)},
+			&radixv1.RadixHealthChecks{LivenessProbe: createProbe(tcpProbe, 1), ReadinessProbe: createProbe(execProbe, 10), StartupProbe: createProbe(httpProbe, 20)},
+		},
+		{
+			"Env healthchecks, component healthchecks, env overrides comp",
+			&radixv1.RadixHealthChecks{LivenessProbe: createProbe(execProbe, 30), ReadinessProbe: createProbe(httpProbe, 10), StartupProbe: createProbe(tcpProbe, 20)},
+			&radixv1.RadixHealthChecks{LivenessProbe: createProbe(tcpProbe, 1), ReadinessProbe: createProbe(execProbe, 40), StartupProbe: createProbe(httpProbe, 20)},
+			&radixv1.RadixHealthChecks{LivenessProbe: createProbe(tcpProbe, 1), ReadinessProbe: createProbe(execProbe, 40), StartupProbe: createProbe(httpProbe, 20)},
+		},
+		{
+			"Env healthchecks, component healthchecks, env merges comp",
+			&radixv1.RadixHealthChecks{ReadinessProbe: createProbe(httpProbe, 10), StartupProbe: createProbe(tcpProbe, 20)},
+			&radixv1.RadixHealthChecks{LivenessProbe: createProbe(tcpProbe, 1), ReadinessProbe: createProbe(execProbe, 10)},
+			&radixv1.RadixHealthChecks{LivenessProbe: createProbe(tcpProbe, 1), ReadinessProbe: createProbe(execProbe, 10), StartupProbe: createProbe(tcpProbe, 20)},
+		},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.description, func(t *testing.T) {
+			envConfig := utils.NewComponentEnvironmentBuilder().WithEnvironment("dev")
+			if testCase.envHealthChecks != nil {
+				envConfig.WithHealthChecks(testCase.envHealthChecks.StartupProbe, testCase.envHealthChecks.ReadinessProbe, testCase.envHealthChecks.LivenessProbe)
+			}
+			compConfig := utils.NewApplicationComponentBuilder().WithName("comp").WithEnvironmentConfig(envConfig)
+			if testCase.compHealthChecks != nil {
+				compConfig.WithHealthChecks(testCase.compHealthChecks.StartupProbe, testCase.compHealthChecks.ReadinessProbe, testCase.compHealthChecks.LivenessProbe)
+			}
+			raBuilder := utils.ARadixApplication().WithComponents(compConfig)
+			ra := raBuilder.BuildRA()
+
+			deployComponents, err := GetRadixComponentsForEnv(context.Background(), ra, nil, "dev", make(pipeline.DeployComponentImages), make(radixv1.EnvVarsMap), nil)
+			require.NoError(t, err)
+			require.Len(t, deployComponents, 1)
+
+			if testCase.expectedHealthChecks == nil {
+				assert.Nil(t, deployComponents[0].HealthChecks)
+			} else {
+				require.NotNil(t, deployComponents[0].HealthChecks)
+				assert.Equal(t, testCase.expectedHealthChecks.ReadinessProbe, deployComponents[0].HealthChecks.ReadinessProbe)
+				assert.Equal(t, testCase.expectedHealthChecks.LivenessProbe, deployComponents[0].HealthChecks.LivenessProbe)
+				assert.Equal(t, testCase.expectedHealthChecks.StartupProbe, deployComponents[0].HealthChecks.StartupProbe)
+			}
+
+		})
+	}
+
+}
+
 func Test_GetRadixComponents_ReplicasOverride(t *testing.T) {
 	componentName := "comp"
 	env := "dev"

pkg/apis/radix/v1/radixapptypes.go

@@ -346,6 +346,11 @@ type RadixComponent struct {
 	// +optional
 	DockerfileName string `json:"dockerfileName,omitempty"`
 
+	// HealthChecks can tell Radix if your application is ready to receive traffic.
+	// Defaults to a TCP check against your first listed port.
+	// If any healthchecks are defined, no defaults will be added and you should add your own readinessProbe.
+	HealthChecks *RadixHealthChecks `json:"healthChecks,omitempty"`
+
 	// Name of an existing container image to use when running the component.
 	// More info: https://www.radix.equinor.com/references/reference-radix-config/#image
 	// +optional
@@ -489,6 +494,11 @@ type RadixEnvironmentConfig struct {
 	// +optional
 	Image string `json:"image,omitempty"`
 
+	// HealthChecks can tell Radix if your application is ready to receive traffic.
+	// Defaults to a TCP check against your first listed port.
+	// If any healthchecks are defined, no defaults will be added and you should add your own readinessProbe.
+	HealthChecks *RadixHealthChecks `json:"healthChecks,omitempty"`
+
 	// Number of desired replicas.
 	// More info: https://www.radix.equinor.com/references/reference-radix-config/#replicas
 	// +kubebuilder:validation:Minimum=0

pkg/apis/radix/v1/radixdeploytypes.go

@@ -128,6 +128,7 @@ type RadixDeployComponent struct {
 	ExternalDNS          []RadixDeployExternalDNS `json:"externalDNS,omitempty"`
 	// Deprecated: For backward compatibility we must still support this field. New code should use ExternalDNS instead.
 	DNSExternalAlias        []string                `json:"dnsExternalAlias,omitempty"`
+	HealthChecks            *RadixHealthChecks      `json:"healthChecks,omitempty"`
 	Monitoring              bool                    `json:"monitoring"`
 	MonitoringConfig        MonitoringConfig        `json:"monitoringConfig,omitempty"`
 	Resources               ResourceRequirements    `json:"resources,omitempty"`
@@ -142,6 +143,19 @@ type RadixDeployComponent struct {
 	Network                 *Network                `json:"network,omitempty"`
 }
 
+func (deployComponent *RadixDeployComponent) GetHealthChecks() *RadixHealthChecks {
+	if deployComponent.HealthChecks == nil {
+		return nil
+	}
+	if deployComponent.HealthChecks.ReadinessProbe == nil &&
+		deployComponent.HealthChecks.LivenessProbe == nil &&
+		deployComponent.HealthChecks.StartupProbe == nil {
+		return nil
+	}
+
+	return deployComponent.HealthChecks
+}
+
 func (deployComponent *RadixDeployComponent) GetName() string {
 	return deployComponent.Name
 }
@@ -429,6 +443,10 @@ type RadixDeployJobComponent struct {
 	FailurePolicy *RadixJobComponentFailurePolicy `json:"failurePolicy,omitempty"`
 }
 
+func (r *RadixDeployJobComponent) GetHealthChecks() *RadixHealthChecks {
+	return nil
+}
+
 type RadixComponentType string
 
 const (
@@ -467,6 +485,7 @@ type RadixCommonDeployComponent interface {
 	GetReadOnlyFileSystem() *bool
 	GetRuntime() *Runtime
 	GetNetwork() *Network
+	GetHealthChecks() *RadixHealthChecks
 }
 
 // RadixCommonDeployComponentFactory defines a common component factory