From d080786156f00e925e0a3128dd6256f5230a933d Mon Sep 17 00:00:00 2001 From: catttam Date: Wed, 20 Sep 2023 09:31:48 +0200 Subject: [PATCH 01/10] Added support to SGX --- pkg/types/service.go | 26 ++++++++++++++++++++++++++ pkg/types/service_test.go | 1 + 2 files changed, 27 insertions(+) diff --git a/pkg/types/service.go b/pkg/types/service.go index be4d99c6..1c4a18e0 100644 --- a/pkg/types/service.go +++ b/pkg/types/service.go @@ -143,6 +143,10 @@ type Service struct { // Optional. (default: false) EnableGPU bool `json:"enable_gpu"` + // EnableSGX parameter to use SGX plugin for security + // Optional. (default: false) + EnableSGX bool `json:"enable_sgx"` + // ImagePrefetch parameter to enable the image cache functionality // Optional. (default: false) ImagePrefetch bool `json:"image_prefetch"` @@ -289,6 +293,10 @@ func (service *Service) ToPodSpec(cfg *Config) (*v1.PodSpec, error) { // Add the required environment variables for the watchdog addWatchdogEnvVars(podSpec, cfg, service) + if service.EnableSGX { + setSecurityContext(*podSpec) + } + return podSpec, nil } @@ -327,6 +335,16 @@ func SetImagePullSecrets(secrets []string) []v1.LocalObjectReference { return objects } +func setSecurityContext(podSpec v1.PodSpec) { + ctx := v1.SecurityContext{ + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{"SYS_RAWIO"}, + }, + } + + podSpec.Containers[0].SecurityContext = &ctx +} + func createResources(service *Service) (v1.ResourceRequirements, error) { resources := v1.ResourceRequirements{ Limits: v1.ResourceList{}, @@ -356,6 +374,14 @@ func createResources(service *Service) (v1.ResourceRequirements, error) { resources.Limits["nvidia.com/gpu"] = gpu } + if service.EnableSGX { + sgx, err := resource.ParseQuantity("1") + if err != nil { + return resources, err + } + resources.Limits["sgx.intel.com/enclave"] = sgx + } + return resources, nil } diff --git a/pkg/types/service_test.go b/pkg/types/service_test.go index 05f2bcc4..20b5242a 100644 --- a/pkg/types/service_test.go +++ b/pkg/types/service_test.go @@ -215,6 +215,7 @@ cpu: "1.0" total_memory: "" total_cpu: "" enable_gpu: false +enable_sgx: false image_prefetch: false synchronous: min_scale: 0 From 04fa67779cbbe43576b9859209b16ab73fde433d Mon Sep 17 00:00:00 2001 From: catttam Date: Wed, 20 Sep 2023 13:47:42 +0200 Subject: [PATCH 02/10] Added SecurityContext flag to knative service --- pkg/backends/knative.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/backends/knative.go b/pkg/backends/knative.go index 49d06447..528078a7 100644 --- a/pkg/backends/knative.go +++ b/pkg/backends/knative.go @@ -310,6 +310,11 @@ func (kn *KnativeBackend) createKNServiceDefinition(service *types.Service) (*kn }, } + if service.EnableSGX { + knSvc.Spec.ConfigurationSpec.Template.ObjectMeta.Annotations["kubernetes.podspec-securitycontext"] = "enabled" + knSvc.Spec.ConfigurationSpec.Template.ObjectMeta.Annotations["kubernetes.containerspec-addcapabilities"] = "enabled" + } + return knSvc, nil } From 30477d23fdf169e2bc09a714d7c2983c1294a122 Mon Sep 17 00:00:00 2001 From: catttam Date: Wed, 11 Oct 2023 10:09:24 +0200 Subject: [PATCH 03/10] Added support to SCONE --- pkg/types/service.go | 18 ++++++++++++++++++ pkg/types/service_test.go | 1 + 2 files changed, 19 insertions(+) diff --git a/pkg/types/service.go b/pkg/types/service.go index be4d99c6..851d5d11 100644 --- a/pkg/types/service.go +++ b/pkg/types/service.go @@ -143,6 +143,10 @@ type Service struct { // Optional. (default: false) EnableGPU bool `json:"enable_gpu"` + // EnableSGX parameter to use the SCONE k8s plugin + // Optional. (default: false) + EnableSGX bool `json:"enable_sgx"` + // ImagePrefetch parameter to enable the image cache functionality // Optional. (default: false) ImagePrefetch bool `json:"image_prefetch"` @@ -286,6 +290,12 @@ func (service *Service) ToPodSpec(cfg *Config) (*v1.PodSpec, error) { }, } + if service.EnableSGX { + podSpec.Containers[0].SecurityContext.Capabilities = &v1.Capabilities{ + Add: []v1.Capability{"SYS_RAWIO"}, + } + } + // Add the required environment variables for the watchdog addWatchdogEnvVars(podSpec, cfg, service) @@ -356,6 +366,14 @@ func createResources(service *Service) (v1.ResourceRequirements, error) { resources.Limits["nvidia.com/gpu"] = gpu } + if service.EnableSGX { + sgx, err := resource.ParseQuantity("1") + if err != nil { + return resources, err + } + resources.Limits["sgx.intel.com/enclave"] = sgx + } + return resources, nil } diff --git a/pkg/types/service_test.go b/pkg/types/service_test.go index 05f2bcc4..20b5242a 100644 --- a/pkg/types/service_test.go +++ b/pkg/types/service_test.go @@ -215,6 +215,7 @@ cpu: "1.0" total_memory: "" total_cpu: "" enable_gpu: false +enable_sgx: false image_prefetch: false synchronous: min_scale: 0 From cb5620879385e824b51f3610128c30f2d482911f Mon Sep 17 00:00:00 2001 From: catttam Date: Fri, 13 Oct 2023 12:37:27 +0200 Subject: [PATCH 04/10] Fixed error enabling SGX plugin --- pkg/types/service.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/types/service.go b/pkg/types/service.go index ad62f920..9cc66dc4 100644 --- a/pkg/types/service.go +++ b/pkg/types/service.go @@ -291,8 +291,10 @@ func (service *Service) ToPodSpec(cfg *Config) (*v1.PodSpec, error) { } if service.EnableSGX { - podSpec.Containers[0].SecurityContext.Capabilities = &v1.Capabilities{ - Add: []v1.Capability{"SYS_RAWIO"}, + podSpec.Containers[0].SecurityContext = &v1.SecurityContext{ + Capabilities: &v1.Capabilities{ + Add: []v1.Capability{"SYS_RAWIO"}, + }, } } From 53a0f0b67cfa8c4a86d44c8857a60743b5c3fbbc Mon Sep 17 00:00:00 2001 From: catttam Date: Mon, 16 Oct 2023 10:41:36 +0200 Subject: [PATCH 05/10] Minor fix on service definition --- pkg/types/service.go | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/pkg/types/service.go b/pkg/types/service.go index 9cc66dc4..4a18b452 100644 --- a/pkg/types/service.go +++ b/pkg/types/service.go @@ -290,19 +290,11 @@ func (service *Service) ToPodSpec(cfg *Config) (*v1.PodSpec, error) { }, } - if service.EnableSGX { - podSpec.Containers[0].SecurityContext = &v1.SecurityContext{ - Capabilities: &v1.Capabilities{ - Add: []v1.Capability{"SYS_RAWIO"}, - }, - } - } - // Add the required environment variables for the watchdog addWatchdogEnvVars(podSpec, cfg, service) if service.EnableSGX { - setSecurityContext(*podSpec) + setSecurityContext(podSpec) } return podSpec, nil @@ -343,7 +335,7 @@ func SetImagePullSecrets(secrets []string) []v1.LocalObjectReference { return objects } -func setSecurityContext(podSpec v1.PodSpec) { +func setSecurityContext(podSpec *v1.PodSpec) { ctx := v1.SecurityContext{ Capabilities: &v1.Capabilities{ Add: []v1.Capability{"SYS_RAWIO"}, From c5e2fdd2e1b8fc2497cb6fede16508f0b0714962 Mon Sep 17 00:00:00 2001 From: catttam Date: Thu, 26 Oct 2023 09:44:04 +0200 Subject: [PATCH 06/10] Added support to SGX and GPU for exposed services --- pkg/types/service.go | 4 ++-- pkg/utils/expose.go | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/pkg/types/service.go b/pkg/types/service.go index 4a18b452..13f83180 100644 --- a/pkg/types/service.go +++ b/pkg/types/service.go @@ -294,7 +294,7 @@ func (service *Service) ToPodSpec(cfg *Config) (*v1.PodSpec, error) { addWatchdogEnvVars(podSpec, cfg, service) if service.EnableSGX { - setSecurityContext(podSpec) + SetSecurityContext(podSpec) } return podSpec, nil @@ -335,7 +335,7 @@ func SetImagePullSecrets(secrets []string) []v1.LocalObjectReference { return objects } -func setSecurityContext(podSpec *v1.PodSpec) { +func SetSecurityContext(podSpec *v1.PodSpec) { ctx := v1.SecurityContext{ Capabilities: &v1.Capabilities{ Add: []v1.Capability{"SYS_RAWIO"}, diff --git a/pkg/utils/expose.go b/pkg/utils/expose.go index 8bb14cf5..df4a7371 100644 --- a/pkg/utils/expose.go +++ b/pkg/utils/expose.go @@ -41,6 +41,8 @@ type Expose struct { MinScale int32 `default:"1"` Port int ` binding:"required" default:"80"` CpuThreshold int32 `default:"80"` + EnableSGX bool + EnableGPU bool } // / Main function that creates all the kubernetes components @@ -173,6 +175,7 @@ func getDeployment(e Expose) *apps.Deployment { }, Status: apps.DeploymentStatus{}, } + return deployment } @@ -232,6 +235,18 @@ func getPodTemplateSpec(e Expose) v1.PodTemplateSpec { Containers: []v1.Container{container}, }, } + + if e.EnableSGX { + types.SetSecurityContext(&template.Spec) + sgx, _ := resource.ParseQuantity("1") + container.Resources.Limits["sgx.intel.com/enclave"] = sgx + } + + if e.EnableGPU { + gpu, _ := resource.ParseQuantity("1") + container.Resources.Limits["nvidia.com/gpu"] = gpu + } + return template } From 83222e90af3288543a38ccf21bcc490146ce53f4 Mon Sep 17 00:00:00 2001 From: catttam Date: Thu, 26 Oct 2023 12:29:19 +0200 Subject: [PATCH 07/10] Fixed support for SGX on exposed services --- pkg/utils/expose.go | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/pkg/utils/expose.go b/pkg/utils/expose.go index df4a7371..8fdcbdb4 100644 --- a/pkg/utils/expose.go +++ b/pkg/utils/expose.go @@ -211,17 +211,7 @@ func getPodTemplateSpec(e Expose) v1.PodTemplateSpec { ContainerPort: int32(e.Port), } cores := resource.NewMilliQuantity(500, resource.DecimalSI) - var container v1.Container = v1.Container{ - Name: e.Name, - Image: e.Image, - Env: types.ConvertEnvVars(e.Variables), - Ports: []v1.ContainerPort{ports}, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - "cpu": *cores, - }, - }, - } + template := v1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Name: e.Name, @@ -232,19 +222,31 @@ func getPodTemplateSpec(e Expose) v1.PodTemplateSpec { }, Spec: v1.PodSpec{ InitContainers: []v1.Container{}, - Containers: []v1.Container{container}, + Containers: []v1.Container{ + { + Name: e.Name, + Image: e.Image, + Env: types.ConvertEnvVars(e.Variables), + Ports: []v1.ContainerPort{ports}, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "cpu": *cores, + }, + }, + }, + }, }, } if e.EnableSGX { types.SetSecurityContext(&template.Spec) sgx, _ := resource.ParseQuantity("1") - container.Resources.Limits["sgx.intel.com/enclave"] = sgx + template.Spec.Containers[0].Resources.Limits["sgx.intel.com/enclave"] = sgx } if e.EnableGPU { gpu, _ := resource.ParseQuantity("1") - container.Resources.Limits["nvidia.com/gpu"] = gpu + template.Spec.Containers[0].Resources.Limits["nvidia.com/gpu"] = gpu } return template From 99337187f6c945e19a8fbe8ddabd3847ae654d3c Mon Sep 17 00:00:00 2001 From: catttam Date: Tue, 31 Oct 2023 11:48:10 +0100 Subject: [PATCH 08/10] Fixed enable SGX on exposed services --- pkg/backends/k8s.go | 1 + pkg/utils/expose.go | 8 ++------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/backends/k8s.go b/pkg/backends/k8s.go index 4d89e38c..eeef1829 100644 --- a/pkg/backends/k8s.go +++ b/pkg/backends/k8s.go @@ -127,6 +127,7 @@ func (k *KubeBackend) CreateService(service types.Service) error { MaxScale: service.Expose.MaxScale, MinScale: service.Expose.MinScale, CpuThreshold: service.Expose.CpuThreshold, + EnableSGX: service.EnableSGX, } utils.CreateExpose(exposeConf, k.kubeClientset, *k.config) } diff --git a/pkg/utils/expose.go b/pkg/utils/expose.go index 8fdcbdb4..b602718c 100644 --- a/pkg/utils/expose.go +++ b/pkg/utils/expose.go @@ -42,7 +42,6 @@ type Expose struct { Port int ` binding:"required" default:"80"` CpuThreshold int32 `default:"80"` EnableSGX bool - EnableGPU bool } // / Main function that creates all the kubernetes components @@ -232,6 +231,8 @@ func getPodTemplateSpec(e Expose) v1.PodTemplateSpec { Requests: v1.ResourceList{ "cpu": *cores, }, + // Empty Limits list initialized in case enabling SGX is needed + Limits: v1.ResourceList{}, }, }, }, @@ -244,11 +245,6 @@ func getPodTemplateSpec(e Expose) v1.PodTemplateSpec { template.Spec.Containers[0].Resources.Limits["sgx.intel.com/enclave"] = sgx } - if e.EnableGPU { - gpu, _ := resource.ParseQuantity("1") - template.Spec.Containers[0].Resources.Limits["nvidia.com/gpu"] = gpu - } - return template } From 415fb7adea88bf87063ccb2273a9c7fea00fff57 Mon Sep 17 00:00:00 2001 From: catttam Date: Thu, 2 Nov 2023 16:29:45 +0100 Subject: [PATCH 09/10] Added debug logs for exposed services --- pkg/backends/k8s.go | 1 + pkg/backends/knative.go | 2 ++ pkg/utils/expose.go | 2 ++ 3 files changed, 5 insertions(+) diff --git a/pkg/backends/k8s.go b/pkg/backends/k8s.go index eeef1829..d583e0da 100644 --- a/pkg/backends/k8s.go +++ b/pkg/backends/k8s.go @@ -217,6 +217,7 @@ func (k *KubeBackend) UpdateService(service types.Service) error { MaxScale: service.Expose.MaxScale, MinScale: service.Expose.MinScale, CpuThreshold: service.Expose.CpuThreshold, + EnableSGX: service.EnableSGX, } utils.UpdateExpose(exposeConf, k.kubeClientset, *k.config) diff --git a/pkg/backends/knative.go b/pkg/backends/knative.go index 528078a7..3a5800fe 100644 --- a/pkg/backends/knative.go +++ b/pkg/backends/knative.go @@ -133,6 +133,7 @@ func (kn *KnativeBackend) CreateService(service types.Service) error { MaxScale: service.Expose.MaxScale, MinScale: service.Expose.MinScale, CpuThreshold: service.Expose.CpuThreshold, + EnableSGX: service.EnableSGX, } utils.CreateExpose(exposeConf, kn.kubeClientset, *kn.config) @@ -224,6 +225,7 @@ func (kn *KnativeBackend) UpdateService(service types.Service) error { MaxScale: service.Expose.MaxScale, MinScale: service.Expose.MinScale, CpuThreshold: service.Expose.CpuThreshold, + EnableSGX: service.EnableSGX, } utils.UpdateExpose(exposeConf, kn.kubeClientset, *kn.config) diff --git a/pkg/utils/expose.go b/pkg/utils/expose.go index b602718c..b5f05f9f 100644 --- a/pkg/utils/expose.go +++ b/pkg/utils/expose.go @@ -46,6 +46,7 @@ type Expose struct { // / Main function that creates all the kubernetes components func CreateExpose(expose Expose, kubeClientset kubernetes.Interface, cfg types.Config) error { + log.Printf("DEBUG: Creating exposed service: \n%v\n", expose) err := createDeployment(expose, kubeClientset) if err != nil { log.Printf("WARNING: %v\n", err) @@ -240,6 +241,7 @@ func getPodTemplateSpec(e Expose) v1.PodTemplateSpec { } if e.EnableSGX { + log.Printf("DEBUG: Enabling components to use SGX plugin\n") types.SetSecurityContext(&template.Spec) sgx, _ := resource.ParseQuantity("1") template.Spec.Containers[0].Resources.Limits["sgx.intel.com/enclave"] = sgx From af3f806bc411af06648a0c84ecb0f3d48484d8f7 Mon Sep 17 00:00:00 2001 From: catttam Date: Thu, 2 Nov 2023 17:21:27 +0100 Subject: [PATCH 10/10] Improved expose logs --- pkg/utils/expose.go | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/pkg/utils/expose.go b/pkg/utils/expose.go index b5f05f9f..6bdecf33 100644 --- a/pkg/utils/expose.go +++ b/pkg/utils/expose.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "log" + "os" "github.com/grycap/oscar/v2/pkg/types" apps "k8s.io/api/apps/v1" @@ -44,22 +45,25 @@ type Expose struct { EnableSGX bool } +// Custom logger +var ExposeLogger = log.New(os.Stdout, "[EXPOSED-SERVICE] ", log.Flags()) + // / Main function that creates all the kubernetes components func CreateExpose(expose Expose, kubeClientset kubernetes.Interface, cfg types.Config) error { - log.Printf("DEBUG: Creating exposed service: \n%v\n", expose) + ExposeLogger.Printf("DEBUG: Creating exposed service: \n%v\n", expose) err := createDeployment(expose, kubeClientset) if err != nil { - log.Printf("WARNING: %v\n", err) + ExposeLogger.Printf("WARNING: %v\n", err) return err } err = createService(expose, kubeClientset) if err != nil { - log.Printf("WARNING: %v\n", err) + ExposeLogger.Printf("WARNING: %v\n", err) return err } err = createIngress(expose, kubeClientset, cfg) if err != nil { - log.Printf("WARNING: %v\n", err) + ExposeLogger.Printf("WARNING: %v\n", err) return err } return nil @@ -69,17 +73,17 @@ func CreateExpose(expose Expose, kubeClientset kubernetes.Interface, cfg types.C func DeleteExpose(expose Expose, kubeClientset kubernetes.Interface) error { err := deleteDeployment(expose, kubeClientset) if err != nil { - log.Printf("WARNING: %v\n", err) + ExposeLogger.Printf("WARNING: %v\n", err) return err } err = deleteService(expose, kubeClientset) if err != nil { - log.Printf("WARNING: %v\n", err) + ExposeLogger.Printf("WARNING: %v\n", err) return err } err = deleteIngress(expose, kubeClientset) if err != nil { - log.Printf("WARNING: %v\n", err) + ExposeLogger.Printf("WARNING: %v\n", err) return err } return nil @@ -102,12 +106,12 @@ func UpdateExpose(expose Expose, kubeClientset kubernetes.Interface, cfg types.C } err := updateDeployment(expose, kubeClientset) if err != nil { - log.Printf("WARNING: %v\n", err) + ExposeLogger.Printf("WARNING: %v\n", err) return err } err2 := updateService(expose, kubeClientset) if err2 != nil { - log.Printf("WARNING: %v\n", err2) + ExposeLogger.Printf("WARNING: %v\n", err2) return err2 } return nil @@ -121,15 +125,15 @@ func ListExpose(expose Expose, kubeClientset kubernetes.Interface) error { services, err2 := listServices(expose, kubeClientset) ingress, err3 := listIngress(expose, kubeClientset) if err != nil { - log.Printf("WARNING: %v\n", err) + ExposeLogger.Printf("WARNING: %v\n", err) return err } if err2 != nil { - log.Printf("WARNING: %v\n", err2) + ExposeLogger.Printf("WARNING: %v\n", err2) return err } if err3 != nil { - log.Printf("WARNING: %v\n", err3) + ExposeLogger.Printf("WARNING: %v\n", err3) return err } fmt.Println(deploy, hpa, services, ingress) @@ -241,7 +245,7 @@ func getPodTemplateSpec(e Expose) v1.PodTemplateSpec { } if e.EnableSGX { - log.Printf("DEBUG: Enabling components to use SGX plugin\n") + ExposeLogger.Printf("DEBUG: Enabling components to use SGX plugin\n") types.SetSecurityContext(&template.Spec) sgx, _ := resource.ParseQuantity("1") template.Spec.Containers[0].Resources.Limits["sgx.intel.com/enclave"] = sgx