diff --git a/container/README.md b/container/README.md index b521c46..7a67b4f 100644 --- a/container/README.md +++ b/container/README.md @@ -4,7 +4,7 @@ There are several docker image files in the sub directories of current directory | Sub directory | Image name | Description | |---|---|---| -| ccnp-device-plugin | ccnp-device-plugin | CCNP Device plugin | +| ccnp-webhook | ccnp-webhook | CCNP webhook | | ccnp-server | ccnp-server | CCNP server | | ccnp-example | ccnp-example | Example image of getting eventlog and measurement using CCNP SDK | | pccs | pccs | PCCS docker image for Intel® TDX remote attestation. Not required for CCNP usage.| @@ -57,5 +57,5 @@ After the script is running successfully, it's supposed to see corresponding CCN $ sudo docker images ccnp-example ccnp-server -ccnp-device-plugin +ccnp-webhook ``` diff --git a/container/ccnp-device-plugin/Dockerfile b/container/ccnp-device-plugin/Dockerfile deleted file mode 100644 index d3d8343..0000000 --- a/container/ccnp-device-plugin/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -From golang:1.20-alpine3.18 AS builder - -RUN apk update \ - && apk add --no-cache protoc make - -WORKDIR /usr/local/go/src/github.com/ccnp-device-plugin/ -COPY device-plugin/ccnp-device-plugin ./ -RUN make - - -From alpine:3.18.5 - -WORKDIR /bin -COPY --from=builder /usr/local/go/src/github.com/ccnp-device-plugin/build/ccnp-device-plugin ./ - -CMD ["/bin/ccnp-device-plugin"] - diff --git a/deployment/README.md b/deployment/README.md index 303b478..0c64aa4 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -96,7 +96,7 @@ After the script is executed successfully, it's supposed to see below docker ima $ sudo docker images ccnp-example ccnp-server -ccnp-device-plugin +ccnp-webhook ``` ## Setup QGS and PCCS on the Host diff --git a/deployment/kubernetes/README.md b/deployment/kubernetes/README.md index 22adab7..d355885 100644 --- a/deployment/kubernetes/README.md +++ b/deployment/kubernetes/README.md @@ -52,18 +52,16 @@ $ sudo ./deploy-ccnp.sh -r -g -d ``` -After it's successful, you should see helm release `ccnp-device-plugin` and 3 DaemonSets in namespace `ccnp`. +After it's successful, you should see DaemonSet in namespace `ccnp`. ``` -$ sudo helm list -NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION -ccnp-device-plugin default 1 2023-12-27 08:12:05.814766198 +0000 UTC deployed ccnp-device-plugin-0.1.0 latest $ sudo kubectl get ds -n ccnp NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE SELECTOR AGE ccnp-server 1 1 1 1 1 intel.feature.node.kubernetes.io/tdx-guest=enabled 24h $ sudo kubectl get pods -n ccnp -NAME READY STATUS RESTARTS AGE -ccnp-server-mqfjx 1/1 Running 2 (39s ago) 24h +NAME READY STATUS RESTARTS AGE +ccnp-webhook-74f88647bd-d4hmk 1/1 Running 0 91m +ccnp-server-h7t46 1/1 Running 0 90m ``` ## CCNP Usage Example diff --git a/device-plugin/ccnp-device-plugin/Makefile b/device-plugin/ccnp-device-plugin/Makefile deleted file mode 100644 index 3fd5fed..0000000 --- a/device-plugin/ccnp-device-plugin/Makefile +++ /dev/null @@ -1,14 +0,0 @@ - -export GO111MODULE=on - -.PHONY: build deploy - -build: - CGO_ENABLED=0 GOOS=linux - @go build -a -installsuffix cgo -o build/ccnp-device-plugin cmd/server/app.go - -deploy: - helm install ccnp-device-plugin deploy/helm/ccnp-device-plugin - -clean: - @rm -f build diff --git a/device-plugin/ccnp-device-plugin/README.md b/device-plugin/ccnp-device-plugin/README.md deleted file mode 100644 index bb7e9b7..0000000 --- a/device-plugin/ccnp-device-plugin/README.md +++ /dev/null @@ -1,150 +0,0 @@ -# CCNP device plugin - -The CCNP device plugin is based on Kubernetes plugin framework to expose host's TEE devices and other required resources to PODs. -This will enable the services in the PODs to be able to communicate to the device for quote, measurements etc. -And will also facilitate other CCNP requirements like mount certain directories to CCNP service PODs and workload PODs. - -## Introduction - -Currently, the CCNP device plugin has following capabilities: -- expose tdx guest device node in TDVM to PODs -- mount Unix Domain Socket dir /run/ccnp/uds into CCNP daemonset PODs and workload PODs to enable intra-node communication - -The CCNP device plugin will respond to following resource request defined in POD definition yaml: -``` - resources: - limits: - tdx.intel.com/tdx-guest: 1 -``` - -## Installation - -### Prerequisite -The CCNP device plugin need to deploy on VM nodes with guest TEE devices(currently only TDX guest device is supported). So the deployment -of the plugin daemonset is based on the node label set by [Node Feature Discovery](https://github.com/kubernetes-sigs/node-feature-discovery/). -So we need to install the NFD and corresponding label rules. - -1. Setup following udev rule to enable other user in the node to read or write to tdx guest device node - - - Check TDX device node - ``` - $ ls /dev/tdx* - ``` - - - If the device node is `/dev/tdx-guest`, add below content in file /etc/udev/rules.d/90-tdx.rules - ``` - SUBSYSTEM=="misc",KERNEL=="tdx-guest",MODE="0666" - ``` - - - If the device node is `/dev/tdx_guest`, add below content in file /etc/udev/rules.d/90-tdx.rules - ``` - SUBSYSTEM=="misc",KERNEL=="tdx_guest",MODE="0666" - ``` - - - After adding the rule, you can restart the node or run following command to trigger the update: - ``` - $ sudo udevadm trigger - ``` - -2. Prepare the shared Unix Domain Socket directory to be mounted to both ccnp service pods and workload pods - ``` - $ sudo mkdir -p /run/ccnp/uds - $ sudo chmod 0757 /run/ccnp/uds - - # Add below content in file /usr/lib/tmpfiles.d/ccnp.conf: - D /run/ccnp/uds 0757 - - - - ``` - -3. Deploy NFD - - From NFD [v0.14](https://github.com/kubernetes-sigs/node-feature-discovery/releases/tag/v0.14.0) release on, it supports Intel TDX guest detection. Please use NFD v0.14.0 and later releases. - - ``` - $ kubectl apply -k https://github.com/kubernetes-sigs/node-feature-discovery/deployment/overlays/default?ref=v0.14 - ``` - -4. Deploy NFD label rules - - ``` - $ kubectl apply -f device-plugin/ccnp-device-plugin/deploy/node-feature-rules.yaml - ``` - - After deployment, following label can be found in the VM node: - - ``` - $ kubectl get node -o json | jq .items[].metadata.labels | grep tdx-guest - "intel.feature.node.kubernetes.io/tdx-guest": "enabled", - ``` - Above label can be used as node selector by CCNP device plugin daemonset and CCNP services daemonset. - - -### Build docker image -The Dockerfile for the service can be found under container/ccnp-device-plugin directory. -Use the following command to build the image under confidential-cloud-native-primitives directory: - -``` -$ docker build -t ccnp-device-plugin: -f container/ccnp-device-plugin/Dockerfile . -``` - -> Note: if you are using containerd as the default runtime for kubernetes, don't forget to use the following commands to import the image into containerd first: -``` -$ docker save -o ccnp-device-plugin.tar ccnp-device-plugin: -$ ctr -n=k8s.io image import ccnp-device-plugin.tar -``` - -### Deploy as DaemonSet -Use below helm command to deploy: -> Note: you may need to edit settings in helm [value.yaml](deploy/helm/ccnp-device-plugin/value.yaml) according to you cluster status. - -``` -$ cd device-plugin/ccnp-device-plugin/ -$ helm install ccnp-device-plugin deploy/helm/ccnp-device-plugin -``` - -After the deployment, for TDVM node, you can see below resource info: -``` -kubectl describe node -... -Capacity: - cpu: 8 -... - memory: 7687708Ki - pods: 110 - tdx.intel.com/tdx-guest: 110 -Allocatable: - cpu: 8 -... - memory: 7585308Ki - pods: 110 - tdx.intel.com/tdx-guest: 110 -... -Allocated resources: - (Total limits may be over 100 percent, i.e., over committed.) - Resource Requests Limits - -------- -------- ------ - cpu 1250m (15%) 600m (7%) - memory 510Mi (6%) 690Mi (9%) -... - tdx.intel.com/tdx-guest 0 0 -... -``` - -### Testing -User can deploy a CCNP quote service with tdx-guest resource request in the DaemonSet definition yaml: -``` -... - resources: - limits: - tdx.intel.com/tdx-guest: 1 -... -``` - -And after the quote server POD is started, following resource and directory can be found in the container of the POD: -``` -$ ls -l /dev/tdx* -crw-rw-rw- 1 root root 10, 126 Jul 12 04:58 /dev/tdx_guest - -$ ls -l /run/ccnp/uds -total 0 -srwxr-xr-x 1 ccnp ccnp 0 Jul 12 04:58 quote-server.sock -``` diff --git a/device-plugin/ccnp-device-plugin/cmd/server/app.go b/device-plugin/ccnp-device-plugin/cmd/server/app.go deleted file mode 100644 index e7871ac..0000000 --- a/device-plugin/ccnp-device-plugin/cmd/server/app.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -* Copyright (c) 2023, Intel Corporation. All rights reserved.
-* SPDX-License-Identifier: Apache-2.0 - */ - -package main - -import ( - "log" - "os" - "path" - - "ccnp-device-plugin/pkg/server" - - "github.com/fsnotify/fsnotify" - "k8s.io/klog/v2" -) - -func main() { - - log.Println("Intel CCNP device plugin starting") - ccnpdpsrv := server.NewCcnpDpServer() - go ccnpdpsrv.Run() - - if err := ccnpdpsrv.RegisterToKubelet(); err != nil { - klog.Errorf("register to kubelet error: %v", err) - } - - watcher, err := fsnotify.NewWatcher() - if err != nil { - klog.Fatalf("Failed to created FS watcher.") - os.Exit(1) - } - defer watcher.Close() - - err = watcher.Add(path.Dir(server.KubeletSocket)) - if err != nil { - klog.Fatalf("watch kubelet error") - return - } - for { - select { - case event := <-watcher.Events: - if event.Name == server.KubeletSocket && event.Op&fsnotify.Create == fsnotify.Create { - klog.Fatalf("restart CCNP device plugin due to kubelet restart") - } - if event.Name == server.CcnpDpSocket && event.Op&fsnotify.Remove == fsnotify.Remove { - klog.Fatalf("restart CCNP device plugin due to device plugin socket being deleted") - } - case err := <-watcher.Errors: - klog.Fatalf("fsnotify watch error: %s", err) - } - } -} diff --git a/device-plugin/ccnp-device-plugin/deploy/helm/ccnp-device-plugin/Chart.yaml b/device-plugin/ccnp-device-plugin/deploy/helm/ccnp-device-plugin/Chart.yaml deleted file mode 100644 index 0f68f61..0000000 --- a/device-plugin/ccnp-device-plugin/deploy/helm/ccnp-device-plugin/Chart.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: v2 -name: ccnp-device-plugin -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. -appVersion: latest diff --git a/device-plugin/ccnp-device-plugin/deploy/helm/ccnp-device-plugin/templates/_helpers.tpl b/device-plugin/ccnp-device-plugin/deploy/helm/ccnp-device-plugin/templates/_helpers.tpl deleted file mode 100644 index c3370a5..0000000 --- a/device-plugin/ccnp-device-plugin/deploy/helm/ccnp-device-plugin/templates/_helpers.tpl +++ /dev/null @@ -1,63 +0,0 @@ -{{/* vim: set filetype=mustache: */}} -{{/* -Expand the name of the chart. -*/}} -{{- define "ccnp-device-plugin.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "ccnp-device-plugin.fullname" -}} -{{- if .Values.fullnameOverride -}} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- $name := default .Chart.Name .Values.nameOverride -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} -{{- end -}} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "ccnp-device-plugin.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} -{{- end -}} - -{{/* -Common labels -*/}} -{{- define "ccnp-device-plugin.labels" -}} -helm.sh/chart: {{ include "ccnp-device-plugin.chart" . }} -{{ include "ccnp-device-plugin.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end -}} - -{{/* -Selector labels -*/}} -{{- define "ccnp-device-plugin.selectorLabels" -}} -app.kubernetes.io/name: {{ include "ccnp-device-plugin.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end -}} - -{{/* -Create the name of the service account to use -*/}} -{{- define "ccnp-device-plugin.serviceAccountName" -}} -{{- if .Values.serviceAccount.create -}} - {{ default (include "ccnp-device-plugin.fullname" .) .Values.serviceAccount.name }} -{{- else -}} - {{ default "default" .Values.serviceAccount.name }} -{{- end -}} -{{- end -}} diff --git a/device-plugin/ccnp-device-plugin/deploy/helm/ccnp-device-plugin/templates/daemonset.yaml b/device-plugin/ccnp-device-plugin/deploy/helm/ccnp-device-plugin/templates/daemonset.yaml deleted file mode 100644 index 2394bca..0000000 --- a/device-plugin/ccnp-device-plugin/deploy/helm/ccnp-device-plugin/templates/daemonset.yaml +++ /dev/null @@ -1,75 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: {{ include "ccnp-device-plugin.fullname" . }} - namespace: {{ .Values.namespace }} - labels: - {{- include "ccnp-device-plugin.labels" . | nindent 4 }} -spec: - updateStrategy: - type: RollingUpdate - selector: - matchLabels: - {{- include "ccnp-device-plugin.selectorLabels" . | nindent 6 }} - template: - metadata: - labels: - {{- include "ccnp-device-plugin.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "ccnp-device-plugin.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - initContainers: - - name: check-tdx-version - image: busybox - imagePullPolicy: IfNotPresent - command: ['sh', '-c', "if [ -c /dev/tdx-guest ]; then touch /run/ccnp/dev/tdx-guest; \ - elif [ -c /dev/tdx_guest ]; then touch /run/ccnp/dev/tdx_guest; \ - elif [ -c /dev/tdx-attest ]; then touch /run/ccnp/dev/tdx-attest; \ - else echo NO-DEVICE-ERROR; \ - fi"] - volumeMounts: - - name: workdir - mountPath: /run/ccnp/dev/ - - name: device-dir - mountPath: /dev - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumeMounts: - - name: device-plugin - mountPath: /var/lib/kubelet/device-plugins - - name: workdir - mountPath: /run/ccnp/dev/ - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - volumes: - - name: device-plugin - hostPath: - type: Directory - path: /var/lib/kubelet/device-plugins - - name: workdir - emptyDir: {} - - name: device-dir - hostPath: - type: Directory - path: /dev diff --git a/device-plugin/ccnp-device-plugin/deploy/helm/ccnp-device-plugin/templates/serviceaccount.yaml b/device-plugin/ccnp-device-plugin/deploy/helm/ccnp-device-plugin/templates/serviceaccount.yaml deleted file mode 100644 index 0107a80..0000000 --- a/device-plugin/ccnp-device-plugin/deploy/helm/ccnp-device-plugin/templates/serviceaccount.yaml +++ /dev/null @@ -1,9 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "ccnp-device-plugin.serviceAccountName" . }} - namespace: {{ .Values.namespace }} - labels: -{{ include "ccnp-device-plugin.labels" . | nindent 4 }} -{{- end -}} diff --git a/device-plugin/ccnp-device-plugin/deploy/helm/ccnp-device-plugin/values.yaml b/device-plugin/ccnp-device-plugin/deploy/helm/ccnp-device-plugin/values.yaml deleted file mode 100644 index 3e5b2e5..0000000 --- a/device-plugin/ccnp-device-plugin/deploy/helm/ccnp-device-plugin/values.yaml +++ /dev/null @@ -1,52 +0,0 @@ -# Default values for ccnp-device-plugin. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: docker.io/library/ccnp-device-plugin - tag: latest - pullPolicy: IfNotPresent - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -namespace: kube-system - -serviceAccount: - # Specifies whether a service account should be created - create: true - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: ccnp-device-plugin-admin - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -nodeSelector: { - intel.feature.node.kubernetes.io/tdx-guest: enabled -} - -affinity: {} diff --git a/device-plugin/ccnp-device-plugin/deploy/node-feature-rules.yaml b/device-plugin/ccnp-device-plugin/deploy/node-feature-rules.yaml deleted file mode 100644 index 72fbd76..0000000 --- a/device-plugin/ccnp-device-plugin/deploy/node-feature-rules.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: nfd.k8s-sigs.io/v1alpha1 -kind: NodeFeatureRule -metadata: - name: ccnp-dp-devices -spec: - rules: - - name: "intel.tdx" - labels: - "intel.feature.node.kubernetes.io/tdx-guest": "enabled" - matchFeatures: - - feature: cpu.security - matchExpressions: - tdx.protected: {op: IsTrue} \ No newline at end of file diff --git a/device-plugin/ccnp-device-plugin/go.mod b/device-plugin/ccnp-device-plugin/go.mod deleted file mode 100644 index 3f1cf26..0000000 --- a/device-plugin/ccnp-device-plugin/go.mod +++ /dev/null @@ -1,21 +0,0 @@ -module ccnp-device-plugin - -go 1.20 - -require ( - github.com/fsnotify/fsnotify v1.6.0 - github.com/go-logr/logr v1.2.3 // indirect - google.golang.org/grpc v1.56.3 - k8s.io/klog/v2 v2.90.1 - k8s.io/kubelet v0.27.3 -) - -require ( - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.17.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/protobuf v1.30.0 // indirect -) diff --git a/device-plugin/ccnp-device-plugin/go.sum b/device-plugin/ccnp-device-plugin/go.sum deleted file mode 100644 index 3c9e1ef..0000000 --- a/device-plugin/ccnp-device-plugin/go.sum +++ /dev/null @@ -1,60 +0,0 @@ -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= -google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= -k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kubelet v0.27.3 h1:5WhTV1iiBu9q/rr+gvy65LQ+K/e7dmgcaYjys5ipLqY= -k8s.io/kubelet v0.27.3/go.mod h1:Mz42qgZZgWgPmOJEYaR5evmh+EoSwFzEvPBozA2y9mg= diff --git a/device-plugin/ccnp-device-plugin/pkg/server/server.go b/device-plugin/ccnp-device-plugin/pkg/server/server.go deleted file mode 100644 index aa48b96..0000000 --- a/device-plugin/ccnp-device-plugin/pkg/server/server.go +++ /dev/null @@ -1,242 +0,0 @@ -/* -* Copyright (c) 2023, Intel Corporation. All rights reserved.
-* SPDX-License-Identifier: Apache-2.0 - */ - -package server - -import ( - "context" - "errors" - "fmt" - "net" - "os" - "path" - "syscall" - "time" - - "google.golang.org/grpc" - "k8s.io/klog/v2" - dpapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" -) - -const ( - Namespace = "tdx.intel.com" - DeviceType = "tdx-guest" - CcnpDpSocket = "/var/lib/kubelet/device-plugins/ccnpdp.sock" - KubeletSocket = "/var/lib/kubelet/device-plugins/kubelet.sock" - CHECK_DEVICE_DIR = "/run/ccnp/dev/" - SYS_DEV_DIR = "/dev/" - TDX_DEVICE_DEPRECATED = "tdx-attest" - TDX_DEVICE_1_0 = "tdx-guest" - TDX_DEVICE_1_5 = "tdx_guest" - TdxDevicePermissions = "rw" - MaxRestartCount = 5 - SocketConnectTimeout = 5 - DefaultPodCount uint = 110 - UDS_WORK_DIR = "/run/ccnp/uds" - MAX_CONCURRENT_STREAMS = 100 -) - -type CcnpDpServer struct { - srv *grpc.Server - devices map[string]*dpapi.Device - ctx context.Context - cancel context.CancelFunc - restartFlag bool - tdxGuestDevice string -} - -func NewCcnpDpServer() *CcnpDpServer { - ctx, cancel := context.WithCancel(context.Background()) - opts := []grpc.ServerOption{ - grpc.MaxConcurrentStreams(MAX_CONCURRENT_STREAMS), - } - return &CcnpDpServer{ - devices: make(map[string]*dpapi.Device), - srv: grpc.NewServer(opts...), - ctx: ctx, - cancel: cancel, - restartFlag: false, - } -} - -func (ccnpdpsrv *CcnpDpServer) getTdxVersion() error { - - if _, err := os.Stat(CHECK_DEVICE_DIR + TDX_DEVICE_DEPRECATED); err == nil { - return errors.New("Deprecated TDX device found") - } - - if _, err := os.Stat(CHECK_DEVICE_DIR + TDX_DEVICE_1_0); err == nil { - ccnpdpsrv.tdxGuestDevice = SYS_DEV_DIR + TDX_DEVICE_1_0 - return nil - } - - if _, err := os.Stat(CHECK_DEVICE_DIR + TDX_DEVICE_1_5); err == nil { - ccnpdpsrv.tdxGuestDevice = SYS_DEV_DIR + TDX_DEVICE_1_5 - return nil - } - - return errors.New("No TDX device found") -} - -func (ccnpdpsrv *CcnpDpServer) scanDevice() error { - - err := ccnpdpsrv.getTdxVersion() - if err != nil { - return err - } - - for i := uint(0); i < DefaultPodCount; i++ { - deviceID := fmt.Sprintf("%s-%d", "tdx-guest", i) - ccnpdpsrv.devices[deviceID] = &dpapi.Device{ - ID: deviceID, - Health: dpapi.Healthy, - } - } - - return nil -} - -func (ccnpdpsrv *CcnpDpServer) Run() error { - - err := ccnpdpsrv.scanDevice() - if err != nil { - klog.Fatalf("scan device error: %v", err) - } - - dpapi.RegisterDevicePluginServer(ccnpdpsrv.srv, ccnpdpsrv) - - err = syscall.Unlink(CcnpDpSocket) - if err != nil && !os.IsNotExist(err) { - return err - } - listen, err := net.Listen("unix", CcnpDpSocket) - if err != nil { - return err - } - - go func() { - failCount := 0 - for { - err = ccnpdpsrv.srv.Serve(listen) - if err == nil { - break - } - - if failCount > MaxRestartCount { - klog.Fatalf("CCNP plugin server crashed. Quitting...") - } - failCount++ - } - }() - - connection, err := ccnpdpsrv.connect(CcnpDpSocket, time.Duration(SocketConnectTimeout)*time.Second) - if err != nil { - return err - } - - connection.Close() - - return nil -} - -func (s *CcnpDpServer) connect(unixSocketPath string, timeout time.Duration) (*grpc.ClientConn, error) { - - connection, err := grpc.Dial(unixSocketPath, grpc.WithInsecure(), grpc.WithBlock(), - grpc.WithTimeout(timeout), - grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) { - return net.DialTimeout("unix", addr, timeout) - }), - ) - if err != nil { - return nil, err - } - - return connection, nil -} - -func (ccnpdpsrv *CcnpDpServer) RegisterToKubelet() error { - - conn, err := ccnpdpsrv.connect(KubeletSocket, time.Duration(MaxRestartCount)*time.Second) - if err != nil { - return err - } - defer conn.Close() - - client := dpapi.NewRegistrationClient(conn) - request := &dpapi.RegisterRequest{ - Version: dpapi.Version, - Endpoint: path.Base(CcnpDpSocket), - ResourceName: Namespace + "/" + DeviceType, - } - - _, err = client.Register(context.Background(), request) - if err != nil { - return err - } - - return nil -} - -func (ccnpdpsrv *CcnpDpServer) ListAndWatch(e *dpapi.Empty, lwSrv dpapi.DevicePlugin_ListAndWatchServer) error { - tdxDevices := make([]*dpapi.Device, len(ccnpdpsrv.devices)) - - i := 0 - for _, tdxDevice := range ccnpdpsrv.devices { - tdxDevices[i] = tdxDevice - i++ - } - - err := lwSrv.Send(&dpapi.ListAndWatchResponse{Devices: tdxDevices}) - if err != nil { - klog.Fatalf("ListAndWatch error: %v", err) - return err - } - - for { - select { - case <-ccnpdpsrv.ctx.Done(): - return nil - } - } -} - -func (ccnpdpsrv *CcnpDpServer) GetDevicePluginOptions(ctx context.Context, e *dpapi.Empty) (*dpapi.DevicePluginOptions, error) { - return &dpapi.DevicePluginOptions{PreStartRequired: true}, nil -} - -func (ccnpdpsrv *CcnpDpServer) GetPreferredAllocation(ctx context.Context, r *dpapi.PreferredAllocationRequest) (*dpapi.PreferredAllocationResponse, error) { - return &dpapi.PreferredAllocationResponse{}, nil -} - -func (ccnpdpsrv *CcnpDpServer) PreStartContainer(ctx context.Context, req *dpapi.PreStartContainerRequest) (*dpapi.PreStartContainerResponse, error) { - return &dpapi.PreStartContainerResponse{}, nil -} - -func (ccnpdpsrv *CcnpDpServer) Allocate(ctx context.Context, reqs *dpapi.AllocateRequest) (*dpapi.AllocateResponse, error) { - response := &dpapi.AllocateResponse{} - - devSpec := dpapi.DeviceSpec{ - HostPath: ccnpdpsrv.tdxGuestDevice, - ContainerPath: ccnpdpsrv.tdxGuestDevice, - Permissions: TdxDevicePermissions, - } - - pluginMount := dpapi.Mount{ - ContainerPath: UDS_WORK_DIR, - HostPath: UDS_WORK_DIR, - } - - for range reqs.ContainerRequests { - klog.Infof("received resource request") - resp := dpapi.ContainerAllocateResponse{ - Envs: make(map[string]string), - Annotations: make(map[string]string), - Devices: []*dpapi.DeviceSpec{&devSpec}, - Mounts: []*dpapi.Mount{&pluginMount}, - } - response.ContainerResponses = append(response.ContainerResponses, &resp) - } - return response, nil -}