refactor(kube-api-rewriter): fixes to work with original KubeVirt

- Rename cluster resources: validatingwebhooks, mutatingwebhooks.
- Rename app.kubernetes.io/managed-by label value.
- Rename devices on nodes, rename unix socket path for Device Plugin API server.
- Remove upload.cdi.kubevirt.io ApiService.
- kube-api-rewriter: Rewrite name and value for labels and annotations.
- kube-api-rewriter: Add Exclude rules to exclude objects from API server.
- kube-api-rewriter: Exclude original CRDs from CRD list, exclude original groups/resources from discovery responses.
- kube-api-rewriter: Preserve original labels with prefix to save them during list-restore-update-rename transformation chain.

Signed-off-by: Ivan Mikheykin <[email protected]>

images/cdi-artifact/Taskfile.yaml

@@ -12,6 +12,7 @@ tasks:
       - task: status
 
   status:
+    desc: "Show git status in cloned repo"
     cmds:
       - |
         dir=$(find . -type d -name __containerized-data-importer_\* -depth 1 | head -n1)
@@ -25,6 +26,7 @@ tasks:
         git status
 
   cleanup:
+    desc: "Remove cloned CDI git repo"
     cmds:
       - |
         find . -type d -name __containerized-data-importer_\* -depth 1

images/cdi-artifact/patches/009-remove-upload-apiservice.patch

@@ -0,0 +1,46 @@
+diff --git a/pkg/operator/resources/cluster/apiserver.go b/pkg/operator/resources/cluster/apiserver.go
+index adf8093fa..f737bba55 100644
+--- a/pkg/operator/resources/cluster/apiserver.go
++++ b/pkg/operator/resources/cluster/apiserver.go
+@@ -48,7 +48,7 @@ func createStaticAPIServerResources(args *FactoryArgs) []client.Object {
+
+ func createDynamicAPIServerResources(args *FactoryArgs) []client.Object {
+ 	return []client.Object{
+-		createAPIService("v1beta1", args.Namespace, args.Client, args.Logger),
++		// createAPIService("v1beta1", args.Namespace, args.Client, args.Logger),
+ 		createDataVolumeValidatingWebhook(args.Namespace, args.Client, args.Logger),
+ 		createDataVolumeMutatingWebhook(args.Namespace, args.Client, args.Logger),
+ 		createCDIValidatingWebhook(args.Namespace, args.Client, args.Logger),
+diff --git a/pkg/operator/resources/cluster/rbac.go b/pkg/operator/resources/cluster/rbac.go
+index 88e14c39a..85635efd9 100644
+--- a/pkg/operator/resources/cluster/rbac.go
++++ b/pkg/operator/resources/cluster/rbac.go
+@@ -58,17 +58,17 @@ func getAdminPolicyRules() []rbacv1.PolicyRule {
+ 				"create",
+ 			},
+ 		},
+-		{
+-			APIGroups: []string{
+-				"upload.cdi.kubevirt.io",
+-			},
+-			Resources: []string{
+-				"uploadtokenrequests",
+-			},
+-			Verbs: []string{
+-				"*",
+-			},
+-		},
++		// {
++		//	APIGroups: []string{
++		//		"upload.cdi.kubevirt.io",
++		//	},
++		//	Resources: []string{
++		//		"uploadtokenrequests",
++		//	},
++		//	Verbs: []string{
++		//		"*",
++		//	},
++		// },
+ 	}
+ }
+

images/cdi-artifact/patches/README.md

@@ -22,6 +22,11 @@ set ContentTypeJson for kubernetes clients.
 #### `008-rename-core-resources.patch`
 Replace "cdi" with "cdi-internal-virtualziation" in the core resource names.
 
+#### `009-remove-upload-apiservice.patch`
+
+Do not install apiservice v1beta1.upload.cdi.kubevirt.io. This APIService is not used
+by DVP, but conflicts with original CDI.
+
 #### `010-rename-apigroups-in-starred-rbac.patch`
 
 Rename apiGroup to internal.virtualization.deckhouse.io for ClusterRole for cdi-deployment to prevent permanent patching:

images/kube-api-proxy/Taskfile.dist.yaml

@@ -11,6 +11,9 @@ vars:
   DevRegistry: "dev-registry.deckhouse.io/virt/dev/$USER"
 
 tasks:
+  default:
+    cmds:
+      - task: dev:status
   dev:build:
     desc: "build latest image with proxy and test-controller"
     cmds:
@@ -27,7 +30,7 @@ tasks:
           exit 1
         fi
       - |
-        kubectl -n kproxy apply -f local/proxy.yaml
+        cat local/proxy.yaml | USER=${USER} envsubst | kubectl -n kproxy apply -f -
 
   dev:restart:
     desc: "restart deployment"

images/kube-api-proxy/local/proxy.yaml

@@ -24,15 +24,15 @@ spec:
         - name: "deckhouse-registry"
       containers:
         - name: proxy-only
-          image: dev-registry.deckhouse.io/virt/dev/{{ .USER }}/kube-api-proxy:latest
+          image: dev-registry.deckhouse.io/virt/dev/${USER}/kube-api-proxy:latest
           imagePullPolicy: Always
           command:
             - /proxy
           env:
             - name: CLIENT_PROXY_PORT
               value: "23916"
         - name: proxy
-          image: dev-registry.deckhouse.io/virt/dev/{{ .USER }}/kube-api-proxy:latest
+          image: dev-registry.deckhouse.io/virt/dev/${USER}/kube-api-proxy:latest
           imagePullPolicy: Always
           command:
             - /proxy
@@ -51,7 +51,7 @@ spec:
             - mountPath: /tmp/proxy-webhook/serving-certs
               name: test-admission-webhook
         - name: controller
-          image: dev-registry.deckhouse.io/virt/dev/{{ .USER }}/kube-api-proxy:latest
+          image: dev-registry.deckhouse.io/virt/dev/${USER}/kube-api-proxy:latest
           imagePullPolicy: Always
           command:
             - /test-controller

images/kube-api-proxy/pkg/kubevirt/kubevirt_rules.go

@@ -44,6 +44,22 @@ var KubevirtRewriteRules = &RewriteRules{
 			// Special cases.
 			{Original: "node-labeller.kubevirt.io/skip-node", Renamed: "node-labeller." + rootPrefix + "/skip-node"},
 			{Original: "node-labeller.kubevirt.io/obsolete-host-model", Renamed: "node-labeller." + internalPrefix + "/obsolete-host-model"},
+			{
+				Original: "app.kubernetes.io/managed-by", OriginalValue: "cdi-operator",
+				Renamed: "app.kubernetes.io/managed-by", RenamedValue: "cdi-operator-internal-virtualization",
+			},
+			{
+				Original: "app.kubernetes.io/managed-by", OriginalValue: "cdi-controller",
+				Renamed: "app.kubernetes.io/managed-by", RenamedValue: "cdi-controller-internal-virtualization",
+			},
+			{
+				Original: "app.kubernetes.io/managed-by", OriginalValue: "virt-operator",
+				Renamed: "app.kubernetes.io/managed-by", RenamedValue: "virt-operator-internal-virtualization",
+			},
+			{
+				Original: "app.kubernetes.io/managed-by", OriginalValue: "kubevirt-operator",
+				Renamed: "app.kubernetes.io/managed-by", RenamedValue: "kubevirt-operator-internal-virtualization",
+			},
 		},
 		Prefixes: []MetadataReplaceRule{
 			// CDI related labels.
@@ -85,6 +101,26 @@ var KubevirtRewriteRules = &RewriteRules{
 			{Original: "operator.cdi.kubevirt.io", Renamed: "operator.cdi." + internalPrefix},
 		},
 	},
+	Excludes: []ExcludeRule{
+		ExcludeRule{
+			Kinds: []string{
+				"PersistentVolumeClaim",
+				"PersistentVolume",
+				"Pod",
+			},
+			MatchLabels: map[string]string{
+				"app.kubernetes.io/managed-by": "cdi-controller",
+			},
+		},
+		ExcludeRule{
+			Kinds: []string{
+				"CDI",
+			},
+			MatchNames: []string{
+				"cdi",
+			},
+		},
+	},
 }
 
 // TODO create generator in golang to produce below rules from Kubevirt and CDI sources so proxy can work with future versions.

images/kube-api-proxy/pkg/proxy/handler.go

@@ -391,12 +391,18 @@ func (h *Handler) transformResponse(targetReq *rewriter.TargetRequest, w http.Re
 	}
 
 	// Step 2. Rewrite response JSON.
+	statusCode := resp.StatusCode
 	rwrBodyBytes, err := h.Rewriter.RewriteJSONPayload(targetReq, origBodyBytes, FromTargetAction(h.ProxyMode))
-	if err != nil {
+	if err != nil && err != rewriter.SkipItem {
 		logger.Error("Error rewriting response", logutil.SlogErr(err))
 		http.Error(w, "can't rewrite response", http.StatusInternalServerError)
 		return
 	}
+	// Return NotFound Status object if rewriter decides to skip resource.
+	if err != nil && err == rewriter.SkipItem {
+		rwrBodyBytes = notFoundJSON(targetReq.OrigResourceType(), origBodyBytes)
+		statusCode = http.StatusNotFound
+	}
 
 	if targetReq.IsWebhook() {
 		logutil.DebugBodyHead(logger, "Response from webhook", targetReq.ResourceForLog(), origBodyBytes)
@@ -411,7 +417,7 @@ func (h *Handler) transformResponse(targetReq *rewriter.TargetRequest, w http.Re
 	if rwrBodyBytes != nil {
 		w.Header().Set("Content-Length", strconv.Itoa(len(rwrBodyBytes)))
 	}
-	w.WriteHeader(resp.StatusCode)
+	w.WriteHeader(statusCode)
 
 	// Step 4. Write non-empty rewritten body to the client.
 	if rwrBodyBytes != nil {
@@ -458,3 +464,24 @@ func (iw *immediateWriter) Write(p []byte) (n int, err error) {
 
 	return
 }
+
+// notFoundJSON constructs Status response of type NotFound
+// for resourceType and object name.
+// Example:
+//
+//	{
+//		"kind":"Status",
+//		"apiVersion":"v1",
+//		"metadata":{},
+//		"status":"Failure",
+//		"message":"pods \"vmi-router-x9mqwdqwd\" not found",
+//		"reason":"NotFound",
+//		"details":{"name":"vmi-router-x9mqwdqwd","kind":"pods"},
+//		"code":404}
+func notFoundJSON(resourceType string, obj []byte) []byte {
+	objName := gjson.GetBytes(obj, "metadata.name").String()
+	details := fmt.Sprintf(`"details":{"name":"%s","kind":"%s"}`, objName, resourceType)
+	message := fmt.Sprintf(`"message":"%s %s not found"`, resourceType, objName)
+	notFoundTpl := `{"kind":"Status","apiVersion":"v1",%s,%s,"metadata":{},"status":"Failure","reason":"NotFound","code":404}`
+	return []byte(fmt.Sprintf(notFoundTpl, message, details))
+}

images/kube-api-proxy/pkg/proxy/stream_handler.go

@@ -120,20 +120,25 @@ func (s *StreamHandler) proxy() {
 			// There is nothing to send to the client: no event decoded.
 		} else {
 			rwrEvent, err = s.transformWatchEvent(&got)
-			if err != nil {
-				s.log.Error(fmt.Sprintf("Watch event '%s': transform error", got.Type), logutil.SlogErr(err))
-				logutil.DebugBodyHead(s.log, fmt.Sprintf("Watch event '%s'", got.Type), s.targetReq.OrigResourceType(), got.Object.Raw)
-			}
-			if rwrEvent == nil {
-				// No rewrite, pass original event as-is.
-				rwrEvent = &got
+			if err != nil && err == rewriter.SkipItem {
+				s.log.Warn(fmt.Sprintf("Watch event '%s': skipped by rewriter", got.Type), logutil.SlogErr(err))
+				logutil.DebugBodyHead(s.log, fmt.Sprintf("Watch event '%s' skipped", got.Type), s.targetReq.OrigResourceType(), got.Object.Raw)
 			} else {
-				// Log changes after rewrite.
-				logutil.DebugBodyChanges(s.log, "Watch event", s.targetReq.OrigResourceType(), got.Object.Raw, rwrEvent.Object.Raw)
+				if err != nil {
+					s.log.Error(fmt.Sprintf("Watch event '%s': transform error", got.Type), logutil.SlogErr(err))
+					logutil.DebugBodyHead(s.log, fmt.Sprintf("Watch event '%s'", got.Type), s.targetReq.OrigResourceType(), got.Object.Raw)
+				}
+				if rwrEvent == nil {
+					// No rewrite, pass original event as-is.
+					rwrEvent = &got
+				} else {
+					// Log changes after rewrite.
+					logutil.DebugBodyChanges(s.log, "Watch event", s.targetReq.OrigResourceType(), got.Object.Raw, rwrEvent.Object.Raw)
+				}
+				// Pass event to the client.
+				logutil.DebugBodyHead(s.log, fmt.Sprintf("WatchEvent type '%s' send back to client %d bytes", rwrEvent.Type, len(rwrEvent.Object.Raw)), s.targetReq.OrigResourceType(), rwrEvent.Object.Raw)
+				s.writeEvent(rwrEvent)
 			}
-			// Pass event to the client.
-			logutil.DebugBodyHead(s.log, fmt.Sprintf("WatchEvent type '%s' send back to client %d bytes", rwrEvent.Type, len(rwrEvent.Object.Raw)), s.targetReq.OrigResourceType(), rwrEvent.Object.Raw)
-			s.writeEvent(rwrEvent)
 		}
 
 		// Check if application is stopped before waiting for the next event.
@@ -217,6 +222,9 @@ func (s *StreamHandler) transformWatchEvent(ev *metav1.WatchEvent) (*metav1.Watc
 		rwrObjBytes, err = s.rewriter.RewriteJSONPayload(s.targetReq, ev.Object.Raw, rewriter.Restore)
 	}
 	if err != nil {
+		if err == rewriter.SkipItem {
+			return nil, err
+		}
 		return nil, fmt.Errorf("rewrite object in WatchEvent '%s': %w", ev.Type, err)
 	}
 

images/kube-api-proxy/pkg/rewriter/affinity.go

@@ -16,6 +16,11 @@ limitations under the License.
 
 package rewriter
 
+import (
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+)
+
 // RewriteAffinity renames or restores labels in labelSelector of affinity structure.
 // See https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity
 func RewriteAffinity(rules *RewriteRules, obj []byte, path string, action Action) ([]byte, error) {
@@ -87,9 +92,20 @@ func rewriteNodeSelectorTerm(rules *RewriteRules, obj []byte, action Action) ([]
 // Selector requirement example:
 // {"key":"app.kubernetes.io/managed-by", "operator": "In", "values": ["Helm"]}
 func rewriteSelectorRequirement(rules *RewriteRules, obj []byte, action Action) ([]byte, error) {
-	return TransformString(obj, "key", func(field string) string {
-		return rules.LabelsRewriter().Rewrite(field, action)
-	})
+	key := gjson.GetBytes(obj, "key").String()
+	valuesArr := gjson.GetBytes(obj, "values").Array()
+	values := make([]string, 0, len(valuesArr))
+	for _, value := range valuesArr {
+		values = append(values, value.String())
+	}
+	rwrKey, rwrValues := rules.LabelsRewriter().RewriteNameValues(key, values, action)
+
+	obj, err := sjson.SetBytes(obj, "key", rwrKey)
+	if err != nil {
+		return nil, err
+	}
+
+	return sjson.SetBytes(obj, "values", rwrValues)
 }
 
 // rewritePodAffinity rewrites PodAffinity and PodAntiAffinity structures.

images/kube-api-proxy/pkg/rewriter/app_test.go

@@ -90,6 +90,10 @@ func createTestRewriterForApp() *RuleBasedRewriter {
 			},
 			Names: []MetadataReplaceRule{
 				{Original: "labelgroup.io", Renamed: "replacedlabelgroup.io"},
+				{
+					Original: "labelgroup.io", OriginalValue: "some-value",
+					Renamed: "replacedlabelgroup.io", RenamedValue: "some-value-renamed",
+				},
 			},
 		},
 		Annotations: MetadataReplace{
@@ -232,7 +236,9 @@ Host: 127.0.0.1
 		{`metadata.annotations.component\.replacedannogroup\.io/annoName`, "annoValue"},
 		{`metadata.annotations.component\.annogroup\.io/annoName`, ""},
 		{`spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.0.podAffinityTerm.labelSelector.matchExpressions.0.key`, "replacedlabelgroup.io"},
+		{`spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.0.podAffinityTerm.labelSelector.matchExpressions.0.values`, `["some-value-renamed"]`},
 		{`spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.0.preference.matchExpressions.0.key`, "replacedlabelgroup.io"},
+		{`spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.0.preference.matchExpressions.0.values`, `["some-value-renamed"]`},
 	}
 
 	for _, tt := range tests {

images/kube-api-proxy/pkg/rewriter/crd.go

@@ -62,6 +62,12 @@ func RestoreCRD(rules *RewriteRules, obj []byte) ([]byte, error) {
 	if !found {
 		return nil, fmt.Errorf("malformed CRD name: should be resourcetype.group, got %s", crdName)
 	}
+
+	// Skip CRD with original group to avoid duplicates in restored List.
+	if rules.HasGroup(group) {
+		return nil, SkipItem
+	}
+
 	// Do not restore CRDs in unknown groups.
 	if group != rules.RenamedGroup {
 		return nil, nil

images/kube-api-proxy/pkg/rewriter/discovery.go

@@ -51,6 +51,10 @@ func RewriteAPIGroupList(rules *RewriteRules, objBytes []byte) ([]byte, error) {
 			rwrGroups = append(rwrGroups, rules.GetAPIGroupList()...)
 			continue
 		}
+		// Remove duplicates if cluster have CRDs with original group names.
+		if rules.HasGroup(groupName) {
+			continue
+		}
 		rwrGroups = append(rwrGroups, runtime.RawExtension{Raw: []byte(group.Raw)})
 	}
 
@@ -312,6 +316,10 @@ func RewriteAPIGroupDiscoveryList(rules *RewriteRules, obj []byte) ([]byte, erro
 		groupName := gjson.GetBytes(itemBytes, "metadata.name").String()
 
 		if groupName != rules.RenamedGroup {
+			// Remove duplicates if cluster have CRDs with original group names.
+			if rules.HasGroup(groupName) {
+				continue
+			}
 			// No transform for non-renamed groups.
 			rwrItems, err = sjson.SetRawBytes(rwrItems, "-1", itemBytes)
 			if err != nil {

images/kube-api-proxy/pkg/rewriter/list.go

@@ -17,6 +17,7 @@ limitations under the License.
 package rewriter
 
 import (
+	"errors"
 	"strings"
 
 	"github.com/tidwall/gjson"
@@ -46,6 +47,9 @@ func RewriteResourceOrList2(payload []byte, transformFn func(singleObj []byte) (
 	return RewriteArray(payload, "items", transformFn)
 }
 
+// SkipItem may be used by the transformFn to indicate that the item should be skipped from the result.
+var SkipItem = errors.New("remove item from the result")
+
 // RewriteArray gets array by path and transforms each item using transformFn.
 // Use Root path to transform object itself.
 func RewriteArray(obj []byte, arrayPath string, transformFn func(item []byte) ([]byte, error)) ([]byte, error) {
@@ -58,6 +62,9 @@ func RewriteArray(obj []byte, arrayPath string, transformFn func(item []byte) ([
 	for _, item := range items {
 		rwrItem, err := transformFn([]byte(item.Raw))
 		if err != nil {
+			if err == SkipItem {
+				continue
+			}
 			return nil, err
 		}
 		// Put original item back.