From f6d37bb4fd51f9f7d738d29e3f3dc911f141a257 Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Thu, 11 Jul 2024 19:51:11 +0300 Subject: [PATCH] refactor(kube-api-rewriter): fixes to work with original KubeVirt - Rename cluster resources: validatingwebhooks, mutatingwebhooks. - Rename app.kubernetes.io/managed-by label value. - Rename devices on nodes, rename unix socket path for Device Plugin API server. - Remove upload.cdi.kubevirt.io ApiService. - kube-api-rewriter: Rewrite name and value for labels and annotations. - kube-api-rewriter: Add Exclude rules to exclude objects from API server. - kube-api-rewriter: Exclude original CRDs from CRD list, exclude original groups/resources from discovery responses. - kube-api-rewriter: Preserve original labels with prefix to save them during list-restore-update-rename transformation chain. Signed-off-by: Ivan Mikheykin --- images/cdi-artifact/Taskfile.yaml | 2 + .../009-remove-upload-apiservice.patch | 46 +++++ images/cdi-artifact/patches/README.md | 5 + images/kube-api-proxy/Taskfile.dist.yaml | 5 +- images/kube-api-proxy/local/proxy.yaml | 6 +- .../pkg/kubevirt/kubevirt_rules.go | 36 ++++ images/kube-api-proxy/pkg/proxy/handler.go | 31 +++- .../pkg/proxy/stream_handler.go | 32 ++-- .../kube-api-proxy/pkg/rewriter/affinity.go | 22 ++- .../kube-api-proxy/pkg/rewriter/app_test.go | 6 + images/kube-api-proxy/pkg/rewriter/crd.go | 6 + .../kube-api-proxy/pkg/rewriter/discovery.go | 8 + images/kube-api-proxy/pkg/rewriter/list.go | 7 + .../kube-api-proxy/pkg/rewriter/metadata.go | 4 +- .../pkg/rewriter/prefixed_name_rewriter.go | 171 ++++++++++++++++-- .../pkg/rewriter/rule_rewriter.go | 69 +++++-- .../pkg/rewriter/rule_rewriter_test.go | 22 +++ images/kube-api-proxy/pkg/rewriter/rules.go | 87 ++++++++- .../kube-api-proxy/pkg/rewriter/rules_test.go | 119 ++++++++++++ images/virt-artifact/Taskfile.yaml | 2 + .../patches/015-rename-core-resources.patch | 31 +++- .../016-rename-install-strategy-labels.patch | 13 ++ .../018-rename-devices-kubevirt-io.patch | 49 +++++ images/virt-artifact/patches/README.md | 16 ++ templates/kubevirt/kubevirt.yaml | 11 +- 25 files changed, 738 insertions(+), 68 deletions(-) create mode 100644 images/cdi-artifact/patches/009-remove-upload-apiservice.patch create mode 100644 images/kube-api-proxy/pkg/rewriter/rules_test.go create mode 100644 images/virt-artifact/patches/018-rename-devices-kubevirt-io.patch diff --git a/images/cdi-artifact/Taskfile.yaml b/images/cdi-artifact/Taskfile.yaml index 56c021f8e9..38d21ff7c2 100644 --- a/images/cdi-artifact/Taskfile.yaml +++ b/images/cdi-artifact/Taskfile.yaml @@ -12,6 +12,7 @@ tasks: - task: status status: + desc: "Show git status in cloned repo" cmds: - | dir=$(find . -type d -name __containerized-data-importer_\* -depth 1 | head -n1) @@ -25,6 +26,7 @@ tasks: git status cleanup: + desc: "Remove cloned CDI git repo" cmds: - | find . -type d -name __containerized-data-importer_\* -depth 1 diff --git a/images/cdi-artifact/patches/009-remove-upload-apiservice.patch b/images/cdi-artifact/patches/009-remove-upload-apiservice.patch new file mode 100644 index 0000000000..2a4daeccbe --- /dev/null +++ b/images/cdi-artifact/patches/009-remove-upload-apiservice.patch @@ -0,0 +1,46 @@ +diff --git a/pkg/operator/resources/cluster/apiserver.go b/pkg/operator/resources/cluster/apiserver.go +index adf8093fa..f737bba55 100644 +--- a/pkg/operator/resources/cluster/apiserver.go ++++ b/pkg/operator/resources/cluster/apiserver.go +@@ -48,7 +48,7 @@ func createStaticAPIServerResources(args *FactoryArgs) []client.Object { + + func createDynamicAPIServerResources(args *FactoryArgs) []client.Object { + return []client.Object{ +- createAPIService("v1beta1", args.Namespace, args.Client, args.Logger), ++ // createAPIService("v1beta1", args.Namespace, args.Client, args.Logger), + createDataVolumeValidatingWebhook(args.Namespace, args.Client, args.Logger), + createDataVolumeMutatingWebhook(args.Namespace, args.Client, args.Logger), + createCDIValidatingWebhook(args.Namespace, args.Client, args.Logger), +diff --git a/pkg/operator/resources/cluster/rbac.go b/pkg/operator/resources/cluster/rbac.go +index 88e14c39a..85635efd9 100644 +--- a/pkg/operator/resources/cluster/rbac.go ++++ b/pkg/operator/resources/cluster/rbac.go +@@ -58,17 +58,17 @@ func getAdminPolicyRules() []rbacv1.PolicyRule { + "create", + }, + }, +- { +- APIGroups: []string{ +- "upload.cdi.kubevirt.io", +- }, +- Resources: []string{ +- "uploadtokenrequests", +- }, +- Verbs: []string{ +- "*", +- }, +- }, ++ // { ++ // APIGroups: []string{ ++ // "upload.cdi.kubevirt.io", ++ // }, ++ // Resources: []string{ ++ // "uploadtokenrequests", ++ // }, ++ // Verbs: []string{ ++ // "*", ++ // }, ++ // }, + } + } + diff --git a/images/cdi-artifact/patches/README.md b/images/cdi-artifact/patches/README.md index ed4ca60c53..61bb09bf6f 100644 --- a/images/cdi-artifact/patches/README.md +++ b/images/cdi-artifact/patches/README.md @@ -22,6 +22,11 @@ set ContentTypeJson for kubernetes clients. #### `008-rename-core-resources.patch` Replace "cdi" with "cdi-internal-virtualziation" in the core resource names. +#### `009-remove-upload-apiservice.patch` + +Do not install apiservice v1beta1.upload.cdi.kubevirt.io. This APIService is not used +by DVP, but conflicts with original CDI. + #### `010-rename-apigroups-in-starred-rbac.patch` Rename apiGroup to internal.virtualization.deckhouse.io for ClusterRole for cdi-deployment to prevent permanent patching: diff --git a/images/kube-api-proxy/Taskfile.dist.yaml b/images/kube-api-proxy/Taskfile.dist.yaml index 1f7eeccce7..cecdb808f2 100644 --- a/images/kube-api-proxy/Taskfile.dist.yaml +++ b/images/kube-api-proxy/Taskfile.dist.yaml @@ -11,6 +11,9 @@ vars: DevRegistry: "dev-registry.deckhouse.io/virt/dev/$USER" tasks: + default: + cmds: + - task: dev:status dev:build: desc: "build latest image with proxy and test-controller" cmds: @@ -27,7 +30,7 @@ tasks: exit 1 fi - | - kubectl -n kproxy apply -f local/proxy.yaml + cat local/proxy.yaml | USER=${USER} envsubst | kubectl -n kproxy apply -f - dev:restart: desc: "restart deployment" diff --git a/images/kube-api-proxy/local/proxy.yaml b/images/kube-api-proxy/local/proxy.yaml index 1cfffa3e42..77d775c5cd 100644 --- a/images/kube-api-proxy/local/proxy.yaml +++ b/images/kube-api-proxy/local/proxy.yaml @@ -24,7 +24,7 @@ spec: - name: "deckhouse-registry" containers: - name: proxy-only - image: dev-registry.deckhouse.io/virt/dev/{{ .USER }}/kube-api-proxy:latest + image: dev-registry.deckhouse.io/virt/dev/${USER}/kube-api-proxy:latest imagePullPolicy: Always command: - /proxy @@ -32,7 +32,7 @@ spec: - name: CLIENT_PROXY_PORT value: "23916" - name: proxy - image: dev-registry.deckhouse.io/virt/dev/{{ .USER }}/kube-api-proxy:latest + image: dev-registry.deckhouse.io/virt/dev/${USER}/kube-api-proxy:latest imagePullPolicy: Always command: - /proxy @@ -51,7 +51,7 @@ spec: - mountPath: /tmp/proxy-webhook/serving-certs name: test-admission-webhook - name: controller - image: dev-registry.deckhouse.io/virt/dev/{{ .USER }}/kube-api-proxy:latest + image: dev-registry.deckhouse.io/virt/dev/${USER}/kube-api-proxy:latest imagePullPolicy: Always command: - /test-controller diff --git a/images/kube-api-proxy/pkg/kubevirt/kubevirt_rules.go b/images/kube-api-proxy/pkg/kubevirt/kubevirt_rules.go index 8949008fec..624a08d5df 100644 --- a/images/kube-api-proxy/pkg/kubevirt/kubevirt_rules.go +++ b/images/kube-api-proxy/pkg/kubevirt/kubevirt_rules.go @@ -44,6 +44,22 @@ var KubevirtRewriteRules = &RewriteRules{ // Special cases. {Original: "node-labeller.kubevirt.io/skip-node", Renamed: "node-labeller." + rootPrefix + "/skip-node"}, {Original: "node-labeller.kubevirt.io/obsolete-host-model", Renamed: "node-labeller." + internalPrefix + "/obsolete-host-model"}, + { + Original: "app.kubernetes.io/managed-by", OriginalValue: "cdi-operator", + Renamed: "app.kubernetes.io/managed-by", RenamedValue: "cdi-operator-internal-virtualization", + }, + { + Original: "app.kubernetes.io/managed-by", OriginalValue: "cdi-controller", + Renamed: "app.kubernetes.io/managed-by", RenamedValue: "cdi-controller-internal-virtualization", + }, + { + Original: "app.kubernetes.io/managed-by", OriginalValue: "virt-operator", + Renamed: "app.kubernetes.io/managed-by", RenamedValue: "virt-operator-internal-virtualization", + }, + { + Original: "app.kubernetes.io/managed-by", OriginalValue: "kubevirt-operator", + Renamed: "app.kubernetes.io/managed-by", RenamedValue: "kubevirt-operator-internal-virtualization", + }, }, Prefixes: []MetadataReplaceRule{ // CDI related labels. @@ -85,6 +101,26 @@ var KubevirtRewriteRules = &RewriteRules{ {Original: "operator.cdi.kubevirt.io", Renamed: "operator.cdi." + internalPrefix}, }, }, + Excludes: []ExcludeRule{ + ExcludeRule{ + Kinds: []string{ + "PersistentVolumeClaim", + "PersistentVolume", + "Pod", + }, + MatchLabels: map[string]string{ + "app.kubernetes.io/managed-by": "cdi-controller", + }, + }, + ExcludeRule{ + Kinds: []string{ + "CDI", + }, + MatchNames: []string{ + "cdi", + }, + }, + }, } // TODO create generator in golang to produce below rules from Kubevirt and CDI sources so proxy can work with future versions. diff --git a/images/kube-api-proxy/pkg/proxy/handler.go b/images/kube-api-proxy/pkg/proxy/handler.go index 8789b79620..3408f2d425 100644 --- a/images/kube-api-proxy/pkg/proxy/handler.go +++ b/images/kube-api-proxy/pkg/proxy/handler.go @@ -391,12 +391,18 @@ func (h *Handler) transformResponse(targetReq *rewriter.TargetRequest, w http.Re } // Step 2. Rewrite response JSON. + statusCode := resp.StatusCode rwrBodyBytes, err := h.Rewriter.RewriteJSONPayload(targetReq, origBodyBytes, FromTargetAction(h.ProxyMode)) - if err != nil { + if err != nil && err != rewriter.SkipItem { logger.Error("Error rewriting response", logutil.SlogErr(err)) http.Error(w, "can't rewrite response", http.StatusInternalServerError) return } + // Return NotFound Status object if rewriter decides to skip resource. + if err != nil && err == rewriter.SkipItem { + rwrBodyBytes = notFoundJSON(targetReq.OrigResourceType(), origBodyBytes) + statusCode = http.StatusNotFound + } if targetReq.IsWebhook() { logutil.DebugBodyHead(logger, "Response from webhook", targetReq.ResourceForLog(), origBodyBytes) @@ -411,7 +417,7 @@ func (h *Handler) transformResponse(targetReq *rewriter.TargetRequest, w http.Re if rwrBodyBytes != nil { w.Header().Set("Content-Length", strconv.Itoa(len(rwrBodyBytes))) } - w.WriteHeader(resp.StatusCode) + w.WriteHeader(statusCode) // Step 4. Write non-empty rewritten body to the client. if rwrBodyBytes != nil { @@ -458,3 +464,24 @@ func (iw *immediateWriter) Write(p []byte) (n int, err error) { return } + +// notFoundJSON constructs Status response of type NotFound +// for resourceType and object name. +// Example: +// +// { +// "kind":"Status", +// "apiVersion":"v1", +// "metadata":{}, +// "status":"Failure", +// "message":"pods \"vmi-router-x9mqwdqwd\" not found", +// "reason":"NotFound", +// "details":{"name":"vmi-router-x9mqwdqwd","kind":"pods"}, +// "code":404} +func notFoundJSON(resourceType string, obj []byte) []byte { + objName := gjson.GetBytes(obj, "metadata.name").String() + details := fmt.Sprintf(`"details":{"name":"%s","kind":"%s"}`, objName, resourceType) + message := fmt.Sprintf(`"message":"%s %s not found"`, resourceType, objName) + notFoundTpl := `{"kind":"Status","apiVersion":"v1",%s,%s,"metadata":{},"status":"Failure","reason":"NotFound","code":404}` + return []byte(fmt.Sprintf(notFoundTpl, message, details)) +} diff --git a/images/kube-api-proxy/pkg/proxy/stream_handler.go b/images/kube-api-proxy/pkg/proxy/stream_handler.go index 8200c37f4e..4dd4a50b23 100644 --- a/images/kube-api-proxy/pkg/proxy/stream_handler.go +++ b/images/kube-api-proxy/pkg/proxy/stream_handler.go @@ -120,20 +120,25 @@ func (s *StreamHandler) proxy() { // There is nothing to send to the client: no event decoded. } else { rwrEvent, err = s.transformWatchEvent(&got) - if err != nil { - s.log.Error(fmt.Sprintf("Watch event '%s': transform error", got.Type), logutil.SlogErr(err)) - logutil.DebugBodyHead(s.log, fmt.Sprintf("Watch event '%s'", got.Type), s.targetReq.OrigResourceType(), got.Object.Raw) - } - if rwrEvent == nil { - // No rewrite, pass original event as-is. - rwrEvent = &got + if err != nil && err == rewriter.SkipItem { + s.log.Warn(fmt.Sprintf("Watch event '%s': skipped by rewriter", got.Type), logutil.SlogErr(err)) + logutil.DebugBodyHead(s.log, fmt.Sprintf("Watch event '%s' skipped", got.Type), s.targetReq.OrigResourceType(), got.Object.Raw) } else { - // Log changes after rewrite. - logutil.DebugBodyChanges(s.log, "Watch event", s.targetReq.OrigResourceType(), got.Object.Raw, rwrEvent.Object.Raw) + if err != nil { + s.log.Error(fmt.Sprintf("Watch event '%s': transform error", got.Type), logutil.SlogErr(err)) + logutil.DebugBodyHead(s.log, fmt.Sprintf("Watch event '%s'", got.Type), s.targetReq.OrigResourceType(), got.Object.Raw) + } + if rwrEvent == nil { + // No rewrite, pass original event as-is. + rwrEvent = &got + } else { + // Log changes after rewrite. + logutil.DebugBodyChanges(s.log, "Watch event", s.targetReq.OrigResourceType(), got.Object.Raw, rwrEvent.Object.Raw) + } + // Pass event to the client. + logutil.DebugBodyHead(s.log, fmt.Sprintf("WatchEvent type '%s' send back to client %d bytes", rwrEvent.Type, len(rwrEvent.Object.Raw)), s.targetReq.OrigResourceType(), rwrEvent.Object.Raw) + s.writeEvent(rwrEvent) } - // Pass event to the client. - logutil.DebugBodyHead(s.log, fmt.Sprintf("WatchEvent type '%s' send back to client %d bytes", rwrEvent.Type, len(rwrEvent.Object.Raw)), s.targetReq.OrigResourceType(), rwrEvent.Object.Raw) - s.writeEvent(rwrEvent) } // Check if application is stopped before waiting for the next event. @@ -217,6 +222,9 @@ func (s *StreamHandler) transformWatchEvent(ev *metav1.WatchEvent) (*metav1.Watc rwrObjBytes, err = s.rewriter.RewriteJSONPayload(s.targetReq, ev.Object.Raw, rewriter.Restore) } if err != nil { + if err == rewriter.SkipItem { + return nil, err + } return nil, fmt.Errorf("rewrite object in WatchEvent '%s': %w", ev.Type, err) } diff --git a/images/kube-api-proxy/pkg/rewriter/affinity.go b/images/kube-api-proxy/pkg/rewriter/affinity.go index 29014031c2..1110d82662 100644 --- a/images/kube-api-proxy/pkg/rewriter/affinity.go +++ b/images/kube-api-proxy/pkg/rewriter/affinity.go @@ -16,6 +16,11 @@ limitations under the License. package rewriter +import ( + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" +) + // RewriteAffinity renames or restores labels in labelSelector of affinity structure. // See https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity func RewriteAffinity(rules *RewriteRules, obj []byte, path string, action Action) ([]byte, error) { @@ -87,9 +92,20 @@ func rewriteNodeSelectorTerm(rules *RewriteRules, obj []byte, action Action) ([] // Selector requirement example: // {"key":"app.kubernetes.io/managed-by", "operator": "In", "values": ["Helm"]} func rewriteSelectorRequirement(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { - return TransformString(obj, "key", func(field string) string { - return rules.LabelsRewriter().Rewrite(field, action) - }) + key := gjson.GetBytes(obj, "key").String() + valuesArr := gjson.GetBytes(obj, "values").Array() + values := make([]string, 0, len(valuesArr)) + for _, value := range valuesArr { + values = append(values, value.String()) + } + rwrKey, rwrValues := rules.LabelsRewriter().RewriteNameValues(key, values, action) + + obj, err := sjson.SetBytes(obj, "key", rwrKey) + if err != nil { + return nil, err + } + + return sjson.SetBytes(obj, "values", rwrValues) } // rewritePodAffinity rewrites PodAffinity and PodAntiAffinity structures. diff --git a/images/kube-api-proxy/pkg/rewriter/app_test.go b/images/kube-api-proxy/pkg/rewriter/app_test.go index 9a75ab84d8..4720deffc3 100644 --- a/images/kube-api-proxy/pkg/rewriter/app_test.go +++ b/images/kube-api-proxy/pkg/rewriter/app_test.go @@ -90,6 +90,10 @@ func createTestRewriterForApp() *RuleBasedRewriter { }, Names: []MetadataReplaceRule{ {Original: "labelgroup.io", Renamed: "replacedlabelgroup.io"}, + { + Original: "labelgroup.io", OriginalValue: "some-value", + Renamed: "replacedlabelgroup.io", RenamedValue: "some-value-renamed", + }, }, }, Annotations: MetadataReplace{ @@ -232,7 +236,9 @@ Host: 127.0.0.1 {`metadata.annotations.component\.replacedannogroup\.io/annoName`, "annoValue"}, {`metadata.annotations.component\.annogroup\.io/annoName`, ""}, {`spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.0.podAffinityTerm.labelSelector.matchExpressions.0.key`, "replacedlabelgroup.io"}, + {`spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.0.podAffinityTerm.labelSelector.matchExpressions.0.values`, `["some-value-renamed"]`}, {`spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.0.preference.matchExpressions.0.key`, "replacedlabelgroup.io"}, + {`spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.0.preference.matchExpressions.0.values`, `["some-value-renamed"]`}, } for _, tt := range tests { diff --git a/images/kube-api-proxy/pkg/rewriter/crd.go b/images/kube-api-proxy/pkg/rewriter/crd.go index 7f37cd4722..90a33a3842 100644 --- a/images/kube-api-proxy/pkg/rewriter/crd.go +++ b/images/kube-api-proxy/pkg/rewriter/crd.go @@ -62,6 +62,12 @@ func RestoreCRD(rules *RewriteRules, obj []byte) ([]byte, error) { if !found { return nil, fmt.Errorf("malformed CRD name: should be resourcetype.group, got %s", crdName) } + + // Skip CRD with original group to avoid duplicates in restored List. + if rules.HasGroup(group) { + return nil, SkipItem + } + // Do not restore CRDs in unknown groups. if group != rules.RenamedGroup { return nil, nil diff --git a/images/kube-api-proxy/pkg/rewriter/discovery.go b/images/kube-api-proxy/pkg/rewriter/discovery.go index cc5fd94f57..ab7f22227e 100644 --- a/images/kube-api-proxy/pkg/rewriter/discovery.go +++ b/images/kube-api-proxy/pkg/rewriter/discovery.go @@ -51,6 +51,10 @@ func RewriteAPIGroupList(rules *RewriteRules, objBytes []byte) ([]byte, error) { rwrGroups = append(rwrGroups, rules.GetAPIGroupList()...) continue } + // Remove duplicates if cluster have CRDs with original group names. + if rules.HasGroup(groupName) { + continue + } rwrGroups = append(rwrGroups, runtime.RawExtension{Raw: []byte(group.Raw)}) } @@ -312,6 +316,10 @@ func RewriteAPIGroupDiscoveryList(rules *RewriteRules, obj []byte) ([]byte, erro groupName := gjson.GetBytes(itemBytes, "metadata.name").String() if groupName != rules.RenamedGroup { + // Remove duplicates if cluster have CRDs with original group names. + if rules.HasGroup(groupName) { + continue + } // No transform for non-renamed groups. rwrItems, err = sjson.SetRawBytes(rwrItems, "-1", itemBytes) if err != nil { diff --git a/images/kube-api-proxy/pkg/rewriter/list.go b/images/kube-api-proxy/pkg/rewriter/list.go index 1bdc8b6954..d8a2fd17e1 100644 --- a/images/kube-api-proxy/pkg/rewriter/list.go +++ b/images/kube-api-proxy/pkg/rewriter/list.go @@ -17,6 +17,7 @@ limitations under the License. package rewriter import ( + "errors" "strings" "github.com/tidwall/gjson" @@ -46,6 +47,9 @@ func RewriteResourceOrList2(payload []byte, transformFn func(singleObj []byte) ( return RewriteArray(payload, "items", transformFn) } +// SkipItem may be used by the transformFn to indicate that the item should be skipped from the result. +var SkipItem = errors.New("remove item from the result") + // RewriteArray gets array by path and transforms each item using transformFn. // Use Root path to transform object itself. func RewriteArray(obj []byte, arrayPath string, transformFn func(item []byte) ([]byte, error)) ([]byte, error) { @@ -58,6 +62,9 @@ func RewriteArray(obj []byte, arrayPath string, transformFn func(item []byte) ([ for _, item := range items { rwrItem, err := transformFn([]byte(item.Raw)) if err != nil { + if err == SkipItem { + continue + } return nil, err } // Put original item back. diff --git a/images/kube-api-proxy/pkg/rewriter/metadata.go b/images/kube-api-proxy/pkg/rewriter/metadata.go index b1462c21b8..8f6fa59a5e 100644 --- a/images/kube-api-proxy/pkg/rewriter/metadata.go +++ b/images/kube-api-proxy/pkg/rewriter/metadata.go @@ -104,13 +104,13 @@ func RenameMetadataPatch(rules *RewriteRules, patch []byte) ([]byte, error) { func RewriteLabelsMap(rules *RewriteRules, obj []byte, path string, action Action) ([]byte, error) { return RewriteMapStringString(obj, path, func(k, v string) (string, string) { - return rules.LabelsRewriter().Rewrite(k, action), v + return rules.LabelsRewriter().RewriteNameValue(k, v, action) }) } func RewriteAnnotationsMap(rules *RewriteRules, obj []byte, path string, action Action) ([]byte, error) { return RewriteMapStringString(obj, path, func(k, v string) (string, string) { - return rules.AnnotationsRewriter().Rewrite(k, action), v + return rules.AnnotationsRewriter().RewriteNameValue(k, v, action) }) } diff --git a/images/kube-api-proxy/pkg/rewriter/prefixed_name_rewriter.go b/images/kube-api-proxy/pkg/rewriter/prefixed_name_rewriter.go index 191553747f..eda507da4c 100644 --- a/images/kube-api-proxy/pkg/rewriter/prefixed_name_rewriter.go +++ b/images/kube-api-proxy/pkg/rewriter/prefixed_name_rewriter.go @@ -18,6 +18,8 @@ package rewriter import "strings" +const PreservedPrefix = "preserved-original-" + type PrefixedNameRewriter struct { namesRenameIdx map[string]string namesRestoreIdx map[string]string @@ -37,13 +39,36 @@ func NewPrefixedNameRewriter(replaceRules MetadataReplace) *PrefixedNameRewriter func (p *PrefixedNameRewriter) Rewrite(name string, action Action) string { switch action { case Rename: - return p.rename(name) + name, _ = p.rename(name, "") case Restore: - return p.restore(name) + name, _ = p.restore(name, "") } return name } +func (p *PrefixedNameRewriter) RewriteNameValue(name, value string, action Action) (string, string) { + switch action { + case Rename: + return p.rename(name, value) + case Restore: + return p.restore(name, value) + } + return name, value +} + +func (p *PrefixedNameRewriter) RewriteNameValues(name string, values []string, action Action) (string, []string) { + if len(values) == 0 { + return p.Rewrite(name, action), values + } + switch action { + case Rename: + return p.rewriteNameValues(name, values, p.rename) + case Restore: + return p.rewriteNameValues(name, values, p.restore) + } + return name, values +} + func (p *PrefixedNameRewriter) RewriteSlice(names []string, action Action) []string { switch action { case Rename: @@ -64,12 +89,12 @@ func (p *PrefixedNameRewriter) RewriteMap(names map[string]string, action Action return names } -func (p *PrefixedNameRewriter) Rename(name string) string { - return p.rename(name) +func (p *PrefixedNameRewriter) Rename(name, value string) (string, string) { + return p.rename(name, value) } -func (p *PrefixedNameRewriter) Restore(name string) string { - return p.restore(name) +func (p *PrefixedNameRewriter) Restore(name, value string) (string, string) { + return p.restore(name, value) } func (p *PrefixedNameRewriter) RenameSlice(names []string) []string { @@ -88,61 +113,153 @@ func (p *PrefixedNameRewriter) RestoreMap(names map[string]string) map[string]st return p.rewriteMap(names, p.restore) } -func (p *PrefixedNameRewriter) rewriteMap(names map[string]string, fn func(string) string) map[string]string { +// rewriteNameValues rewrite name and values, e.g. for matchExpressions. +// Method uses all rules to detect a new name, first matching rule is applied. +// Values may be rewritten partially depending on specified name-value rules. +func (p *PrefixedNameRewriter) rewriteNameValues(name string, values []string, fn func(string, string) (string, string)) (string, []string) { + rwrName := name + rwrValues := make([]string, 0, len(values)) + + for _, value := range values { + n, v := fn(name, value) + // Set new name only for the first matching rule. + if n != name && rwrName == name { + rwrName = n + } + rwrValues = append(rwrValues, v) + } + + return rwrName, rwrValues +} + +func (p *PrefixedNameRewriter) rewriteMap(names map[string]string, fn func(string, string) (string, string)) map[string]string { if names == nil { return nil } result := make(map[string]string) for name, value := range names { - result[fn(name)] = value + rwrName, rwrValue := fn(name, value) + result[rwrName] = rwrValue } return result } -func (p *PrefixedNameRewriter) rewriteSlice(names []string, fn func(string) string) []string { +// rewriteSlice do not rewrite values, only names. +func (p *PrefixedNameRewriter) rewriteSlice(names []string, fn func(string, string) (string, string)) []string { if names == nil { return nil } result := make([]string, 0, len(names)) for _, name := range names { - result = append(result, fn(name)) + rwrName, _ := fn(name, "") + result = append(result, rwrName) } return result } -func (p *PrefixedNameRewriter) rename(name string) string { +// rename rewrites original names and values. If label was preserved, rewrite it to original state. +func (p *PrefixedNameRewriter) rename(name, value string) (string, string) { + if p.isPreserved(name) { + return p.restorePreservedName(name), value + } + + // First try to find name and value. + if value != "" { + idxKey := joinKV(name, value) + if renamedIdxValue, ok := p.namesRenameIdx[idxKey]; ok { + return splitKV(renamedIdxValue) + } + } + // No exact rule for name and value, try to find exact name match. if renamed, ok := p.namesRenameIdx[name]; ok { - return renamed + return renamed, value } // No exact name, find prefix. prefix, remainder, found := strings.Cut(name, "/") if !found { - return name + return name, value } if renamedPrefix, ok := p.prefixRenameIdx[prefix]; ok { - return renamedPrefix + "/" + remainder + return renamedPrefix + "/" + remainder, value } - return name + return name, value } -func (p *PrefixedNameRewriter) restore(name string) string { +// restore rewrites renamed names and values to their original state. +// If name is already original, preserve it with prefix, to make it unknown for client but keep in place for UPDATE/PATCH operations. +func (p *PrefixedNameRewriter) restore(name, value string) (string, string) { + if p.isOriginal(name, value) { + return p.preserveName(name), value + } + + // First try to find name and value. + if value != "" { + idxKey := joinKV(name, value) + if restoredIdxValue, ok := p.namesRestoreIdx[idxKey]; ok { + return splitKV(restoredIdxValue) + } + } + // No exact rule for name and value, try to find exact name match. if restored, ok := p.namesRestoreIdx[name]; ok { - return restored + return restored, value } // No exact name, find prefix. prefix, remainder, found := strings.Cut(name, "/") if !found { - return name + return name, value } if restoredPrefix, ok := p.prefixRestoreIdx[prefix]; ok { - return restoredPrefix + "/" + remainder + return restoredPrefix + "/" + remainder, value } - return name + return name, value +} + +// isOriginal returns true if label should be renamed. +func (p *PrefixedNameRewriter) isOriginal(name, value string) bool { + if value != "" { + // Label is "original" if there is rule for renaming name and value. + idxKey := joinKV(name, value) + if _, ok := p.namesRenameIdx[idxKey]; ok { + return true + } + } + + // Try to find rule for exact name match. + if _, ok := p.namesRenameIdx[name]; ok { + return true + } + // No exact name, find rule for prefix. + prefix, _, found := strings.Cut(name, "/") + if !found { + // Label is only a name, but no rule for name found, so it is not "original". + return false + } + if _, ok := p.prefixRenameIdx[prefix]; ok { + return true + } + return false +} + +func (p *PrefixedNameRewriter) isPreserved(name string) bool { + return strings.HasSuffix(name, PreservedPrefix) +} + +func (p *PrefixedNameRewriter) preserveName(name string) string { + return PreservedPrefix + name +} + +func (p *PrefixedNameRewriter) restorePreservedName(name string) string { + return strings.TrimPrefix(name, PreservedPrefix) } func indexRules(rules []MetadataReplaceRule) map[string]string { idx := make(map[string]string, len(rules)) for _, rule := range rules { + if rule.OriginalValue != "" && rule.RenamedValue != "" { + idxKey := joinKV(rule.Original, rule.OriginalValue) + idx[idxKey] = rule.Renamed + "=" + rule.RenamedValue + continue + } idx[rule.Original] = rule.Renamed } return idx @@ -151,7 +268,21 @@ func indexRules(rules []MetadataReplaceRule) map[string]string { func indexRulesReverse(rules []MetadataReplaceRule) map[string]string { idx := make(map[string]string, len(rules)) for _, rule := range rules { + if rule.OriginalValue != "" && rule.RenamedValue != "" { + idxKey := joinKV(rule.Renamed, rule.RenamedValue) + idx[idxKey] = rule.Original + "=" + rule.OriginalValue + continue + } idx[rule.Renamed] = rule.Original } return idx } + +func joinKV(name, value string) string { + return name + "=" + value +} + +func splitKV(idxValue string) (name, value string) { + name, value, _ = strings.Cut(idxValue, "=") + return +} diff --git a/images/kube-api-proxy/pkg/rewriter/rule_rewriter.go b/images/kube-api-proxy/pkg/rewriter/rule_rewriter.go index a0d443afa3..f92ea69757 100644 --- a/images/kube-api-proxy/pkg/rewriter/rule_rewriter.go +++ b/images/kube-api-proxy/pkg/rewriter/rule_rewriter.go @@ -206,7 +206,7 @@ func (rw *RuleBasedRewriter) rewriteLabelSelector(rawQuery string) string { rwrMatchExpressions := make([]metav1.LabelSelectorRequirement, 0) for _, expr := range labelSelector.MatchExpressions { rwrExpr := expr - rwrExpr.Key = rw.Rules.LabelsRewriter().Rename(rwrExpr.Key) + rwrExpr.Key, rwrExpr.Values = rw.Rules.LabelsRewriter().RewriteNameValues(rwrExpr.Key, rwrExpr.Values, Rename) rwrMatchExpressions = append(rwrMatchExpressions, rwrExpr) } @@ -232,20 +232,10 @@ func (rw *RuleBasedRewriter) RewriteJSONPayload(targetReq *TargetRequest, obj [] var rwrBytes []byte var err error - //// Handle core resources: rewrite only for specific kind. - //if targetReq.IsCore() { - // pass := true - // switch kind { - // case "APIGroupList": - // case "APIGroup": - // case "APIResourceList": - // default: - // pass = shouldPassCoreResource(kind) - // } - // if pass { - // return obj, nil - // } - //} + obj, err = rw.FilterExcludes(obj, action) + if err != nil { + return obj, err + } switch kind { case "APIGroupList": @@ -349,6 +339,39 @@ func (rw *RuleBasedRewriter) RewritePatch(targetReq *TargetRequest, patchBytes [ return RenameMetadataPatch(rw.Rules, patchBytes) } +// FilterExcludes removes excluded resources from the list or return SkipItem if resource itself is excluded. +func (rw *RuleBasedRewriter) FilterExcludes(obj []byte, action Action) ([]byte, error) { + if action != Restore { + return obj, nil + } + + kind := gjson.GetBytes(obj, "kind").String() + if !isExcludableKind(kind) { + return obj, nil + } + + if rw.Rules.ShouldExclude(obj, kind) { + return obj, SkipItem + } + + // Also check each item if obj is List + if !strings.HasSuffix(kind, "List") { + return obj, nil + } + + singleKind := strings.TrimSuffix(kind, "List") + obj, err := RewriteResourceOrList2(obj, func(singleObj []byte) ([]byte, error) { + if rw.Rules.ShouldExclude(singleObj, singleKind) { + return nil, SkipItem + } + return nil, nil + }) + if err != nil { + return obj, err + } + return obj, nil +} + func shouldRewriteOwnerReferences(resourceType string) bool { switch resourceType { case CRDKind, CRDListKind, @@ -377,3 +400,19 @@ func shouldRewriteOwnerReferences(resourceType string) bool { return false } + +// isExcludeKind returns true if kind may be excluded from rewriting. +// Discovery kinds and AdmissionReview have special schemas, it is sane to +// exclude resources in particular rewriters. +func isExcludableKind(kind string) bool { + switch kind { + case "APIGroupList", + "APIGroup", + "APIResourceList", + "APIGroupDiscoveryList", + "AdmissionReview": + return false + } + + return true +} diff --git a/images/kube-api-proxy/pkg/rewriter/rule_rewriter_test.go b/images/kube-api-proxy/pkg/rewriter/rule_rewriter_test.go index 8018a5b054..3216bfcc5e 100644 --- a/images/kube-api-proxy/pkg/rewriter/rule_rewriter_test.go +++ b/images/kube-api-proxy/pkg/rewriter/rule_rewriter_test.go @@ -100,6 +100,10 @@ func createTestRewriter() *RuleBasedRewriter { }, Names: []MetadataReplaceRule{ {Original: "labelgroup.io", Renamed: "replacedlabelgroup.io"}, + { + Original: "labelgroup.io", OriginalValue: "labelValueToRename", + Renamed: "replacedlabelgroup.io", RenamedValue: "renamedLabelValue", + }, }, }, Annotations: MetadataReplace{ @@ -193,6 +197,24 @@ func TestRewriteAPIEndpoint(t *testing.T) { "/apis/apps/v1/deployments", "labelSelector=replacedlabelgroup.io+notin+%28labelValue%2Cvalue-one%29&limit=500", }, + { + "labelSelector label name and renamed value", + "/api/v1/namespaces/d8-virtualization/pods?labelSelector=labelgroup.io%3DlabelValueToRename&limit=500", + "/api/v1/namespaces/d8-virtualization/pods", + "labelSelector=replacedlabelgroup.io%3DrenamedLabelValue&limit=500", + }, + { + "labelSelector label name and renamed values", + "/api/v1/namespaces/d8-virtualization/pods?labelSelector=labelgroup.io+notin+%28value-one%2ClabelValueToRename%29&limit=500", + "/api/v1/namespaces/d8-virtualization/pods", + "labelSelector=replacedlabelgroup.io+notin+%28renamedLabelValue%2Cvalue-one%29&limit=500", + }, + { + "labelSelector label name and renamed values for deployments", + "/apis/apps/v1/deployments?labelSelector=labelgroup.io+notin+%28value-one%2ClabelValueToRename%29&limit=500", + "/apis/apps/v1/deployments", + "labelSelector=replacedlabelgroup.io+notin+%28renamedLabelValue%2Cvalue-one%29&limit=500", + }, } for _, tt := range tests { diff --git a/images/kube-api-proxy/pkg/rewriter/rules.go b/images/kube-api-proxy/pkg/rewriter/rules.go index 448d15590c..925198f910 100644 --- a/images/kube-api-proxy/pkg/rewriter/rules.go +++ b/images/kube-api-proxy/pkg/rewriter/rules.go @@ -18,6 +18,8 @@ package rewriter import ( "strings" + + "github.com/tidwall/gjson" ) type RewriteRules struct { @@ -31,6 +33,7 @@ type RewriteRules struct { Labels MetadataReplace `json:"labels"` Annotations MetadataReplace `json:"annotations"` Finalizers MetadataReplace `json:"finalizers"` + Excludes []ExcludeRule `json:"excludes"` // TODO move these indexed rewriters into the RuleBasedRewriter. labelsRewriter *PrefixedNameRewriter @@ -43,6 +46,17 @@ func (rr *RewriteRules) Init() { rr.labelsRewriter = NewPrefixedNameRewriter(rr.Labels) rr.annotationsRewriter = NewPrefixedNameRewriter(rr.Annotations) rr.finalizersRewriter = NewPrefixedNameRewriter(rr.Finalizers) + + // Add all original Kinds and KindList as implicit excludes. + originalKinds := make([]string, 0) + for _, apiGroupRule := range rr.Rules { + for _, resourceRule := range apiGroupRule.ResourceRules { + originalKinds = append(originalKinds, resourceRule.Kind, resourceRule.ListKind) + } + } + if len(originalKinds) > 0 { + rr.Excludes = append(rr.Excludes, ExcludeRule{Kinds: originalKinds}) + } } type APIGroupRule struct { @@ -79,8 +93,16 @@ type MetadataReplace struct { } type MetadataReplaceRule struct { - Original string `json:"original"` - Renamed string `json:"renamed"` + Original string `json:"original"` + Renamed string `json:"renamed"` + OriginalValue string `json:"originalValue"` + RenamedValue string `json:"renamedValue"` +} + +type ExcludeRule struct { + Kinds []string `json:"kinds"` + MatchNames []string `json:"matchNames"` + MatchLabels map[string]string `json:"matchLabels"` } // GetAPIGroupList returns an array of groups in format applicable to use in APIGroupList: @@ -288,3 +310,64 @@ func (rr *RewriteRules) AnnotationsRewriter() *PrefixedNameRewriter { func (rr *RewriteRules) FinalizersRewriter() *PrefixedNameRewriter { return rr.finalizersRewriter } + +// ShouldExclude returns true if object should be excluded from response back to the client. +// Set kind when obj has no kind, e.g. a list item. +func (rr *RewriteRules) ShouldExclude(obj []byte, kind string) bool { + for _, exclude := range rr.Excludes { + if exclude.Match(obj, kind) { + return true + } + } + return false +} + +// Match returns true if object matches all conditions in the exclude rule. +func (r ExcludeRule) Match(obj []byte, kind string) bool { + objKind := kind + if objKind == "" { + objKind = gjson.GetBytes(obj, "kind").String() + } + kindMatch := len(r.Kinds) == 0 + for _, kind := range r.Kinds { + if objKind == kind { + kindMatch = true + break + } + } + + objLabels := mapStringStringFromBytes(obj, "metadata.labels") + matchLabels := len(r.MatchLabels) == 0 || mapContainsMap(objLabels, r.MatchLabels) + + matchName := len(r.MatchNames) == 0 + objName := gjson.GetBytes(obj, "metadata.name").String() + for _, name := range r.MatchNames { + if objName == name { + matchName = true + break + } + } + + // Return true if every condition match. + return kindMatch && matchLabels && matchName +} + +func mapStringStringFromBytes(obj []byte, path string) map[string]string { + result := make(map[string]string) + for field, value := range gjson.GetBytes(obj, path).Map() { + result[field] = value.String() + } + return result +} + +func mapContainsMap(obj, match map[string]string) bool { + if len(match) == 0 { + return true + } + for k, v := range match { + if obj[k] != v { + return false + } + } + return true +} diff --git a/images/kube-api-proxy/pkg/rewriter/rules_test.go b/images/kube-api-proxy/pkg/rewriter/rules_test.go new file mode 100644 index 0000000000..44159600d2 --- /dev/null +++ b/images/kube-api-proxy/pkg/rewriter/rules_test.go @@ -0,0 +1,119 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rewriter + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func newTestExcludeRules() *RewriteRules { + rules := RewriteRules{ + Rules: map[string]APIGroupRule{ + "originalgroup.io": { + ResourceRules: map[string]ResourceRule{ + "someresources": { + Kind: "SomeResource", + ListKind: "SomeResourceList", + }, + }, + }, + "anothergroup.io": { + ResourceRules: map[string]ResourceRule{ + "anotheresources": { + Kind: "AnotherResource", + ListKind: "AnotherResourceList", + }, + }, + }, + }, + Excludes: []ExcludeRule{ + { + Kinds: []string{"RoleBinding"}, + MatchLabels: map[string]string{ + "labelName": "labelValue", + }, + }, + { + Kinds: []string{"Role"}, + MatchNames: []string{"role1", "role2"}, + }, + }, + } + rules.Init() + return &rules +} + +func TestExcludeRuleKindsOnly(t *testing.T) { + rules := newTestExcludeRules() + + tests := []struct { + name string + obj string + expectExcluded bool + }{ + { + "original kind SomeResource in excludes", + `{"kind":"SomeResource"}`, + true, + }, + { + "kind UnknownResource not in excludes", + `{"kind":"UnknownResource"}`, + false, + }, + { + "RoleBinding with label in excludes", + `{"kind":"RoleBinding","metadata":{"labels":{"labelName":"labelValue"}}}`, + true, + }, + { + "RoleBinding with label not in excludes", + `{"kind":"RoleBinding","metadata":{"labels":{"labelName":"nonExcludedValue"}}}`, + false, + }, + { + "Role with name in excludes", + `{"kind":"Role","metadata":{"name":"role1"}}`, + true, + }, + { + "Role with name not in excludes", + `{"kind":"Role","metadata":{"name":"role-not-excluded"}}`, + false, + }, + { + "RoleBinding with name as role in excludes", + `{"kind":"RoleBinding","metadata":{"name":"role1"}}`, + false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := rules.ShouldExclude([]byte(tt.obj), "") + + if tt.expectExcluded { + require.True(t, actual, "'%s' should be excluded. Not excluded obj: %s", tt.name, tt.obj) + } else { + require.False(t, actual, "'%s' should not be excluded. Excluded obj: %s", tt.name, tt.obj) + + } + }) + } +} diff --git a/images/virt-artifact/Taskfile.yaml b/images/virt-artifact/Taskfile.yaml index 876f6c504a..7583401cfe 100644 --- a/images/virt-artifact/Taskfile.yaml +++ b/images/virt-artifact/Taskfile.yaml @@ -12,6 +12,7 @@ tasks: - task: status status: + desc: "Show git status in cloned repo" cmds: - | dir=$(find . -type d -name __kubevirt_\* -depth 1 | head -n1) @@ -25,6 +26,7 @@ tasks: git status cleanup: + desc: "Remove cloned kubevirt git repo" cmds: - | find . -type d -name __kubevirt_\* -depth 1 diff --git a/images/virt-artifact/patches/015-rename-core-resources.patch b/images/virt-artifact/patches/015-rename-core-resources.patch index b9c50c273f..40bdfb3709 100644 --- a/images/virt-artifact/patches/015-rename-core-resources.patch +++ b/images/virt-artifact/patches/015-rename-core-resources.patch @@ -92,7 +92,7 @@ index 0948629bb..9aca3b3bd 100644 +++ b/pkg/virt-operator/resource/generate/components/serviceaccountnames.go @@ -1,9 +1,9 @@ package components - + const ( - ApiServiceAccountName = "kubevirt-apiserver" - ControllerServiceAccountName = "kubevirt-controller" @@ -104,6 +104,29 @@ index 0948629bb..9aca3b3bd 100644 + HandlerServiceAccountName = "kubevirt-internal-virtualization-handler" OperatorServiceAccountName = "kubevirt-operator" ) +diff --git a/pkg/virt-operator/resource/generate/components/webhooks.go b/pkg/virt-operator/resource/generate/components/webhooks.go +index 2600f8f39..22381d9e9 100644 +--- a/pkg/virt-operator/resource/generate/components/webhooks.go ++++ b/pkg/virt-operator/resource/generate/components/webhooks.go +@@ -833,15 +833,15 @@ const VirtHandlerServiceName = "virt-handler" + + const VirtExportProxyServiceName = "virt-exportproxy" + +-const VirtAPIValidatingWebhookName = "virt-api-validator" ++const VirtAPIValidatingWebhookName = "virt-internal-virtualization-api-validator" + + const VirtOperatorServiceName = "kubevirt-operator-webhook" + +-const VirtAPIMutatingWebhookName = "virt-api-mutator" ++const VirtAPIMutatingWebhookName = "virt-internal-virtualization-api-mutator" + + const KubevirtOperatorWebhookServiceName = "kubevirt-operator-webhook" + +-const KubeVirtOperatorValidatingWebhookName = "virt-operator-validator" ++const KubeVirtOperatorValidatingWebhookName = "virt-internal-virtualization-operator-validator" + + const VMSnapshotValidatePath = "/virtualmachinesnapshots-validate" + diff --git a/pkg/virt-operator/resource/generate/rbac/apiserver.go b/pkg/virt-operator/resource/generate/rbac/apiserver.go index 932f7391e..76c79d452 100644 --- a/pkg/virt-operator/resource/generate/rbac/apiserver.go @@ -166,13 +189,13 @@ index 071ed91f9..ebc9f2adb 100644 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "kubevirt.io/kubevirt/pkg/virt-operator/resource/generate/components" - + virtv1 "kubevirt.io/api/core/v1" ) - + -const ExportProxyServiceAccountName = "kubevirt-exportproxy" +const ExportProxyServiceAccountName = components.ExportProxyServiceAccountName - + func GetAllExportProxy(namespace string) []runtime.Object { return []runtime.Object{ diff --git a/pkg/virt-operator/resource/generate/rbac/operator.go b/pkg/virt-operator/resource/generate/rbac/operator.go diff --git a/images/virt-artifact/patches/016-rename-install-strategy-labels.patch b/images/virt-artifact/patches/016-rename-install-strategy-labels.patch index a5537813f9..977ad2d422 100644 --- a/images/virt-artifact/patches/016-rename-install-strategy-labels.patch +++ b/images/virt-artifact/patches/016-rename-install-strategy-labels.patch @@ -1,3 +1,16 @@ +diff --git a/pkg/virt-operator/resource/generate/install/strategy.go b/pkg/virt-operator/resource/generate/install/strategy.go +index 2f7c0c51b..f88108fdd 100644 +--- a/pkg/virt-operator/resource/generate/install/strategy.go ++++ b/pkg/virt-operator/resource/generate/install/strategy.go +@@ -253,7 +253,7 @@ func NewInstallStrategyConfigMap(config *operatorutil.KubeVirtDeploymentConfig, + GenerateName: "kubevirt-install-strategy-", + Namespace: config.GetNamespace(), + Labels: map[string]string{ +- v1.ManagedByLabel: v1.ManagedByLabelOperatorValue, ++ v1.ManagedByLabel: "virt-operator-internal-virtualization", + v1.InstallStrategyLabel: "", + }, + Annotations: map[string]string{ diff --git a/staging/src/kubevirt.io/api/core/v1/types.go b/staging/src/kubevirt.io/api/core/v1/types.go index cf7648a1f..e633c3919 100644 --- a/staging/src/kubevirt.io/api/core/v1/types.go diff --git a/images/virt-artifact/patches/018-rename-devices-kubevirt-io.patch b/images/virt-artifact/patches/018-rename-devices-kubevirt-io.patch new file mode 100644 index 0000000000..a4a43bec21 --- /dev/null +++ b/images/virt-artifact/patches/018-rename-devices-kubevirt-io.patch @@ -0,0 +1,49 @@ +diff --git a/pkg/virt-controller/services/template.go b/pkg/virt-controller/services/template.go +index d4124ea56..6285dacf6 100644 +--- a/pkg/virt-controller/services/template.go ++++ b/pkg/virt-controller/services/template.go +@@ -66,12 +66,12 @@ const ( + virtExporter = "virt-exporter" + ) + +-const KvmDevice = "devices.kubevirt.io/kvm" +-const TunDevice = "devices.kubevirt.io/tun" +-const VhostNetDevice = "devices.kubevirt.io/vhost-net" +-const SevDevice = "devices.kubevirt.io/sev" +-const VhostVsockDevice = "devices.kubevirt.io/vhost-vsock" +-const PrDevice = "devices.kubevirt.io/pr-helper" ++const KvmDevice = "devices.virtualization.deckhouse.io/kvm" ++const TunDevice = "devices.virtualization.deckhouse.io/tun" ++const VhostNetDevice = "devices.virtualization.deckhouse.io/vhost-net" ++const SevDevice = "devices.virtualization.deckhouse.io/sev" ++const VhostVsockDevice = "devices.virtualization.deckhouse.io/vhost-vsock" ++const PrDevice = "devices.virtualization.deckhouse.io/pr-helper" + + const debugLogs = "debugLogs" + const logVerbosity = "logVerbosity" +diff --git a/pkg/virt-handler/device-manager/common.go b/pkg/virt-handler/device-manager/common.go +index e3f86b117..d7a6d3456 100644 +--- a/pkg/virt-handler/device-manager/common.go ++++ b/pkg/virt-handler/device-manager/common.go +@@ -230,7 +230,7 @@ func gRPCConnect(socketPath string, timeout time.Duration) (*grpc.ClientConn, er + } + + func SocketPath(deviceName string) string { +- return filepath.Join(v1beta1.DevicePluginPath, fmt.Sprintf("kubevirt-%s.sock", deviceName)) ++ return filepath.Join(v1beta1.DevicePluginPath, fmt.Sprintf("virtualization-deckhouse-%s.sock", deviceName)) + } + + func IsChanClosed(ch <-chan struct{}) bool { +diff --git a/pkg/virt-handler/device-manager/generic_device.go b/pkg/virt-handler/device-manager/generic_device.go +index 7baceee6c..3b12f94fd 100644 +--- a/pkg/virt-handler/device-manager/generic_device.go ++++ b/pkg/virt-handler/device-manager/generic_device.go +@@ -41,7 +41,7 @@ import ( + ) + + const ( +- DeviceNamespace = "devices.kubevirt.io" ++ DeviceNamespace = "devices.virtualization.deckhouse.io" + connectionTimeout = 5 * time.Second + ) + diff --git a/images/virt-artifact/patches/README.md b/images/virt-artifact/patches/README.md index e5220e71f9..ffd8230974 100644 --- a/images/virt-artifact/patches/README.md +++ b/images/virt-artifact/patches/README.md @@ -62,6 +62,14 @@ Do not create Kubevirt APIService. #### `015-rename-core-resources.patch` Replace "kubevirt" with "kubevirt-internal-virtualziation" in the core resource names. +#### `016-rename-install-strategy-labels.patch` + +Rename kubevirt.io/install-strategy-registry labels to install.internal.virtualization.deckhouse.io/install-strategy-registry. +Rename app.kubernetes.io/managed-b value from virt-operator to virt-operator-internal-virtualization. + +Rewrite these labels with patch, because strategy generator Job starts without kube-api-rewriter. + + #### `016-rename-apigroups-in-starred-rbac.patch` Rename apiGroup to internal.virtualization.deckhouse.io for ClusterRole for virt-controller to prevent permanent patching: @@ -69,3 +77,11 @@ Rename apiGroup to internal.virtualization.deckhouse.io for ClusterRole for virt ``` {"component":"virt-operator","level":"info","msg":"clusterrole kubevirt-internal-virtualization-controller patched","pos":"core.go:142","timestamp":"2024-07-09T16:03:18.138751Z"} ``` + + +#### `018-rename-devices-kubevirt-io.patch` + +Rename additional resources previded with Device Plugin API to not overlap with original Kubevirt. + +Rename unix-socket path used for register devices. + diff --git a/templates/kubevirt/kubevirt.yaml b/templates/kubevirt/kubevirt.yaml index 16dafe4c6c..6162f616fc 100644 --- a/templates/kubevirt/kubevirt.yaml +++ b/templates/kubevirt/kubevirt.yaml @@ -124,7 +124,8 @@ spec: # Patch was produced with this jq command: # kubectl get validatingwebhookconfigurations.admissionregistration.k8s.io virt-api-validator -o json | jq '{"webhooks": .webhooks|map({"name":.name, "clientConfig":{"service":{"port":24192}}}) }' # virt-api-webhook-proxy service is created separately. - - resourceName: virt-api-validator + # NOTE virt-api-validator is renamed to virt-internal-virtualization-api-validator. + - resourceName: virt-internal-virtualization-api-validator resourceType: ValidatingWebhookConfiguration patch: | { @@ -212,7 +213,8 @@ spec: # Patch was produced with this jq command: # kubectl get mutatingwebhookconfigurations.admissionregistration.k8s.io virt-api-mutator -o json | jq '{"webhooks": .webhooks|map({"name":.name, "clientConfig":{"service":{"port":24192}}}) }' # virt-api-webhook-proxy service is created separately. - - resourceName: virt-api-mutator + # NOTE virt-api-mutator is renamed to virt-internal-virtualization-api-mutator. + - resourceName: virt-internal-virtualization-api-mutator resourceType: MutatingWebhookConfiguration patch: | { @@ -238,9 +240,10 @@ spec: type: strategic # Change service in webhook configuration to point to the rewriter proxy. # Patch was produced with this jq command: - # kubectl get validatingwebhookconfigurations.admissionregistration.k8s.io virt-operator-validator -o json | jq '{"webhooks": .webhooks|map({"name":.name, "clientConfig":{"service":{"port":24192}}}) }' + # kubectl get validatingwebhookconfigurations.admissionregistration.k8s.io virt-internal-virtualization-operator-validator -o json | jq '{"webhooks": .webhooks|map({"name":.name, "clientConfig":{"service":{"port":24192}}}) }' # kubevirt-operator-webhook-proxy service is created separately. - - resourceName: virt-operator-validator + # NOTE: original virt-operator-validator is renamed to virt-internal-virtualization-operator-validator. + - resourceName: virt-internal-virtualization-operator-validator resourceType: ValidatingWebhookConfiguration patch: | {