RUN-20245 Status updater topology configmap race fix (#87)

run-ai · Jul 17, 2024 · 031a001 · 031a001
1 parent bf6abc2
commit 031a001
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 1 deletion.
diff --git a/deploy/fake-gpu-operator/templates/status-updater/deployment.yaml b/deploy/fake-gpu-operator/templates/status-updater/deployment.yaml
@@ -14,6 +14,8 @@ spec:
   replicas: 1
   template:
     metadata:
+      annotations:
+        checksum/topology: {{ include (print $.Template.BasePath "/topology-cm.yml") . | sha256sum }}
       labels:
         app: status-updater
         component: status-updater

diff --git a/internal/status-updater/handlers/node/labels.go b/internal/status-updater/handlers/node/labels.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 )
@@ -34,7 +35,7 @@ func (p *NodeHandler) unlabelNode(node *v1.Node) error {
 		dcgmExporterLabelKey: nil,
 		devicePluginLabelKey: nil,
 	})
-	if err != nil {
+	if err != nil && !errors.IsNotFound(err) {
 		return fmt.Errorf("failed to unlabel node %s: %w", node.Name, err)
 	}