From a3018d113ca34f0fe8525e8d8c704616500c0bb9 Mon Sep 17 00:00:00 2001 From: Smriti Dahal Date: Tue, 11 Jun 2024 14:19:57 -0700 Subject: [PATCH 01/11] fix shield guard issues --- charts/virtual-kubelet/README.md | 4 ++-- charts/virtual-kubelet/templates/deployment.yaml | 8 ++++++++ charts/virtual-kubelet/values.yaml | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/charts/virtual-kubelet/README.md b/charts/virtual-kubelet/README.md index c5b1a22c..eff03e56 100644 --- a/charts/virtual-kubelet/README.md +++ b/charts/virtual-kubelet/README.md @@ -60,10 +60,10 @@ The following table lists the configurable parameters of the azure-aci chart and | image.repository | Image repository. | `mcr.microsoft.com` | | image.name | Image name. | `oss/virtual-kubelet/virtual-kubelet` | | image.tag | Image release version/tag. | `latest` | -| image.pullPolicy | Image pull policy. | `Always` | +| image.pullPolicy | Image pull policy. | `IfNotPresent` | | initImage.name | Init container image name. | `oss/virtual-kubelet/init-validation` | | initImage.initTag | Init container image release version/tag. | `0.2.0` | -| initImage.pullPolicy | Init container image pull policy. | `Always` | +| initImage.pullPolicy | Init container image pull policy. | `IfNotPresent` | | nodeName | The node name that will be assigned to be the VK one. | `virtual-node-aci-linux-helm` | | nodeOsType | The node/VM type. Values should be `Windows` or `Linux`. | `Linux` | | monitoredNamespace | Kubernetes namespace. default values means monitor `all` | `""` | diff --git a/charts/virtual-kubelet/templates/deployment.yaml b/charts/virtual-kubelet/templates/deployment.yaml index 27315dae..15b31567 100644 --- a/charts/virtual-kubelet/templates/deployment.yaml +++ b/charts/virtual-kubelet/templates/deployment.yaml @@ -21,6 +21,10 @@ spec: - name: init-validation image: "{{ .Values.initImage.repository }}/{{ .Values.initImage.name }}:{{ .Values.initImage.initTag }}" imagePullPolicy: {{ .Values.initImage.pullPolicy }} + securityContext: + capabilities: + drop: + - ALL env: - name: KUBELET_PORT value: "10250" @@ -111,6 +115,10 @@ spec: - name: {{ template "vk.fullname" . }} image: "{{ .Values.image.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + capabilities: + drop: + - ALL env: - name: KUBELET_PORT value: "10250" diff --git a/charts/virtual-kubelet/values.yaml b/charts/virtual-kubelet/values.yaml index c3f7ef15..96f2995a 100644 --- a/charts/virtual-kubelet/values.yaml +++ b/charts/virtual-kubelet/values.yaml @@ -3,13 +3,13 @@ image: repository: mcr.microsoft.com name: oss/virtual-kubelet/virtual-kubelet tag: 1.6.1 - pullPolicy: Always + pullPolicy: IfNotPresent initImage: repository: mcr.microsoft.com name: oss/virtual-kubelet/init-validation initTag: 0.2.0 - pullPolicy: Always + pullPolicy: IfNotPresent namespace: vk-azure-aci From 85bbd7132f3bc88bcbaf21dbd1db87fa73421db3 Mon Sep 17 00:00:00 2001 From: Smriti Dahal Date: Thu, 13 Jun 2024 14:25:05 -0700 Subject: [PATCH 02/11] added new clusterrole --- charts/virtual-kubelet/README.md | 4 ++-- .../templates/clusterrole.yaml | 20 +++++++++++++++++++ .../templates/clusterrolebinding.yaml | 2 +- .../virtual-kubelet/templates/deployment.yaml | 8 -------- charts/virtual-kubelet/values.yaml | 6 ++---- 5 files changed, 25 insertions(+), 15 deletions(-) create mode 100644 charts/virtual-kubelet/templates/clusterrole.yaml diff --git a/charts/virtual-kubelet/README.md b/charts/virtual-kubelet/README.md index eff03e56..c5b1a22c 100644 --- a/charts/virtual-kubelet/README.md +++ b/charts/virtual-kubelet/README.md @@ -60,10 +60,10 @@ The following table lists the configurable parameters of the azure-aci chart and | image.repository | Image repository. | `mcr.microsoft.com` | | image.name | Image name. | `oss/virtual-kubelet/virtual-kubelet` | | image.tag | Image release version/tag. | `latest` | -| image.pullPolicy | Image pull policy. | `IfNotPresent` | +| image.pullPolicy | Image pull policy. | `Always` | | initImage.name | Init container image name. | `oss/virtual-kubelet/init-validation` | | initImage.initTag | Init container image release version/tag. | `0.2.0` | -| initImage.pullPolicy | Init container image pull policy. | `IfNotPresent` | +| initImage.pullPolicy | Init container image pull policy. | `Always` | | nodeName | The node name that will be assigned to be the VK one. | `virtual-node-aci-linux-helm` | | nodeOsType | The node/VM type. Values should be `Windows` or `Linux`. | `Linux` | | monitoredNamespace | Kubernetes namespace. default values means monitor `all` | `""` | diff --git a/charts/virtual-kubelet/templates/clusterrole.yaml b/charts/virtual-kubelet/templates/clusterrole.yaml new file mode 100644 index 00000000..1ed5b527 --- /dev/null +++ b/charts/virtual-kubelet/templates/clusterrole.yaml @@ -0,0 +1,20 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "vk.fullname" . }}-clusterrole +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] +- apiGroups: [""] + resources: ["nodes", "pods/status", "nodes/status"] + verbs: ["get", "list", "watch", "update"] +- apiGroups: [""] + resources: ["persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"] + verbs: ["get", "list", "watch"] +- apiGroups: [ "" ] + resources: [ "services" ] + verbs: [ "get","list","watch"] +- apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get","list","watch","create", "delete", "update", "patch"] \ No newline at end of file diff --git a/charts/virtual-kubelet/templates/clusterrolebinding.yaml b/charts/virtual-kubelet/templates/clusterrolebinding.yaml index ba3acc21..a7330156 100644 --- a/charts/virtual-kubelet/templates/clusterrolebinding.yaml +++ b/charts/virtual-kubelet/templates/clusterrolebinding.yaml @@ -11,5 +11,5 @@ subjects: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: {{ .Values.rbac.roleRef }} + name: {{ include "vk.fullname" . }}-clusterrole {{ end }} diff --git a/charts/virtual-kubelet/templates/deployment.yaml b/charts/virtual-kubelet/templates/deployment.yaml index 15b31567..27315dae 100644 --- a/charts/virtual-kubelet/templates/deployment.yaml +++ b/charts/virtual-kubelet/templates/deployment.yaml @@ -21,10 +21,6 @@ spec: - name: init-validation image: "{{ .Values.initImage.repository }}/{{ .Values.initImage.name }}:{{ .Values.initImage.initTag }}" imagePullPolicy: {{ .Values.initImage.pullPolicy }} - securityContext: - capabilities: - drop: - - ALL env: - name: KUBELET_PORT value: "10250" @@ -115,10 +111,6 @@ spec: - name: {{ template "vk.fullname" . }} image: "{{ .Values.image.repository }}/{{ .Values.image.name }}:{{ .Values.image.tag }}" imagePullPolicy: {{ .Values.image.pullPolicy }} - securityContext: - capabilities: - drop: - - ALL env: - name: KUBELET_PORT value: "10250" diff --git a/charts/virtual-kubelet/values.yaml b/charts/virtual-kubelet/values.yaml index 96f2995a..30a1f538 100644 --- a/charts/virtual-kubelet/values.yaml +++ b/charts/virtual-kubelet/values.yaml @@ -3,13 +3,13 @@ image: repository: mcr.microsoft.com name: oss/virtual-kubelet/virtual-kubelet tag: 1.6.1 - pullPolicy: IfNotPresent + pullPolicy: Always initImage: repository: mcr.microsoft.com name: oss/virtual-kubelet/init-validation initTag: 0.2.0 - pullPolicy: IfNotPresent + pullPolicy: Always namespace: vk-azure-aci @@ -74,5 +74,3 @@ rbac: serviceAccountName: virtual-kubelet-helm ## RBAC api version apiVersion: v1 - ## Cluster role reference - roleRef: cluster-admin From 75948a9df861efc38954c9546a50090f4357ddac Mon Sep 17 00:00:00 2001 From: Smriti Dahal Date: Fri, 14 Jun 2024 12:06:27 -0700 Subject: [PATCH 03/11] indent fix --- charts/virtual-kubelet/templates/clusterrole.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/virtual-kubelet/templates/clusterrole.yaml b/charts/virtual-kubelet/templates/clusterrole.yaml index 1ed5b527..627eaaab 100644 --- a/charts/virtual-kubelet/templates/clusterrole.yaml +++ b/charts/virtual-kubelet/templates/clusterrole.yaml @@ -10,11 +10,11 @@ rules: resources: ["nodes", "pods/status", "nodes/status"] verbs: ["get", "list", "watch", "update"] - apiGroups: [""] - resources: ["persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"] - verbs: ["get", "list", "watch"] + resources: ["persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"] + verbs: ["get", "list", "watch"] - apiGroups: [ "" ] - resources: [ "services" ] - verbs: [ "get","list","watch"] + resources: [ "services" ] + verbs: [ "get","list","watch"] - apiGroups: ["apps"] - resources: ["deployments"] - verbs: ["get","list","watch","create", "delete", "update", "patch"] \ No newline at end of file + resources: ["deployments"] + verbs: ["get","list","watch","create", "delete", "update", "patch"] \ No newline at end of file From 343d3cbf862b1b1b2aa0010d19a04449a192c8b9 Mon Sep 17 00:00:00 2001 From: Smriti Dahal Date: Mon, 17 Jun 2024 10:52:56 -0700 Subject: [PATCH 04/11] testing e2e --- e2e/pods_win_test.go | 152 +++++++++++++++++++++--------------------- hack/e2e/aks-addon.sh | 50 +++++++------- 2 files changed, 101 insertions(+), 101 deletions(-) diff --git a/e2e/pods_win_test.go b/e2e/pods_win_test.go index e0bec8dc..3764387a 100644 --- a/e2e/pods_win_test.go +++ b/e2e/pods_win_test.go @@ -1,87 +1,87 @@ package e2e -import ( - "fmt" - "os" - "os/exec" - "testing" - "time" -) +// import ( +// "fmt" +// "os" +// "os/exec" +// "testing" +// "time" +// ) -func TestWindowsPodLifecycle(t *testing.T) { - // delete the namespace first - cmd := kubectl("delete", "namespace", "vk-test", "--ignore-not-found") - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatal(string(out)) - } +// func TestWindowsPodLifecycle(t *testing.T) { +// // delete the namespace first +// cmd := kubectl("delete", "namespace", "vk-test", "--ignore-not-found") +// if out, err := cmd.CombinedOutput(); err != nil { +// t.Fatal(string(out)) +// } - // create namespace - cmd = kubectl("apply", "-f", "fixtures/namespace.yml") - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatal(string(out)) - } +// // create namespace +// cmd = kubectl("apply", "-f", "fixtures/namespace.yml") +// if out, err := cmd.CombinedOutput(); err != nil { +// t.Fatal(string(out)) +// } - winNodeName := os.Getenv("TEST_WINDOWS_NODE_NAME") - cmd = exec.Command("sed", "-i", "-e", fmt.Sprintf("s|TEST_WINDOWS_NODE_NAME|%s|g", winNodeName), "fixtures/win-pod.yml") - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatal(string(out)) - } +// winNodeName := os.Getenv("TEST_WINDOWS_NODE_NAME") +// cmd = exec.Command("sed", "-i", "-e", fmt.Sprintf("s|TEST_WINDOWS_NODE_NAME|%s|g", winNodeName), "fixtures/win-pod.yml") +// if out, err := cmd.CombinedOutput(); err != nil { +// t.Fatal(string(out)) +// } - cmd = kubectl("apply", "-f", "fixtures/win-pod.yml", "--namespace=vk-test") - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatal(string(out)) - } +// cmd = kubectl("apply", "-f", "fixtures/win-pod.yml", "--namespace=vk-test") +// if out, err := cmd.CombinedOutput(); err != nil { +// t.Fatal(string(out)) +// } - deadline, ok := t.Deadline() - timeout := time.Until(deadline) - if !ok { - timeout = 300 * time.Second - } - cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/vk-e2e-windows", "--namespace=vk-test") - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatal(string(out)) - } - t.Log("success create pod") +// deadline, ok := t.Deadline() +// timeout := time.Until(deadline) +// if !ok { +// timeout = 300 * time.Second +// } +// cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/vk-e2e-windows", "--namespace=vk-test") +// if out, err := cmd.CombinedOutput(); err != nil { +// t.Fatal(string(out)) +// } +// t.Log("success create pod") - // check container logs - t.Log("get container logs ....") - cmd = kubectl("logs", "pod/vk-e2e-windows", "--namespace=vk-test", "--tail=5") - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - if string(out) == "" { - t.Fatal("failed to get container's logs") - } - t.Logf("success query container logs %s", string(out)) +// // check container logs +// t.Log("get container logs ....") +// cmd = kubectl("logs", "pod/vk-e2e-windows", "--namespace=vk-test", "--tail=5") +// out, err := cmd.CombinedOutput() +// if err != nil { +// t.Fatal(string(out)) +// } +// if string(out) == "" { +// t.Fatal("failed to get container's logs") +// } +// t.Logf("success query container logs %s", string(out)) - // check pod status - t.Log("get pod status ....") - cmd = kubectl("get", "pod", "--field-selector=status.phase=Running", "--namespace=vk-test", "--output=jsonpath={.items..metadata.name}") - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - if string(out) != "vk-e2e-windows" { - t.Fatal("failed to get pod's status") - } - t.Logf("success query pod status %s", string(out)) +// // check pod status +// t.Log("get pod status ....") +// cmd = kubectl("get", "pod", "--field-selector=status.phase=Running", "--namespace=vk-test", "--output=jsonpath={.items..metadata.name}") +// out, err = cmd.CombinedOutput() +// if err != nil { +// t.Fatal(string(out)) +// } +// if string(out) != "vk-e2e-windows" { +// t.Fatal("failed to get pod's status") +// } +// t.Logf("success query pod status %s", string(out)) - // check container status - t.Log("get container status ....") - cmd = kubectl("get", "pod", "vk-e2e-windows", "--namespace=vk-test", "--output=jsonpath={.status.containerStatuses[0].ready}") - out, err = cmd.CombinedOutput() - if err != nil { - t.Fatal(string(out)) - } - if string(out) != "true" { - t.Fatal("failed to get pod's status") - } - t.Logf("success query container status %s", string(out)) +// // check container status +// t.Log("get container status ....") +// cmd = kubectl("get", "pod", "vk-e2e-windows", "--namespace=vk-test", "--output=jsonpath={.status.containerStatuses[0].ready}") +// out, err = cmd.CombinedOutput() +// if err != nil { +// t.Fatal(string(out)) +// } +// if string(out) != "true" { +// t.Fatal("failed to get pod's status") +// } +// t.Logf("success query container status %s", string(out)) - t.Log("clean up pod") - cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found") - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatal(string(out)) - } -} +// t.Log("clean up pod") +// cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found") +// if out, err := cmd.CombinedOutput(); err != nil { +// t.Fatal(string(out)) +// } +// } diff --git a/hack/e2e/aks-addon.sh b/hack/e2e/aks-addon.sh index 401cc830..bad1eba4 100755 --- a/hack/e2e/aks-addon.sh +++ b/hack/e2e/aks-addon.sh @@ -177,31 +177,31 @@ kubectl wait --for=condition=Ready --timeout=300s node "$TEST_NODE_NAME" export TEST_NODE_NAME ## Windows VK -helm install \ - --kubeconfig="${KUBECONFIG}" \ - --set nodeOsType=Windows \ - --set "image.repository=${IMG_URL}" \ - --set "image.tag=${IMG_TAG}" \ - --set "image.name=${IMG_REPO}" \ - --set "initImage.repository=${IMG_URL}" \ - --set "initImage.name=${INIT_IMG_REPO}" \ - --set "initImage.tag=${INIT_IMG_TAG}" \ - --set "nodeName=${TEST_WINDOWS_NODE_NAME}" \ - --set "providers.azure.masterUri=$MASTER_URI" \ - --set "providers.azure.managedIdentityID=$ACI_USER_IDENTITY" \ - "$WIN_CHART_NAME" \ - ./charts/virtual-kubelet - -kubectl wait --for=condition=available deploy "${TEST_WINDOWS_NODE_NAME}-virtual-kubelet-azure-aci" -n vk-azure-aci --timeout=500s - -while true; do - kubectl get node "$TEST_WINDOWS_NODE_NAME" &> /dev/null && break - sleep 3 -done - -kubectl wait --for=condition=Ready --timeout=300s node "$TEST_WINDOWS_NODE_NAME" - -export TEST_WINDOWS_NODE_NAME=$TEST_WINDOWS_NODE_NAME +# helm install \ +# --kubeconfig="${KUBECONFIG}" \ +# --set nodeOsType=Windows \ +# --set "image.repository=${IMG_URL}" \ +# --set "image.tag=${IMG_TAG}" \ +# --set "image.name=${IMG_REPO}" \ +# --set "initImage.repository=${IMG_URL}" \ +# --set "initImage.name=${INIT_IMG_REPO}" \ +# --set "initImage.tag=${INIT_IMG_TAG}" \ +# --set "nodeName=${TEST_WINDOWS_NODE_NAME}" \ +# --set "providers.azure.masterUri=$MASTER_URI" \ +# --set "providers.azure.managedIdentityID=$ACI_USER_IDENTITY" \ +# "$WIN_CHART_NAME" \ +# ./charts/virtual-kubelet + +# kubectl wait --for=condition=available deploy "${TEST_WINDOWS_NODE_NAME}-virtual-kubelet-azure-aci" -n vk-azure-aci --timeout=500s + +# while true; do +# kubectl get node "$TEST_WINDOWS_NODE_NAME" &> /dev/null && break +# sleep 3 +# done + +# kubectl wait --for=condition=Ready --timeout=300s node "$TEST_WINDOWS_NODE_NAME" + +# export TEST_WINDOWS_NODE_NAME=$TEST_WINDOWS_NODE_NAME ## CSI Driver test az storage account create -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g $RESOURCE_GROUP -l $LOCATION --sku Standard_LRS From 929460ef34bf3bd5239d02ee9f39ee8a94a92a25 Mon Sep 17 00:00:00 2001 From: Smriti Dahal Date: Mon, 17 Jun 2024 11:29:53 -0700 Subject: [PATCH 05/11] testing windows vk --- e2e/pods_win_test.go | 152 +++++++++++++++++++++--------------------- hack/e2e/aks-addon.sh | 62 ++++++++--------- 2 files changed, 107 insertions(+), 107 deletions(-) diff --git a/e2e/pods_win_test.go b/e2e/pods_win_test.go index 3764387a..e0bec8dc 100644 --- a/e2e/pods_win_test.go +++ b/e2e/pods_win_test.go @@ -1,87 +1,87 @@ package e2e -// import ( -// "fmt" -// "os" -// "os/exec" -// "testing" -// "time" -// ) +import ( + "fmt" + "os" + "os/exec" + "testing" + "time" +) -// func TestWindowsPodLifecycle(t *testing.T) { -// // delete the namespace first -// cmd := kubectl("delete", "namespace", "vk-test", "--ignore-not-found") -// if out, err := cmd.CombinedOutput(); err != nil { -// t.Fatal(string(out)) -// } +func TestWindowsPodLifecycle(t *testing.T) { + // delete the namespace first + cmd := kubectl("delete", "namespace", "vk-test", "--ignore-not-found") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } -// // create namespace -// cmd = kubectl("apply", "-f", "fixtures/namespace.yml") -// if out, err := cmd.CombinedOutput(); err != nil { -// t.Fatal(string(out)) -// } + // create namespace + cmd = kubectl("apply", "-f", "fixtures/namespace.yml") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } -// winNodeName := os.Getenv("TEST_WINDOWS_NODE_NAME") -// cmd = exec.Command("sed", "-i", "-e", fmt.Sprintf("s|TEST_WINDOWS_NODE_NAME|%s|g", winNodeName), "fixtures/win-pod.yml") -// if out, err := cmd.CombinedOutput(); err != nil { -// t.Fatal(string(out)) -// } + winNodeName := os.Getenv("TEST_WINDOWS_NODE_NAME") + cmd = exec.Command("sed", "-i", "-e", fmt.Sprintf("s|TEST_WINDOWS_NODE_NAME|%s|g", winNodeName), "fixtures/win-pod.yml") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } -// cmd = kubectl("apply", "-f", "fixtures/win-pod.yml", "--namespace=vk-test") -// if out, err := cmd.CombinedOutput(); err != nil { -// t.Fatal(string(out)) -// } + cmd = kubectl("apply", "-f", "fixtures/win-pod.yml", "--namespace=vk-test") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } -// deadline, ok := t.Deadline() -// timeout := time.Until(deadline) -// if !ok { -// timeout = 300 * time.Second -// } -// cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/vk-e2e-windows", "--namespace=vk-test") -// if out, err := cmd.CombinedOutput(); err != nil { -// t.Fatal(string(out)) -// } -// t.Log("success create pod") + deadline, ok := t.Deadline() + timeout := time.Until(deadline) + if !ok { + timeout = 300 * time.Second + } + cmd = kubectl("wait", "--for=condition=ready", "--timeout="+timeout.String(), "pod/vk-e2e-windows", "--namespace=vk-test") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } + t.Log("success create pod") -// // check container logs -// t.Log("get container logs ....") -// cmd = kubectl("logs", "pod/vk-e2e-windows", "--namespace=vk-test", "--tail=5") -// out, err := cmd.CombinedOutput() -// if err != nil { -// t.Fatal(string(out)) -// } -// if string(out) == "" { -// t.Fatal("failed to get container's logs") -// } -// t.Logf("success query container logs %s", string(out)) + // check container logs + t.Log("get container logs ....") + cmd = kubectl("logs", "pod/vk-e2e-windows", "--namespace=vk-test", "--tail=5") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + if string(out) == "" { + t.Fatal("failed to get container's logs") + } + t.Logf("success query container logs %s", string(out)) -// // check pod status -// t.Log("get pod status ....") -// cmd = kubectl("get", "pod", "--field-selector=status.phase=Running", "--namespace=vk-test", "--output=jsonpath={.items..metadata.name}") -// out, err = cmd.CombinedOutput() -// if err != nil { -// t.Fatal(string(out)) -// } -// if string(out) != "vk-e2e-windows" { -// t.Fatal("failed to get pod's status") -// } -// t.Logf("success query pod status %s", string(out)) + // check pod status + t.Log("get pod status ....") + cmd = kubectl("get", "pod", "--field-selector=status.phase=Running", "--namespace=vk-test", "--output=jsonpath={.items..metadata.name}") + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + if string(out) != "vk-e2e-windows" { + t.Fatal("failed to get pod's status") + } + t.Logf("success query pod status %s", string(out)) -// // check container status -// t.Log("get container status ....") -// cmd = kubectl("get", "pod", "vk-e2e-windows", "--namespace=vk-test", "--output=jsonpath={.status.containerStatuses[0].ready}") -// out, err = cmd.CombinedOutput() -// if err != nil { -// t.Fatal(string(out)) -// } -// if string(out) != "true" { -// t.Fatal("failed to get pod's status") -// } -// t.Logf("success query container status %s", string(out)) + // check container status + t.Log("get container status ....") + cmd = kubectl("get", "pod", "vk-e2e-windows", "--namespace=vk-test", "--output=jsonpath={.status.containerStatuses[0].ready}") + out, err = cmd.CombinedOutput() + if err != nil { + t.Fatal(string(out)) + } + if string(out) != "true" { + t.Fatal("failed to get pod's status") + } + t.Logf("success query container status %s", string(out)) -// t.Log("clean up pod") -// cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found") -// if out, err := cmd.CombinedOutput(); err != nil { -// t.Fatal(string(out)) -// } -// } + t.Log("clean up pod") + cmd = kubectl("delete", "namespace", "vk-test", "--ignore-not-found") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatal(string(out)) + } +} diff --git a/hack/e2e/aks-addon.sh b/hack/e2e/aks-addon.sh index bad1eba4..947b1f07 100755 --- a/hack/e2e/aks-addon.sh +++ b/hack/e2e/aks-addon.sh @@ -177,40 +177,40 @@ kubectl wait --for=condition=Ready --timeout=300s node "$TEST_NODE_NAME" export TEST_NODE_NAME ## Windows VK -# helm install \ -# --kubeconfig="${KUBECONFIG}" \ -# --set nodeOsType=Windows \ -# --set "image.repository=${IMG_URL}" \ -# --set "image.tag=${IMG_TAG}" \ -# --set "image.name=${IMG_REPO}" \ -# --set "initImage.repository=${IMG_URL}" \ -# --set "initImage.name=${INIT_IMG_REPO}" \ -# --set "initImage.tag=${INIT_IMG_TAG}" \ -# --set "nodeName=${TEST_WINDOWS_NODE_NAME}" \ -# --set "providers.azure.masterUri=$MASTER_URI" \ -# --set "providers.azure.managedIdentityID=$ACI_USER_IDENTITY" \ -# "$WIN_CHART_NAME" \ -# ./charts/virtual-kubelet - -# kubectl wait --for=condition=available deploy "${TEST_WINDOWS_NODE_NAME}-virtual-kubelet-azure-aci" -n vk-azure-aci --timeout=500s - -# while true; do -# kubectl get node "$TEST_WINDOWS_NODE_NAME" &> /dev/null && break -# sleep 3 -# done - -# kubectl wait --for=condition=Ready --timeout=300s node "$TEST_WINDOWS_NODE_NAME" - -# export TEST_WINDOWS_NODE_NAME=$TEST_WINDOWS_NODE_NAME +helm install \ + --kubeconfig="${KUBECONFIG}" \ + --set nodeOsType=Windows \ + --set "image.repository=${IMG_URL}" \ + --set "image.tag=${IMG_TAG}" \ + --set "image.name=${IMG_REPO}" \ + --set "initImage.repository=${IMG_URL}" \ + --set "initImage.name=${INIT_IMG_REPO}" \ + --set "initImage.tag=${INIT_IMG_TAG}" \ + --set "nodeName=${TEST_WINDOWS_NODE_NAME}" \ + --set "providers.azure.masterUri=$MASTER_URI" \ + --set "providers.azure.managedIdentityID=$ACI_USER_IDENTITY" \ + "$WIN_CHART_NAME" \ + ./charts/virtual-kubelet + +kubectl wait --for=condition=available deploy "${TEST_WINDOWS_NODE_NAME}-virtual-kubelet-azure-aci" -n vk-azure-aci --timeout=500s + +while true; do + kubectl get node "$TEST_WINDOWS_NODE_NAME" &> /dev/null && break + sleep 3 +done + +kubectl wait --for=condition=Ready --timeout=300s node "$TEST_WINDOWS_NODE_NAME" + +export TEST_WINDOWS_NODE_NAME=$TEST_WINDOWS_NODE_NAME ## CSI Driver test -az storage account create -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g $RESOURCE_GROUP -l $LOCATION --sku Standard_LRS -export AZURE_STORAGE_CONNECTION_STRING=$(az storage account show-connection-string -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g "$RESOURCE_GROUP" -o tsv) +# az storage account create -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g $RESOURCE_GROUP -l $LOCATION --sku Standard_LRS +# export AZURE_STORAGE_CONNECTION_STRING=$(az storage account show-connection-string -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g "$RESOURCE_GROUP" -o tsv) -az storage share create -n "$CSI_DRIVER_SHARE_NAME" --connection-string "$AZURE_STORAGE_CONNECTION_STRING" -CSI_DRIVER_STORAGE_ACCOUNT_KEY=$(az storage account keys list --resource-group "$RESOURCE_GROUP" --account-name "$CSI_DRIVER_STORAGE_ACCOUNT_NAME" --query "[0].value" -o tsv) +# az storage share create -n "$CSI_DRIVER_SHARE_NAME" --connection-string "$AZURE_STORAGE_CONNECTION_STRING" +# CSI_DRIVER_STORAGE_ACCOUNT_KEY=$(az storage account keys list --resource-group "$RESOURCE_GROUP" --account-name "$CSI_DRIVER_STORAGE_ACCOUNT_NAME" --query "[0].value" -o tsv) -export CSI_DRIVER_STORAGE_ACCOUNT_NAME=$CSI_DRIVER_STORAGE_ACCOUNT_NAME -export CSI_DRIVER_STORAGE_ACCOUNT_KEY=$CSI_DRIVER_STORAGE_ACCOUNT_KEY +# export CSI_DRIVER_STORAGE_ACCOUNT_NAME=$CSI_DRIVER_STORAGE_ACCOUNT_NAME +# export CSI_DRIVER_STORAGE_ACCOUNT_KEY=$CSI_DRIVER_STORAGE_ACCOUNT_KEY $@ From eff36bb2291287cdb9ce3df643fe47792b210741 Mon Sep 17 00:00:00 2001 From: Smriti Dahal Date: Mon, 17 Jun 2024 11:51:52 -0700 Subject: [PATCH 06/11] updating clusterrole --- charts/virtual-kubelet/templates/clusterrole.yaml | 2 +- hack/e2e/aks-addon.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/charts/virtual-kubelet/templates/clusterrole.yaml b/charts/virtual-kubelet/templates/clusterrole.yaml index 627eaaab..7151043f 100644 --- a/charts/virtual-kubelet/templates/clusterrole.yaml +++ b/charts/virtual-kubelet/templates/clusterrole.yaml @@ -10,7 +10,7 @@ rules: resources: ["nodes", "pods/status", "nodes/status"] verbs: ["get", "list", "watch", "update"] - apiGroups: [""] - resources: ["persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"] + resources: ["persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces", "configmaps", "secrets"] verbs: ["get", "list", "watch"] - apiGroups: [ "" ] resources: [ "services" ] diff --git a/hack/e2e/aks-addon.sh b/hack/e2e/aks-addon.sh index 947b1f07..401cc830 100755 --- a/hack/e2e/aks-addon.sh +++ b/hack/e2e/aks-addon.sh @@ -204,13 +204,13 @@ kubectl wait --for=condition=Ready --timeout=300s node "$TEST_WINDOWS_NODE_NAME" export TEST_WINDOWS_NODE_NAME=$TEST_WINDOWS_NODE_NAME ## CSI Driver test -# az storage account create -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g $RESOURCE_GROUP -l $LOCATION --sku Standard_LRS -# export AZURE_STORAGE_CONNECTION_STRING=$(az storage account show-connection-string -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g "$RESOURCE_GROUP" -o tsv) +az storage account create -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g $RESOURCE_GROUP -l $LOCATION --sku Standard_LRS +export AZURE_STORAGE_CONNECTION_STRING=$(az storage account show-connection-string -n $CSI_DRIVER_STORAGE_ACCOUNT_NAME -g "$RESOURCE_GROUP" -o tsv) -# az storage share create -n "$CSI_DRIVER_SHARE_NAME" --connection-string "$AZURE_STORAGE_CONNECTION_STRING" -# CSI_DRIVER_STORAGE_ACCOUNT_KEY=$(az storage account keys list --resource-group "$RESOURCE_GROUP" --account-name "$CSI_DRIVER_STORAGE_ACCOUNT_NAME" --query "[0].value" -o tsv) +az storage share create -n "$CSI_DRIVER_SHARE_NAME" --connection-string "$AZURE_STORAGE_CONNECTION_STRING" +CSI_DRIVER_STORAGE_ACCOUNT_KEY=$(az storage account keys list --resource-group "$RESOURCE_GROUP" --account-name "$CSI_DRIVER_STORAGE_ACCOUNT_NAME" --query "[0].value" -o tsv) -# export CSI_DRIVER_STORAGE_ACCOUNT_NAME=$CSI_DRIVER_STORAGE_ACCOUNT_NAME -# export CSI_DRIVER_STORAGE_ACCOUNT_KEY=$CSI_DRIVER_STORAGE_ACCOUNT_KEY +export CSI_DRIVER_STORAGE_ACCOUNT_NAME=$CSI_DRIVER_STORAGE_ACCOUNT_NAME +export CSI_DRIVER_STORAGE_ACCOUNT_KEY=$CSI_DRIVER_STORAGE_ACCOUNT_KEY $@ From fa66b39bf937954724ace6839301fd7b452951b1 Mon Sep 17 00:00:00 2001 From: Smriti Dahal Date: Mon, 17 Jun 2024 12:21:58 -0700 Subject: [PATCH 07/11] updating clusterrole --- charts/virtual-kubelet/templates/clusterrole.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/virtual-kubelet/templates/clusterrole.yaml b/charts/virtual-kubelet/templates/clusterrole.yaml index 7151043f..dca6f996 100644 --- a/charts/virtual-kubelet/templates/clusterrole.yaml +++ b/charts/virtual-kubelet/templates/clusterrole.yaml @@ -4,10 +4,10 @@ metadata: name: {{ include "vk.fullname" . }}-clusterrole rules: - apiGroups: [""] - resources: ["pods"] + resources: ["pods, nodes"] verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] - apiGroups: [""] - resources: ["nodes", "pods/status", "nodes/status"] + resources: ["pods/status", "nodes/status"] verbs: ["get", "list", "watch", "update"] - apiGroups: [""] resources: ["persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces", "configmaps", "secrets"] From 76ace5061c2c6fc0ecb70aaae19a72ba06e05483 Mon Sep 17 00:00:00 2001 From: Smriti Dahal Date: Wed, 19 Jun 2024 13:47:39 -0700 Subject: [PATCH 08/11] update clusterrole --- charts/virtual-kubelet/README.md | 1 - .../templates/clusterrole.yaml | 33 ++++++++++--------- .../templates/clusterrolebinding.yaml | 6 ++-- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/charts/virtual-kubelet/README.md b/charts/virtual-kubelet/README.md index eff03e56..a9f3e64c 100644 --- a/charts/virtual-kubelet/README.md +++ b/charts/virtual-kubelet/README.md @@ -103,4 +103,3 @@ The following table lists the configurable parameters of the azure-aci chart and | rbac.install | Install Default RBAC roles and bindings. | `true` | | rbac.serviceAccountName | RBAC service account name. | `virtual-kubelet-helm` | | rbac.apiVersion | RBAC api version. | `v1` | -| rbac.roleRef | Cluster role reference. | `cluster-admin` | diff --git a/charts/virtual-kubelet/templates/clusterrole.yaml b/charts/virtual-kubelet/templates/clusterrole.yaml index dca6f996..60963fe3 100644 --- a/charts/virtual-kubelet/templates/clusterrole.yaml +++ b/charts/virtual-kubelet/templates/clusterrole.yaml @@ -1,20 +1,21 @@ -apiVersion: rbac.authorization.k8s.io/v1 +apiVersion: "rbac.authorization.k8s.io/{{ .Values.rbac.apiVersion }}" kind: ClusterRole metadata: name: {{ include "vk.fullname" . }}-clusterrole +{{ include "vk.labels" . | indent 2 }} rules: -- apiGroups: [""] - resources: ["pods, nodes"] - verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] -- apiGroups: [""] - resources: ["pods/status", "nodes/status"] - verbs: ["get", "list", "watch", "update"] -- apiGroups: [""] - resources: ["persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces", "configmaps", "secrets"] - verbs: ["get", "list", "watch"] -- apiGroups: [ "" ] - resources: [ "services" ] - verbs: [ "get","list","watch"] -- apiGroups: ["apps"] - resources: ["deployments"] - verbs: ["get","list","watch","create", "delete", "update", "patch"] \ No newline at end of file + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch", "create", "delete", "update", "patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["pods/status", "nodes/status"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces", "configmaps", "secrets", "services"] + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get","list","watch","create", "delete", "update", "patch"] \ No newline at end of file diff --git a/charts/virtual-kubelet/templates/clusterrolebinding.yaml b/charts/virtual-kubelet/templates/clusterrolebinding.yaml index a7330156..3a189231 100644 --- a/charts/virtual-kubelet/templates/clusterrolebinding.yaml +++ b/charts/virtual-kubelet/templates/clusterrolebinding.yaml @@ -5,9 +5,9 @@ metadata: name: {{ template "vk.fullname" . }}-{{ .Values.rbac.serviceAccountName }} {{ include "vk.labels" . | indent 2 }} subjects: -- kind: ServiceAccount - name: {{ template "vk.fullname" . }}-{{ .Values.rbac.serviceAccountName }} - namespace: {{ .Values.namespace }} + - kind: ServiceAccount + name: {{ template "vk.fullname" . }}-{{ .Values.rbac.serviceAccountName }} + namespace: {{ .Values.namespace }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole From caa1a7e0fd73b0672746b86281b50505a107f640 Mon Sep 17 00:00:00 2001 From: Smriti Dahal Date: Wed, 19 Jun 2024 16:03:57 -0700 Subject: [PATCH 09/11] clusterrole update --- charts/virtual-kubelet/templates/clusterrole.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/virtual-kubelet/templates/clusterrole.yaml b/charts/virtual-kubelet/templates/clusterrole.yaml index 60963fe3..5095bceb 100644 --- a/charts/virtual-kubelet/templates/clusterrole.yaml +++ b/charts/virtual-kubelet/templates/clusterrole.yaml @@ -12,7 +12,7 @@ rules: verbs: ["get", "list", "watch", "create", "update", "patch"] - apiGroups: [""] resources: ["pods/status", "nodes/status"] - verbs: ["get", "list", "watch", "update"] + verbs: ["get", "list", "watch", "update", "patch"] - apiGroups: [""] resources: ["persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces", "configmaps", "secrets", "services"] verbs: ["get", "list", "watch"] From c31e3fc7aae388cd57c2988c0ecb14c6a40978b6 Mon Sep 17 00:00:00 2001 From: Smriti Dahal Date: Wed, 19 Jun 2024 19:47:24 -0700 Subject: [PATCH 10/11] update clusterrole --- charts/virtual-kubelet/templates/clusterrole.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/charts/virtual-kubelet/templates/clusterrole.yaml b/charts/virtual-kubelet/templates/clusterrole.yaml index 5095bceb..7d7ea732 100644 --- a/charts/virtual-kubelet/templates/clusterrole.yaml +++ b/charts/virtual-kubelet/templates/clusterrole.yaml @@ -18,4 +18,7 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: ["apps"] resources: ["deployments"] - verbs: ["get","list","watch","create", "delete", "update", "patch"] \ No newline at end of file + verbs: ["get","list","watch","create", "delete", "update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "create", "update"] \ No newline at end of file From da5085721562ffbd3d719a1593e1c34431482c2d Mon Sep 17 00:00:00 2001 From: Smriti Dahal Date: Thu, 20 Jun 2024 19:14:32 -0700 Subject: [PATCH 11/11] updating sku --- hack/e2e/aks-addon.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/e2e/aks-addon.sh b/hack/e2e/aks-addon.sh index 401cc830..0dd7fe08 100755 --- a/hack/e2e/aks-addon.sh +++ b/hack/e2e/aks-addon.sh @@ -114,7 +114,7 @@ az aks create \ -g "$RESOURCE_GROUP" \ -l "$LOCATION" \ -c "$NODE_COUNT" \ - --node-vm-size standard_d8_v3 \ + --node-vm-size standard_d8s_v3 \ -n "$CLUSTER_NAME" \ --network-plugin azure \ --vnet-subnet-id "$cluster_subnet_id" \ @@ -128,7 +128,7 @@ az aks create \ -g "$RESOURCE_GROUP" \ -l "$LOCATION" \ -c "$NODE_COUNT" \ - --node-vm-size standard_d8_v3 \ + --node-vm-size standard_d8s_v3 \ -n "$CLUSTER_NAME" \ --network-plugin azure \ --vnet-subnet-id "$cluster_subnet_id" \