Telnet on TLS (#103)

.github/RELEASE_TEMPLATE.md

@@ -44,7 +44,7 @@ spec:
     coolDown: 10m
 EOF
 
-kubectl apply create deployment --image=nginx nginx
+kubectl create deployment --image=nginx nginx
 ```
 
 ### Build your own version

Dockerfile

@@ -1,3 +1,11 @@
+# Build CSI drivers
+FROM tinygo/tinygo@sha256:65dc1c3e54f88aabe1efe073c3aadb1393593a56355a6ac03df5f18e6c3855dd as drivers
+
+COPY drivers/ /go/src
+
+RUN cd /go/src/csi.storageos.com ; go mod tidy && tinygo build -o main.wasm -target wasi --no-debug main.go
+RUN cd /go/src/ebs.csi.aws.com ; go mod tidy && tinygo build -o main.wasm -target wasi --no-debug main.go
+
 # Build the manager binary
 FROM golang@sha256:5b75b529da0f2196ee8561a90e5b99aceee56e125c6ef09a3da4e32cf3cc6c20 as builder
 
@@ -20,27 +28,19 @@ COPY schedulers/ schedulers/
 # Build
 RUN GOOS=linux GOARCH=amd64 go build -a -o manager main.go
 
-# Build CSI drivers
-FROM tinygo/tinygo@sha256:65dc1c3e54f88aabe1efe073c3aadb1393593a56355a6ac03df5f18e6c3855dd as drivers
-
-COPY drivers/ /go/src
-
-RUN cd /go/src/csi.storageos.com ; go mod tidy && tinygo build -o main.wasm -target wasi --no-debug main.go
-RUN cd /go/src/ebs.csi.aws.com ; go mod tidy && tinygo build -o main.wasm -target wasi --no-debug main.go
-
 # Use UBI as minimal base image to package the manager binary
 FROM redhat/ubi8-micro@sha256:4f6f8db9a6dc949d9779a57c43954b251957bd4d019a37edbbde8ed5228fe90a
 
-LABEL org.opencontainers.image.title "Discoblocks" 
-LABEL org.opencontainers.image.vendor "Discoblocks.io" 
-LABEL org.opencontainers.image.licenses "Apache-2.0 License" 
-LABEL org.opencontainers.image.source "https://github.com/ondat/discoblocks" 
-LABEL org.opencontainers.image.description "Discoblocks is an open-source declarative disk configuration system for Kubernetes helping to automate CRUD (Create, Read, Update, Delete) operations for cloud disk device resources attached to Kubernetes cluster nodes." 
-LABEL org.opencontainers.image.documentation "https://github.com/ondat/discoblocks/wiki" 
+LABEL org.opencontainers.image.title "Discoblocks"
+LABEL org.opencontainers.image.vendor "Discoblocks.io"
+LABEL org.opencontainers.image.licenses "Apache-2.0 License"
+LABEL org.opencontainers.image.source "https://github.com/ondat/discoblocks"
+LABEL org.opencontainers.image.description "Discoblocks is an open-source declarative disk configuration system for Kubernetes helping to automate CRUD (Create, Read, Update, Delete) operations for cloud disk device resources attached to Kubernetes cluster nodes."
+LABEL org.opencontainers.image.documentation "https://github.com/ondat/discoblocks/wiki"
 
 WORKDIR /
-COPY --from=builder /workspace/manager .
 COPY --from=drivers /go/src /drivers
+COPY --from=builder /workspace/manager .
 COPY --from=builder /go/pkg/mod/github.com/wasmerio/[email protected]/wasmer/packaged/lib/linux-amd64/libwasmer.so /lib64
 COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 

config/certmanager/certificate.yaml

@@ -22,4 +22,18 @@ spec:
   issuerRef:
     kind: Issuer
     name: selfsigned-issuer
-  secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize
+  secretName: discoblocks-webhook-cert # this secret will not be prefixed, since it's not managed by kustomize
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: metrics-cert
+  namespace: system
+spec:
+  dnsNames:
+  - discoblocks-proxy-service.kube-system.svc
+  - discoblocks-proxy-service.kube-system.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: discoblocks-metrics-cert # this secret will not be prefixed, since it's not managed by kustomize

config/default/manager_webhook_patch.yaml

@@ -14,10 +14,10 @@ spec:
           protocol: TCP
         volumeMounts:
         - mountPath: /tmp/k8s-webhook-server/serving-certs
-          name: cert
+          name: webhook-cert
           readOnly: true
       volumes:
-      - name: cert
+      - name: webhook-cert
         secret:
           defaultMode: 420
-          secretName: webhook-server-cert
+          secretName: discoblocks-webhook-cert

config/manager/kustomization.yaml

@@ -1,6 +1,7 @@
 resources:
 - config.yaml
 - manager.yaml
+- service.yaml
 
 generatorOptions:
   disableNameSuffixHash: true

config/manager/manager.yaml

@@ -25,8 +25,9 @@ spec:
       labels:
         control-plane: controller-manager
     spec:
-      securityContext:
-        runAsNonRoot: true
+      # We have to disable enforment because of metrics proxy nixery image
+      # securityContext:
+      #   runAsNonRoot: true
       containers:
       - command:
         - /manager
@@ -46,6 +47,9 @@ spec:
         volumeMounts:
           - name: config-volume
             mountPath: /etc/kubernetes/discoblocks-scheduler
+          - name: metrics-cert
+            mountPath: /tmp/k8s-webhook-server/metrics-certs
+            readOnly: true
         livenessProbe:
           httpGet:
             path: /healthz
@@ -59,15 +63,58 @@ spec:
           initialDelaySeconds: 5
           periodSeconds: 10
         resources:
-          limits:
-            cpu: 500m
-            memory: 512Mi
+          requests:
+            cpu: 20m
+            memory: 256Mi
+      - name: metrics-proxy
+        # Set runAsNonRoot to true once image has replaced
+        image: nixery.dev/shell/frp
+        command:
+        - sh
+        - -c
+        - |
+          cat <<EOF > /tmp/frps.ini
+          [common]
+          ; log_level = trace
+          disable_log_color = true
+          bind_port = 63535
+          proxy_bind_addr = 127.0.0.1
+          enable_prometheus = true
+          user_conn_timeout = 5
+          max_ports_per_client = 1
+          max_pool_count = 1
+          heartbeat_timeout = 10
+          detailed_errors_to_client = true
+          dashboard_addr = 127.0.0.1
+          dashboard_port = 8000
+          tls_only = true
+          tls_enable = true
+          tls_cert_file = /tmp/k8s-webhook-server/metrics-certs/tls.crt
+          tls_key_file = /tmp/k8s-webhook-server/metrics-certs/tls.key
+          tls_trusted_ca_file = /tmp/k8s-webhook-server/metrics-certs/ca.crt
+          EOF
+          trap exit SIGTERM ;
+          while true; do frps -c /tmp/frps.ini & c=$! wait $c; done
+        securityContext:
+          privileged: false
+        volumeMounts:
+        - mountPath: /tmp/k8s-webhook-server/metrics-certs
+          name: metrics-cert
+          readOnly: true
+        ports:
+        - containerPort: 63535
+          protocol: TCP
+        resources:
           requests:
             cpu: 10m
-            memory: 64Mi
+            memory: 128Mi
       serviceAccountName: controller-manager
       terminationGracePeriodSeconds: 10
       volumes:
         - name: config-volume
           configMap:
             name: discoblocks-scheduler
+        - name: metrics-cert
+          secret:
+            defaultMode: 420
+            secretName: discoblocks-metrics-cert

config/manager/service.yaml

@@ -0,0 +1,14 @@
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: proxy-service
+  namespace: system
+spec:
+  ports:
+    - port: 63535
+      protocol: TCP
+      targetPort: 63535
+  selector:
+    app: discoblocks
+    app.kubernetes.io/component: discoblocks

config/rbac/role.yaml

@@ -51,6 +51,12 @@ rules:
   verbs:
   - delete
   - list
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
 - apiGroups:
   - apps
   resources:

config/samples/core_v1_pod.yaml

@@ -4,6 +4,7 @@ metadata:
   labels:
     discoblocks: diskconfig-sample
   name: diskconfig-sample
+  namespace: default
 spec:
   # hostPID: true
   terminationGracePeriodSeconds: 1

config/samples/discoblocks.ondat.io_v1_diskconfig-csi.storageos.com.yaml

@@ -2,6 +2,7 @@ apiVersion: discoblocks.ondat.io/v1
 kind: DiskConfig
 metadata:
   name: diskconfig-sample
+  namespace: default
 spec:
   storageClassName: storageos
   capacity: 1Gi

controllers/diskconfig_controller.go

@@ -87,7 +87,7 @@ func (r *DiskConfigReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	case err != nil && apierrors.IsNotFound(err):
 		logger.Info("DiskConfig not found")
 
-		return ctrl.Result{}, nil
+		return r.reconcileDelete(ctx, req.Name, req.Namespace, logger.WithValues("mode", "delete"))
 	case err != nil:
 		metrics.NewError("DiskConfig", req.Name, req.Namespace, "Kube API", "get")
 

controllers/pvc_controller.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/go-logr/logr"
 	discoblocksondatiov1 "github.com/ondat/discoblocks/api/v1"
+	"github.com/ondat/discoblocks/pkg/diskinfo"
 	"github.com/ondat/discoblocks/pkg/drivers"
 	"github.com/ondat/discoblocks/pkg/metrics"
 	"github.com/ondat/discoblocks/pkg/utils"
@@ -294,7 +295,8 @@ func (r *PVCReconciler) MonitorVolumes() {
 		for p := range pods.Items {
 			pod := pods.Items[p]
 
-			if pod.DeletionTimestamp != nil {
+			// Skip monitoring of new Pods
+			if pod.DeletionTimestamp != nil || pod.CreationTimestamp.Add(config.Spec.Policy.CoolDown.Duration).After(time.Now()) {
 				continue
 			}
 
@@ -316,7 +318,7 @@ func (r *PVCReconciler) MonitorVolumes() {
 
 				logger.Info("Fetch DiskInfo...")
 
-				diskInfo, err := utils.FetchDiskInfo(fmt.Sprintf("%s:9100", pod.Status.PodIP))
+				diskInfo, err := diskinfo.Fetch(pod.Name, pod.Namespace)
 				if err != nil {
 					metrics.NewError("Pod", pod.Name, pod.Namespace, "DiscoBlocks", "metrics")
 
@@ -394,7 +396,7 @@ func (r *PVCReconciler) MonitorVolumes() {
 
 						logger.Error(err, "Unable to find metrics", "disk_info", diskInfo)
 
-						if err := r.EventService.SendWarning(pod.Namespace, "Discoblocks", "PVC Monitor", fmt.Sprintf("Failed to find metrics of %s: %s", lastPVC.Name, lastMountPoint), err.Error(), &pod, nil); err != nil {
+						if err := r.EventService.SendWarning(pod.Namespace, "Discoblocks", "PVC Monitor", fmt.Sprintf("Failed to find metrics of %s: %s", lastPVC.Name, lastMountPoint), "Unable to find metrics", &pod, nil); err != nil {
 							metrics.NewError("Event", "", "", "Kube API", "create")
 
 							logger.Error(err, "Failed to create event")